diff --git a/cmd/template_resourceworkloads.go b/cmd/template_resourceworkloads.go new file mode 100644 index 00000000..3e52f58b --- /dev/null +++ b/cmd/template_resourceworkloads.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + generator "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + hpatemplate "github.com/uselagoon/build-deploy-tool/internal/templating/resources/hpa" + pdbtemplate "github.com/uselagoon/build-deploy-tool/internal/templating/resources/pdb" +) + +var resourceWorkloadGeneration = &cobra.Command{ + Use: "resource-workloads", + Aliases: []string{"rw"}, + Short: "Generate the resource workload templates for a Lagoon build", + RunE: func(cmd *cobra.Command, args []string) error { + generator, err := generator.GenerateInput(*rootCmd, true) + if err != nil { + return err + } + return ResourceWorkloadTemplateGeneration(generator) + }, +} + +// IngressTemplateGeneration . +func ResourceWorkloadTemplateGeneration(g generator.GeneratorInput) error { + lagoonBuild, err := generator.NewGenerator( + g, + ) + if err != nil { + return err + } + savedTemplates := g.SavedTemplatesPath + + // generate the templates + if g.Debug { + fmt.Println(fmt.Sprintf("Templating HPA manifests to %s", fmt.Sprintf("%s/%s.yaml", savedTemplates, "hpas"))) + } + templateYAML, err := hpatemplate.GenerateHPATemplate(*lagoonBuild.BuildValues) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "hpas"), templateYAML) + if g.Debug { + fmt.Println(fmt.Sprintf("Templating HPA manifests to %s", fmt.Sprintf("%s/%s.yaml", savedTemplates, "pdbs"))) + } + templateYAML, err = pdbtemplate.GeneratePDBTemplate(*lagoonBuild.BuildValues) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "pdbs"), templateYAML) + return nil +} + +func init() { + templateCmd.AddCommand(resourceWorkloadGeneration) +} diff --git a/cmd/template_resourceworkloads_test.go b/cmd/template_resourceworkloads_test.go new file mode 100644 index 00000000..d1612e4d --- /dev/null +++ b/cmd/template_resourceworkloads_test.go @@ -0,0 +1,160 @@ +package cmd + +import ( + "fmt" + "os" + "reflect" + "testing" + + "github.com/uselagoon/build-deploy-tool/internal/helpers" + "github.com/uselagoon/build-deploy-tool/internal/testdata" + + // changes the testing to source from root so paths to test resources must be defined from repo root + _ "github.com/uselagoon/build-deploy-tool/internal/testing" +) + +func TestResourceWorkloadTemplateGeneration(t *testing.T) { + tests := []struct { + name string + args testdata.TestData + templatePath string + workloadJSONfile string + resourceWorkloadOverrides string + want string + }{ + { + name: "test1 no resource workloads", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/node/lagoon.yml", + }, true), + templatePath: "testdata/output", + want: "internal/testdata/node/resource-templates/resource1", + }, + { + name: "test2 node hpa", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/node/lagoon.resources1.yml", + }, true), + templatePath: "testdata/output", + workloadJSONfile: "internal/testdata/node/workload.resources1.json", + want: "internal/testdata/node/resource-templates/resource2", + }, + { + name: "test3 nginx hpa and pdb", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/complex/lagoon.resource1.yml", + }, true), + templatePath: "testdata/output", + workloadJSONfile: "internal/testdata/complex/workload.resources1.json", + want: "internal/testdata/complex/resource-templates/resource1", + }, + { + name: "test4 nginx hpa and pdb with resource override from feature flag", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/complex/lagoon.yml", + }, true), + templatePath: "testdata/output", + workloadJSONfile: "internal/testdata/complex/workload.resources2.json", + resourceWorkloadOverrides: "nginx-php:nginx-php-performance", + want: "internal/testdata/complex/resource-templates/resource2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // set the environment variables from args + err := os.Setenv("LAGOON_FEATURE_FLAG_DEFAULT_WORKLOAD_RESOURCES", helpers.ReadFileBase64Encode(tt.workloadJSONfile)) + if err != nil { + t.Errorf("%v", err) + } + err = os.Setenv("LAGOON_FEATURE_FLAG_DEFAULT_WORKLOAD_RESOURCE_TYPES", tt.resourceWorkloadOverrides) + if err != nil { + t.Errorf("%v", err) + } + savedTemplates := tt.templatePath + generator, err := testdata.SetupEnvironment(*rootCmd, savedTemplates, tt.args) + if err != nil { + t.Errorf("%v", err) + } + + err = os.MkdirAll(savedTemplates, 0755) + if err != nil { + t.Errorf("couldn't create directory %v: %v", savedTemplates, err) + } + + defer os.RemoveAll(savedTemplates) + + err = ResourceWorkloadTemplateGeneration(generator) + if err != nil { + t.Errorf("%v", err) + } + + files, err := os.ReadDir(savedTemplates) + if err != nil { + t.Errorf("couldn't read directory %v: %v", savedTemplates, err) + } + results, err := os.ReadDir(tt.want) + if err != nil { + t.Errorf("couldn't read directory %v: %v", tt.want, err) + } + if len(files) != len(results) { + for _, f := range files { + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + fmt.Println(string(f1)) + } + t.Errorf("number of generated templates doesn't match results %v/%v: %v", len(files), len(results), err) + } + fCount := 0 + for _, f := range files { + for _, r := range results { + if f.Name() == r.Name() { + fCount++ + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + r1, err := os.ReadFile(fmt.Sprintf("%s/%s", tt.want, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + if !reflect.DeepEqual(f1, r1) { + fmt.Println(string(f1)) + t.Errorf("resulting templates do not match") + } + } + } + } + if fCount != len(files) { + for _, f := range files { + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + fmt.Println(string(f1)) + } + t.Errorf("resulting templates do not match") + } + t.Cleanup(func() { + helpers.UnsetEnvVars([]helpers.EnvironmentVariable{{Name: "LAGOON_FEATURE_FLAG_DEFAULT_INGRESS_CLASS"}}) + }) + }) + } +} diff --git a/internal/generator/buildvalues.go b/internal/generator/buildvalues.go index ed1be72b..b576c9f5 100644 --- a/internal/generator/buildvalues.go +++ b/internal/generator/buildvalues.go @@ -4,7 +4,9 @@ import ( composetypes "github.com/compose-spec/compose-go/types" "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/lagoon" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" ) const ( @@ -83,6 +85,8 @@ type BuildValues struct { ForcePullImages []string `json:"forcePullImages"` Volumes []ComposeVolume `json:"volumes,omitempty" description:"stores any additional persistent volume definitions"` PodAntiAffinity bool `json:"podAntiAffinity"` + ResourceWorkloads map[string]ResourceWorkloads `json:"resourceWorkloads"` + ResourceWorkloadOverrides string `json:"resourceWorkloadOverrides"` } type Resources struct { @@ -117,6 +121,28 @@ type PodSecurityContext struct { OnRootMismatch bool `json:"onRootMismatch"` } +type ResourceWorkloads struct { + ServiceType string `json:"serviceType"` + HPA *HPASpec `json:"hpa,omitempty"` + PDB *PDBSpec `json:"pdb,omitempty"` + Resources []Resource `json:"resources"` +} + +type Resource struct { + Name string `json:"name"` + Resources corev1.ResourceRequirements `json:"resources"` +} + +type HPASpec struct { + Spec autoscalingv2.HorizontalPodAutoscalerSpec `json:"spec"` +} + +type PDBSpec struct { + Spec policyv1.PodDisruptionBudgetSpec `json:"spec"` +} + +// type Resources map[string]corev1.ResourceRequirements + type Fastly struct { ServiceID string `json:"serviceId"` APISecretName string `json:"apiSecretName"` @@ -200,6 +226,7 @@ type ServiceValues struct { IsDBaaS bool `json:"isDBaaS"` IsSingle bool `json:"isSingle"` AdditionalVolumes []ServiceVolume `json:"additonalVolumes,omitempty"` + ResourceWorkload string `json:"resourceWorkload,omitempty"` } type ImageBuild struct { diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 38182ee3..6548bfce 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -373,6 +373,9 @@ func NewGenerator( lagoonDBaaSEnvironmentTypes, _ := lagoon.GetLagoonVariable("LAGOON_DBAAS_ENVIRONMENT_TYPES", nil, buildValues.EnvironmentVariables) buildValues.DBaaSEnvironmentTypeOverrides = lagoonDBaaSEnvironmentTypes + lagoonResourceWorkloads := CheckFeatureFlag("WORKLOAD_RESOURCE_TYPES", buildValues.EnvironmentVariables, generator.Debug) + buildValues.ResourceWorkloadOverrides = lagoonResourceWorkloads + // check autogenerated routes for fastly `LAGOON_FEATURE_FLAG(_FORCE|_DEFAULT)_FASTLY_AUTOGENERATED` using feature flags // @TODO: eventually deprecate fastly functionality in favour of a more generic implementation autogeneratedRoutesFastly := CheckFeatureFlag("FASTLY_AUTOGENERATED", buildValues.EnvironmentVariables, generator.Debug) @@ -412,6 +415,11 @@ func NewGenerator( } /* end backups configuration */ + /* calculate resource workloads */ + resourceWorkloads, err := getResourcesFromAPIEnvVar(buildValues.EnvironmentVariables, generator.Debug) + buildValues.ResourceWorkloads = *resourceWorkloads + /* end resource workload calculation */ + /* start compose->service configuration !! IMPORTANT !! diff --git a/internal/generator/resources.go b/internal/generator/resources.go new file mode 100644 index 00000000..e9fbcc86 --- /dev/null +++ b/internal/generator/resources.go @@ -0,0 +1,41 @@ +package generator + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/uselagoon/build-deploy-tool/internal/lagoon" +) + +func getResourcesFromAPIEnvVar( + envVars []lagoon.EnvironmentVariable, + debug bool, +) (*map[string]ResourceWorkloads, error) { + resWorkloads := &map[string]ResourceWorkloads{} + // TODO: this is still to be determined how the data will be consumed from the API, it may eventually come from + // a configmap or some other means, or a combination of configmap and envvar merging + // for now, consume from featureflag var + resourceWorkloadsJSON := CheckFeatureFlag("WORKLOAD_RESOURCES", envVars, debug) + // only from envvar from api, not feature flagable + // resourceWorkloadsJSONvar, _ := lagoon.GetLagoonVariable("LAGOON_WORKLOAD_RESOURCES", []string{"build", "global"}, envVars) + // if resourceWorkloadsJSONvar != nil { + // resourceWorkloadsJSON = resourceWorkloadsJSONvar.Value + // } + if resourceWorkloadsJSON != "" { + if debug { + fmt.Println("Collecting resource workloads from WORKLOAD_RESOURCES variable") + } + // if the routesJSON is populated, then attempt to decode and unmarshal it + rawJSONStr, err := base64.StdEncoding.DecodeString(resourceWorkloadsJSON) + if err != nil { + return nil, fmt.Errorf("couldn't decode resource workloads from Lagoon API, is it actually base64 encoded?: %v", err) + } + rawJSON := []byte(rawJSONStr) + err = json.Unmarshal(rawJSON, resWorkloads) + if err != nil { + return nil, fmt.Errorf("couldn't unmarshal resource workloads from Lagoon API, is it actually JSON that has been base64 encoded?: %v", err) + } + } + return resWorkloads, nil +} diff --git a/internal/generator/resources_test.go b/internal/generator/resources_test.go new file mode 100644 index 00000000..014ff5f3 --- /dev/null +++ b/internal/generator/resources_test.go @@ -0,0 +1,163 @@ +package generator + +import ( + "encoding/json" + "reflect" + "testing" + + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/uselagoon/build-deploy-tool/internal/helpers" + "github.com/uselagoon/build-deploy-tool/internal/lagoon" +) + +func Test_getResourcesFromAPIEnvVar(t *testing.T) { + type args struct { + envVars []lagoon.EnvironmentVariable + debug bool + } + tests := []struct { + name string + args args + want *map[string]ResourceWorkloads + wantErr bool + }{ + { + name: "test1 - check that a scaling parameters are correctly defined", + args: args{ + envVars: []lagoon.EnvironmentVariable{ + { + Name: "LAGOON_FEATURE_FLAG_WORKLOAD_RESOURCES", + Value: helpers.ReadFileBase64Encode("test-resources/resources/test1-workload.json"), + Scope: "global", + }, + }, + }, + want: &map[string]ResourceWorkloads{ + "nginx": { + ServiceType: "nginx", + HPA: &HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(8), + MaxReplicas: *helpers.Int32Ptr(16), + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(3000), + }, + }, + }, + }, + }, + }, + PDB: &PDBSpec{ + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + IntVal: 1, + Type: intstr.Int, + }, + }, + }, + Resources: []Resource{ + { + Name: "php", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + }, + { + Name: "nginx", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, + }, + }, + }, + }, + }, + { + name: "test2 - check that a scaling parameters are correctly defined for multiple services", + args: args{ + envVars: []lagoon.EnvironmentVariable{ + { + Name: "LAGOON_FEATURE_FLAG_WORKLOAD_RESOURCES", + Value: helpers.ReadFileBase64Encode("test-resources/resources/test2-workload.json"), + Scope: "global", + }, + }, + }, + want: &map[string]ResourceWorkloads{ + "nginx": { + ServiceType: "nginx", + HPA: &HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(8), + MaxReplicas: *helpers.Int32Ptr(16), + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(3000), + }, + }, + }, + }, + }, + }, + }, + "node": { + ServiceType: "node", + HPA: &HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(8), + MaxReplicas: *helpers.Int32Ptr(16), + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(3000), + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getResourcesFromAPIEnvVar(tt.args.envVars, tt.args.debug) + if (err != nil) != tt.wantErr { + t.Errorf("getResourcesFromAPIEnvVar() error = %v, wantErr %v", err, tt.wantErr) + return + } + lValues, _ := json.Marshal(got) + wValues, _ := json.Marshal(tt.want) + if !reflect.DeepEqual(string(lValues), string(wValues)) { + t.Errorf("getResourcesFromAPIEnvVar() = %v, want %v", string(lValues), string(wValues)) + } + }) + } +} diff --git a/internal/generator/services.go b/internal/generator/services.go index f3e8908e..51c91b5a 100644 --- a/internal/generator/services.go +++ b/internal/generator/services.go @@ -239,7 +239,7 @@ func composeToServiceValues( // if there is an override name, check all other services already existing for _, service := range buildValues.Services { // if there is an existing service with this same override name, then disable autogenerated routes - // for this service + // for this service as it is probably a dual container service (eg, nginx-php/nginx-php-persistent) if service.OverrideName == lagoonOverrideName { autogenEnabled = false } @@ -498,6 +498,34 @@ func composeToServiceValues( } } + // check if the service has a workloadresource request + serviceResourceWorkload := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.workloadresource") + // check for an api override on the resource workload or if one isn't found in the docker-compose file + getResourceWorkloadOverride(buildValues, &serviceResourceWorkload, lagoonOverrideName, lagoonType, debug) + for _, service := range buildValues.Services { + // if there is an existing service with this same override name, then disable resource workloads for this service as it is probably a + // dual container service (eg, nginx-php/nginx-php-persistent) + if service.OverrideName == lagoonOverrideName { + serviceResourceWorkload = "" + } + } + if serviceResourceWorkload != "" { + // check if the requested resource workload actually exists + resWork, ok := buildValues.ResourceWorkloads[serviceResourceWorkload] + if !ok { + return nil, fmt.Errorf( + "the requested scaling parameter group %s for service %s is not valid", + serviceResourceWorkload, composeService, + ) + } + if resWork.ServiceType != lagoonType { + return nil, fmt.Errorf( + "the requested scaling parameter group %s for service %s is not valid for this service type %s", + serviceResourceWorkload, composeService, lagoonType, + ) + } + } + // check if this service is one that supports autogenerated routes if !helpers.Contains(supportedAutogeneratedTypes, lagoonType) { autogenEnabled = false @@ -535,6 +563,7 @@ func composeToServiceValues( IsSingle: svcIsSingle, BackupsEnabled: backupsEnabled, AdditionalVolumes: serviceVolumes, + ResourceWorkload: serviceResourceWorkload, } // work out the images here and the associated dockerfile and contexts @@ -573,3 +602,22 @@ func composeToServiceValues( return cService, nil } } + +// getResourceWorkloadOverride will check the api variables for a resource override +func getResourceWorkloadOverride( + buildValues *BuildValues, + workloadOverrides *string, + lagoonOverrideName, + lagoonType string, + debug bool, +) { + if buildValues.ResourceWorkloadOverrides != "" { + workloadOverridesSplit := strings.Split(buildValues.ResourceWorkloadOverrides, ",") + for _, sType := range workloadOverridesSplit { + sTypeSplit := strings.Split(sType, ":") + if sTypeSplit[0] == lagoonOverrideName { + *workloadOverrides = sTypeSplit[1] + } + } + } +} diff --git a/internal/generator/services_test.go b/internal/generator/services_test.go index 94d84323..ccc98b9a 100644 --- a/internal/generator/services_test.go +++ b/internal/generator/services_test.go @@ -10,6 +10,8 @@ import ( "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/helpers" "github.com/uselagoon/build-deploy-tool/internal/lagoon" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" ) func Test_composeToServiceValues(t *testing.T) { @@ -1236,6 +1238,177 @@ func Test_composeToServiceValues(t *testing.T) { want: nil, wantErr: true, }, + { + name: "test24 - scaling parameter exists", + args: args{ + buildValues: &BuildValues{ + Namespace: "example-project-main", + Project: "example-project", + ImageRegistry: "harbor.example", + Environment: "main", + Branch: "main", + BuildType: "branch", + ServiceTypeOverrides: &lagoon.EnvironmentVariable{}, + LagoonYAML: lagoon.YAML{ + Environments: lagoon.Environments{ + "main": lagoon.Environment{}, + }, + }, + ResourceWorkloads: map[string]ResourceWorkloads{ + "nginx": { + ServiceType: "nginx", + HPA: &HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(1), + MaxReplicas: 2, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(3000), + }, + }, + }, + }, + }, + }, + }, + }, + }, + composeService: "nginx", + composeServiceValues: composetypes.ServiceConfig{ + Labels: composetypes.Labels{ + "lagoon.type": "nginx", + "lagoon.workloadresource": "nginx", + }, + Build: &composetypes.BuildConfig{ + Context: ".", + Dockerfile: "../testdata/basic/docker/basic.dockerfile", //intentionally wrong, override by lagoon.yml + }, + }, + }, + want: &ServiceValues{ + Name: "nginx", + OverrideName: "nginx", + Type: "nginx", + AutogeneratedRoutesEnabled: true, + AutogeneratedRoutesTLSAcme: true, + ResourceWorkload: "nginx", + InPodCronjobs: []lagoon.Cronjob{}, + NativeCronjobs: []lagoon.Cronjob{}, + ImageBuild: &ImageBuild{ + TemporaryImage: "example-project-main-nginx", + Context: ".", + DockerFile: "../testdata/basic/docker/basic.dockerfile", + BuildImage: "harbor.example/example-project/main/nginx:latest", + }, + }, + }, + { + name: "test25 - scaling parameter doesn't exist (error)", + args: args{ + buildValues: &BuildValues{ + Namespace: "example-project-main", + Project: "example-project", + ImageRegistry: "harbor.example", + Environment: "main", + Branch: "main", + BuildType: "branch", + ServiceTypeOverrides: &lagoon.EnvironmentVariable{}, + LagoonYAML: lagoon.YAML{ + Environments: lagoon.Environments{ + "main": lagoon.Environment{}, + }, + }, + ResourceWorkloads: map[string]ResourceWorkloads{}, + }, + composeService: "nginx", + composeServiceValues: composetypes.ServiceConfig{ + Labels: composetypes.Labels{ + "lagoon.type": "nginx", + "lagoon.workloadresource": "nginx", + }, + Build: &composetypes.BuildConfig{ + Context: ".", + Dockerfile: "../testdata/basic/docker/basic.dockerfile", //intentionally wrong, override by lagoon.yml + }, + }, + }, + want: nil, + wantErr: true, + }, + { + name: "test26 - scaling parameter exists, but only in envvar", + args: args{ + buildValues: &BuildValues{ + Namespace: "example-project-main", + Project: "example-project", + ImageRegistry: "harbor.example", + Environment: "main", + Branch: "main", + BuildType: "branch", + ServiceTypeOverrides: &lagoon.EnvironmentVariable{}, + LagoonYAML: lagoon.YAML{ + Environments: lagoon.Environments{ + "main": lagoon.Environment{}, + }, + }, + ResourceWorkloadOverrides: "nginx:nginx-hpa", + ResourceWorkloads: map[string]ResourceWorkloads{ + "nginx-hpa": { + ServiceType: "nginx", + HPA: &HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(1), + MaxReplicas: 2, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(3000), + }, + }, + }, + }, + }, + }, + }, + }, + }, + composeService: "nginx", + composeServiceValues: composetypes.ServiceConfig{ + Labels: composetypes.Labels{ + "lagoon.type": "nginx", + }, + Build: &composetypes.BuildConfig{ + Context: ".", + Dockerfile: "../testdata/basic/docker/basic.dockerfile", //intentionally wrong, override by lagoon.yml + }, + }, + }, + want: &ServiceValues{ + Name: "nginx", + OverrideName: "nginx", + Type: "nginx", + AutogeneratedRoutesEnabled: true, + AutogeneratedRoutesTLSAcme: true, + ResourceWorkload: "nginx-hpa", + InPodCronjobs: []lagoon.Cronjob{}, + NativeCronjobs: []lagoon.Cronjob{}, + ImageBuild: &ImageBuild{ + TemporaryImage: "example-project-main-nginx", + Context: ".", + DockerFile: "../testdata/basic/docker/basic.dockerfile", + BuildImage: "harbor.example/example-project/main/nginx:latest", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/generator/test-resources/resources/test1-workload.json b/internal/generator/test-resources/resources/test1-workload.json new file mode 100644 index 00000000..8d5a2c3f --- /dev/null +++ b/internal/generator/test-resources/resources/test1-workload.json @@ -0,0 +1,48 @@ +{ + "nginx": { + "serviceType": "nginx", + "hpa": { + "spec": { + "minReplicas": 8, + "maxReplicas": 16, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 3000 + } + } + } + ] + } + }, + "pdb": { + "spec": { + "minAvailable": 1 + } + }, + "resources": [ + { + "name": "php", + "resources": { + "requests": { + "cpu": "10m", + "memory": "10Mi" + } + } + }, + { + "name": "nginx", + "resources": { + "requests": { + "cpu": "10m", + "memory": "10Mi" + } + } + } + ] + } +} \ No newline at end of file diff --git a/internal/generator/test-resources/resources/test2-workload.json b/internal/generator/test-resources/resources/test2-workload.json new file mode 100644 index 00000000..517aaf71 --- /dev/null +++ b/internal/generator/test-resources/resources/test2-workload.json @@ -0,0 +1,44 @@ +{ + "nginx": { + "serviceType": "nginx", + "hpa": { + "spec": { + "minReplicas": 8, + "maxReplicas": 16, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 3000 + } + } + } + ] + } + } + }, + "node": { + "serviceType": "node", + "hpa": { + "spec": { + "minReplicas": 8, + "maxReplicas": 16, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 3000 + } + } + } + ] + } + } + } +} \ No newline at end of file diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 8a01db88..865cf881 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -4,6 +4,7 @@ import ( "crypto/md5" "crypto/sha256" "encoding/base32" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -292,3 +293,14 @@ func AppendIfMissing(slice []string, i string) []string { } return append(slice, i) } + +// helper function to read a file and base64 encode the result +// returns an empty string on any error +// mainly used in tests to consume JSON or YAML test resources +func ReadFileBase64Encode(file string) string { + raw, err := os.ReadFile(file) + if err != nil { + return "" + } + return base64.StdEncoding.EncodeToString(raw) +} diff --git a/internal/templating/backups/template_prebackuppod.go b/internal/templating/backups/template_prebackuppod.go index 43ce0c9c..051b5a03 100644 --- a/internal/templating/backups/template_prebackuppod.go +++ b/internal/templating/backups/template_prebackuppod.go @@ -59,8 +59,8 @@ func GeneratePreBackupPod( additionalAnnotations["lagoon.sh/prBaseBranch"] = lValues.PRBaseBranch } additionalLabels["app.kubernetes.io/name"] = serviceValues.Type - additionalLabels["app.kubernetes.io/instance"] = serviceValues.Name - additionalLabels["lagoon.sh/service"] = serviceValues.Name + additionalLabels["app.kubernetes.io/instance"] = serviceValues.OverrideName + additionalLabels["lagoon.sh/service"] = serviceValues.OverrideName additionalLabels["lagoon.sh/service-type"] = serviceValues.Type if _, ok := preBackupPodSpecs[serviceValues.Type]; ok { switch lValues.Backup.K8upVersion { @@ -71,14 +71,14 @@ func GeneratePreBackupPod( APIVersion: k8upv1alpha1.GroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-prebackuppod", serviceValues.Name), + Name: fmt.Sprintf("%s-prebackuppod", serviceValues.OverrideName), }, Spec: k8upv1alpha1.PreBackupPodSpec{}, } prebackuppod.ObjectMeta.Labels = labels prebackuppod.ObjectMeta.Annotations = annotations - prebackuppod.ObjectMeta.Labels["prebackuppod"] = serviceValues.Name + prebackuppod.ObjectMeta.Labels["prebackuppod"] = serviceValues.OverrideName var pbp bytes.Buffer tmpl, _ := template.New("").Funcs(funcMap).Parse(preBackupPodSpecs[serviceValues.Type]) diff --git a/internal/templating/dbaas/template_dbaas.go b/internal/templating/dbaas/template_dbaas.go index 74325022..1a979c01 100644 --- a/internal/templating/dbaas/template_dbaas.go +++ b/internal/templating/dbaas/template_dbaas.go @@ -57,9 +57,9 @@ func GenerateDBaaSTemplate( if helpers.Contains(dbaasTypes, serviceValues.Type) { var consumerBytes []byte additionalLabels["app.kubernetes.io/name"] = serviceValues.Type - additionalLabels["app.kubernetes.io/instance"] = serviceValues.Name + additionalLabels["app.kubernetes.io/instance"] = serviceValues.OverrideName additionalLabels["lagoon.sh/template"] = fmt.Sprintf("%s-%s", serviceValues.Type, "0.1.0") - additionalLabels["lagoon.sh/service"] = serviceValues.Name + additionalLabels["lagoon.sh/service"] = serviceValues.OverrideName additionalLabels["lagoon.sh/service-type"] = serviceValues.Type switch serviceValues.Type { case "mariadb-dbaas": @@ -70,7 +70,7 @@ func GenerateDBaaSTemplate( APIVersion: mariadbv1.GroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceValues.Name, + Name: serviceValues.OverrideName, }, Spec: mariadbv1.MariaDBConsumerSpec{ Environment: serviceValues.DBaaSEnvironment, @@ -88,13 +88,13 @@ func GenerateDBaaSTemplate( // validate any annotations if err := apivalidation.ValidateAnnotations(mariaDBConsumer.ObjectMeta.Annotations, nil); err != nil { if len(err) != 0 { - return nil, fmt.Errorf("the annotations for %s are not valid: %v", serviceValues.Name, err) + return nil, fmt.Errorf("the annotations for %s are not valid: %v", serviceValues.OverrideName, err) } } // validate any labels if err := metavalidation.ValidateLabels(mariaDBConsumer.ObjectMeta.Labels, nil); err != nil { if len(err) != 0 { - return nil, fmt.Errorf("the labels for %s are not valid: %v", serviceValues.Name, err) + return nil, fmt.Errorf("the labels for %s are not valid: %v", serviceValues.OverrideName, err) } } @@ -116,7 +116,7 @@ func GenerateDBaaSTemplate( APIVersion: mongodbv1.GroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceValues.Name, + Name: serviceValues.OverrideName, }, Spec: mongodbv1.MongoDBConsumerSpec{ Environment: serviceValues.DBaaSEnvironment, @@ -134,13 +134,13 @@ func GenerateDBaaSTemplate( // validate any annotations if err := apivalidation.ValidateAnnotations(mongodbConsumer.ObjectMeta.Annotations, nil); err != nil { if len(err) != 0 { - return nil, fmt.Errorf("the annotations for %s are not valid: %v", serviceValues.Name, err) + return nil, fmt.Errorf("the annotations for %s are not valid: %v", serviceValues.OverrideName, err) } } // validate any labels if err := metavalidation.ValidateLabels(mongodbConsumer.ObjectMeta.Labels, nil); err != nil { if len(err) != 0 { - return nil, fmt.Errorf("the labels for %s are not valid: %v", serviceValues.Name, err) + return nil, fmt.Errorf("the labels for %s are not valid: %v", serviceValues.OverrideName, err) } } // check length of labels @@ -161,7 +161,7 @@ func GenerateDBaaSTemplate( APIVersion: postgresv1.GroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceValues.Name, + Name: serviceValues.OverrideName, }, Spec: postgresv1.PostgreSQLConsumerSpec{ Environment: serviceValues.DBaaSEnvironment, @@ -179,13 +179,13 @@ func GenerateDBaaSTemplate( // validate any annotations if err := apivalidation.ValidateAnnotations(postgresqlConsumer.ObjectMeta.Annotations, nil); err != nil { if len(err) != 0 { - return nil, fmt.Errorf("the annotations for %s are not valid: %v", serviceValues.Name, err) + return nil, fmt.Errorf("the annotations for %s are not valid: %v", serviceValues.OverrideName, err) } } // validate any labels if err := metavalidation.ValidateLabels(postgresqlConsumer.ObjectMeta.Labels, nil); err != nil { if len(err) != 0 { - return nil, fmt.Errorf("the labels for %s are not valid: %v", serviceValues.Name, err) + return nil, fmt.Errorf("the labels for %s are not valid: %v", serviceValues.OverrideName, err) } } diff --git a/internal/templating/resources/hpa/template_hpa.go b/internal/templating/resources/hpa/template_hpa.go new file mode 100644 index 00000000..f1335ac5 --- /dev/null +++ b/internal/templating/resources/hpa/template_hpa.go @@ -0,0 +1,119 @@ +package hpa + +import ( + "fmt" + + "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + + autoscalingv2 "k8s.io/api/autoscaling/v2" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + + "sigs.k8s.io/yaml" +) + +func GenerateHPATemplate( + lValues generator.BuildValues, +) ([]byte, error) { + // generate the template spec + + var result []byte + separator := []byte("---\n") + + // add the default labels + labels := map[string]string{ + "app.kubernetes.io/managed-by": "build-deploy-tool", + "lagoon.sh/project": lValues.Project, + "lagoon.sh/environment": lValues.Environment, + "lagoon.sh/environmentType": lValues.EnvironmentType, + "lagoon.sh/buildType": lValues.BuildType, + } + + // add the default annotations + annotations := map[string]string{ + "lagoon.sh/version": lValues.LagoonVersion, + } + + // create the hpas + for _, serviceValues := range lValues.Services { + if serviceValues.ResourceWorkload != "" && (lValues.ResourceWorkloads[serviceValues.ResourceWorkload].HPA != nil) { + // add any additional labels + additionalLabels := map[string]string{} + additionalAnnotations := map[string]string{} + if lValues.BuildType == "branch" { + additionalAnnotations["lagoon.sh/branch"] = lValues.Branch + } else if lValues.BuildType == "pullrequest" { + additionalAnnotations["lagoon.sh/prNumber"] = lValues.PRNumber + additionalAnnotations["lagoon.sh/prHeadBranch"] = lValues.PRHeadBranch + additionalAnnotations["lagoon.sh/prBaseBranch"] = lValues.PRBaseBranch + } + additionalLabels["app.kubernetes.io/name"] = serviceValues.Type + additionalLabels["app.kubernetes.io/instance"] = serviceValues.OverrideName + additionalLabels["lagoon.sh/service"] = serviceValues.OverrideName + additionalLabels["lagoon.sh/service-type"] = serviceValues.Type + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + Kind: "HorizontalPodAutoscaler", + APIVersion: autoscalingv2.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-hpa", serviceValues.OverrideName), + }, + Spec: lValues.ResourceWorkloads[serviceValues.ResourceWorkload].HPA.Spec, + } + + // set the scale target to the service that requested it + // since all lagoon deployed services are deployments at the moment + // default this set to the deployment kind, refactor in the future if lagoon supports + // additional types (statefulsets/daemonsets?) + hpa.Spec.ScaleTargetRef = autoscalingv2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: serviceValues.OverrideName, + APIVersion: "apps/v1", + } + + hpa.ObjectMeta.Labels = labels + hpa.ObjectMeta.Annotations = annotations + + for key, value := range additionalLabels { + hpa.ObjectMeta.Labels[key] = value + } + // add any additional annotations + for key, value := range additionalAnnotations { + hpa.ObjectMeta.Annotations[key] = value + } + // validate any annotations + if err := apivalidation.ValidateAnnotations(hpa.ObjectMeta.Annotations, nil); err != nil { + if len(err) != 0 { + return nil, fmt.Errorf("the annotations for %s/%s are not valid: %v", "hpa", serviceValues.Name, err) + } + } + // validate any labels + if err := metavalidation.ValidateLabels(hpa.ObjectMeta.Labels, nil); err != nil { + if len(err) != 0 { + return nil, fmt.Errorf("the labels for %s/%s are not valid: %v", "hpa", serviceValues.Name, err) + } + } + + // check length of labels + err := helpers.CheckLabelLength(hpa.ObjectMeta.Labels) + if err != nil { + return nil, err + } + // @TODO: we should review this in the future when we stop doing `kubectl apply` in the builds :) + // marshal the resulting ingress + hpaBytes, err := yaml.Marshal(hpa) + if err != nil { + return nil, err + } + // add the seperator to the template so that it can be `kubectl apply` in bulk as part + // of the current build process + restoreResult := append(separator[:], hpaBytes[:]...) + result = append(result, restoreResult[:]...) + } + } + + return result, nil +} diff --git a/internal/templating/resources/hpa/template_hpa_test.go b/internal/templating/resources/hpa/template_hpa_test.go new file mode 100644 index 00000000..87d9251a --- /dev/null +++ b/internal/templating/resources/hpa/template_hpa_test.go @@ -0,0 +1,149 @@ +package hpa + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" + "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" +) + +func TestGenerateHPATemplate(t *testing.T) { + type args struct { + lValues generator.BuildValues + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test1 - nginx hpa", + args: args{ + lValues: generator.BuildValues{ + Project: "example-project", + Environment: "brancha", + EnvironmentType: "production", + Namespace: "myexample-project-brancha", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "brancha", + Services: []generator.ServiceValues{ + { + Name: "nginx", + OverrideName: "nginx", + Type: "nginx-php-persistent", + ResourceWorkload: "nginx-php-performance", + }, + { + Name: "php", + OverrideName: "nginx", + Type: "nginx-php-persistent", + }, + }, + ResourceWorkloads: map[string]generator.ResourceWorkloads{ + "nginx": { + ServiceType: "nginx", + HPA: &generator.HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(2), + MaxReplicas: *helpers.Int32Ptr(5), + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(3000), + }, + }, + }, + }}, + }, + }, + "nginx-php-performance": { + ServiceType: "nginx-php-persistent", + HPA: &generator.HPASpec{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: helpers.Int32Ptr(8), + MaxReplicas: *helpers.Int32Ptr(16), + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: helpers.Int32Ptr(1500), + }, + }, + }, + }}, + }, + }, + }, + }, + }, + want: "test-resources/result-nginx.yaml", + }, + { + name: "test2 - no resources", + args: args{ + lValues: generator.BuildValues{ + Project: "example-project", + Environment: "brancha", + EnvironmentType: "production", + Namespace: "myexample-project-brancha", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "brancha", + Services: []generator.ServiceValues{ + { + Name: "nginx", + OverrideName: "nginx", + Type: "nginx-php-persistent", + }, + { + Name: "php", + OverrideName: "nginx", + Type: "nginx-php-persistent", + }, + }, + ResourceWorkloads: map[string]generator.ResourceWorkloads{}, + }, + }, + want: "test-resources/result-no-resources.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // add dbaasclient overrides for tests + tt.args.lValues.DBaaSClient = dbaasclient.NewClient(dbaasclient.Client{ + RetryMax: 5, + RetryWaitMin: time.Duration(10) * time.Millisecond, + RetryWaitMax: time.Duration(50) * time.Millisecond, + }) + got, err := GenerateHPATemplate(tt.args.lValues) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateHPATemplate() error = %v, wantErr %v", err, tt.wantErr) + return + } + r1, err := os.ReadFile(tt.want) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + if !reflect.DeepEqual(string(got), string(r1)) { + t.Errorf("GenerateHPATemplate() = %v, want %v", string(got), string(r1)) + } + }) + } +} diff --git a/internal/templating/resources/hpa/test-resources/result-nginx.yaml b/internal/templating/resources/hpa/test-resources/result-nginx.yaml new file mode 100644 index 00000000..e0d3f3c4 --- /dev/null +++ b/internal/templating/resources/hpa/test-resources/result-nginx.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + lagoon.sh/branch: brancha + lagoon.sh/version: v2.x.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: nginx + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: nginx-php-persistent + lagoon.sh/buildType: branch + lagoon.sh/environment: brancha + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: nginx + lagoon.sh/service-type: nginx-php-persistent + name: nginx-hpa +spec: + maxReplicas: 16 + metrics: + - resource: + name: cpu + target: + averageUtilization: 1500 + type: Utilization + type: Resource + minReplicas: 8 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx +status: + currentMetrics: null + desiredReplicas: 0 diff --git a/internal/templating/resources/hpa/test-resources/result-no-resources.yaml b/internal/templating/resources/hpa/test-resources/result-no-resources.yaml new file mode 100644 index 00000000..e69de29b diff --git a/internal/templating/resources/pdb/template_pdb.go b/internal/templating/resources/pdb/template_pdb.go new file mode 100644 index 00000000..74670136 --- /dev/null +++ b/internal/templating/resources/pdb/template_pdb.go @@ -0,0 +1,115 @@ +package pdb + +import ( + "fmt" + + "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + + policyv1 "k8s.io/api/policy/v1" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + + "sigs.k8s.io/yaml" +) + +func GeneratePDBTemplate( + lValues generator.BuildValues, +) ([]byte, error) { + // generate the template spec + + var result []byte + separator := []byte("---\n") + + // add the default labels + labels := map[string]string{ + "app.kubernetes.io/managed-by": "build-deploy-tool", + "lagoon.sh/project": lValues.Project, + "lagoon.sh/environment": lValues.Environment, + "lagoon.sh/environmentType": lValues.EnvironmentType, + "lagoon.sh/buildType": lValues.BuildType, + } + + // add the default annotations + annotations := map[string]string{ + "lagoon.sh/version": lValues.LagoonVersion, + } + + // create the pdbs + for _, serviceValues := range lValues.Services { + if serviceValues.ResourceWorkload != "" && (lValues.ResourceWorkloads[serviceValues.ResourceWorkload].PDB != nil) { + // add any additional labels + additionalLabels := map[string]string{} + additionalAnnotations := map[string]string{} + if lValues.BuildType == "branch" { + additionalAnnotations["lagoon.sh/branch"] = lValues.Branch + } else if lValues.BuildType == "pullrequest" { + additionalAnnotations["lagoon.sh/prNumber"] = lValues.PRNumber + additionalAnnotations["lagoon.sh/prHeadBranch"] = lValues.PRHeadBranch + additionalAnnotations["lagoon.sh/prBaseBranch"] = lValues.PRBaseBranch + } + additionalLabels["app.kubernetes.io/name"] = serviceValues.Type + additionalLabels["app.kubernetes.io/instance"] = serviceValues.OverrideName + additionalLabels["lagoon.sh/service"] = serviceValues.OverrideName + additionalLabels["lagoon.sh/service-type"] = serviceValues.Type + pdb := &policyv1.PodDisruptionBudget{ + TypeMeta: metav1.TypeMeta{ + Kind: "PodDisruptionBudget", + APIVersion: policyv1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-pdb", serviceValues.OverrideName), + }, + Spec: lValues.ResourceWorkloads[serviceValues.ResourceWorkload].PDB.Spec, + } + + // set the selector target to the service that requested it) + pdb.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "lagoon.sh/service": serviceValues.OverrideName, + }, + } + + pdb.ObjectMeta.Labels = labels + pdb.ObjectMeta.Annotations = annotations + + for key, value := range additionalLabels { + pdb.ObjectMeta.Labels[key] = value + } + // add any additional annotations + for key, value := range additionalAnnotations { + pdb.ObjectMeta.Annotations[key] = value + } + // validate any annotations + if err := apivalidation.ValidateAnnotations(pdb.ObjectMeta.Annotations, nil); err != nil { + if len(err) != 0 { + return nil, fmt.Errorf("the annotations for %s/%s are not valid: %v", "pdb", serviceValues.Name, err) + } + } + // validate any labels + if err := metavalidation.ValidateLabels(pdb.ObjectMeta.Labels, nil); err != nil { + if len(err) != 0 { + return nil, fmt.Errorf("the labels for %s/%s are not valid: %v", "pdb", serviceValues.Name, err) + } + } + + // check length of labels + err := helpers.CheckLabelLength(pdb.ObjectMeta.Labels) + if err != nil { + return nil, err + } + // @TODO: we should review this in the future when we stop doing `kubectl apply` in the builds :) + // marshal the resulting ingress + pdbBytes, err := yaml.Marshal(pdb) + if err != nil { + return nil, err + } + // add the seperator to the template so that it can be `kubectl apply` in bulk as part + // of the current build process + restoreResult := append(separator[:], pdbBytes[:]...) + result = append(result, restoreResult[:]...) + } + } + return result, nil +} diff --git a/internal/templating/resources/pdb/template_pdb_test.go b/internal/templating/resources/pdb/template_pdb_test.go new file mode 100644 index 00000000..10ed8ec5 --- /dev/null +++ b/internal/templating/resources/pdb/template_pdb_test.go @@ -0,0 +1,97 @@ +package pdb + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" + "github.com/uselagoon/build-deploy-tool/internal/generator" + policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestGeneratePDBTemplate(t *testing.T) { + type args struct { + lValues generator.BuildValues + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test1 - nginx pdb", + args: args{ + lValues: generator.BuildValues{ + Project: "example-project", + Environment: "brancha", + EnvironmentType: "production", + Namespace: "myexample-project-brancha", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "brancha", + Services: []generator.ServiceValues{ + { + Name: "nginx", + OverrideName: "nginx", + Type: "nginx-php-persistent", + ResourceWorkload: "nginx-php-performance", + }, + }, + ResourceWorkloads: map[string]generator.ResourceWorkloads{ + "nginx": { + ServiceType: "nginx", + PDB: &generator.PDBSpec{ + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + IntVal: 1, + Type: intstr.Int, + }, + }, + }, + }, + "nginx-php-performance": { + ServiceType: "nginx-php-persistent", + + PDB: &generator.PDBSpec{ + Spec: policyv1.PodDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + IntVal: 3, + Type: intstr.Int, + }, + }, + }, + }, + }, + }, + }, + want: "test-resources/result-nginx.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // add dbaasclient overrides for tests + tt.args.lValues.DBaaSClient = dbaasclient.NewClient(dbaasclient.Client{ + RetryMax: 5, + RetryWaitMin: time.Duration(10) * time.Millisecond, + RetryWaitMax: time.Duration(50) * time.Millisecond, + }) + got, err := GeneratePDBTemplate(tt.args.lValues) + if (err != nil) != tt.wantErr { + t.Errorf("GeneratePDBTemplate() error = %v, wantErr %v", err, tt.wantErr) + return + } + r1, err := os.ReadFile(tt.want) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + if !reflect.DeepEqual(string(got), string(r1)) { + t.Errorf("GeneratePDBTemplate() = %v, want %v", string(got), string(r1)) + } + }) + } +} diff --git a/internal/templating/resources/pdb/test-resources/result-nginx.yaml b/internal/templating/resources/pdb/test-resources/result-nginx.yaml new file mode 100644 index 00000000..05197a88 --- /dev/null +++ b/internal/templating/resources/pdb/test-resources/result-nginx.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + annotations: + lagoon.sh/branch: brancha + lagoon.sh/version: v2.x.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: nginx + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: nginx-php-persistent + lagoon.sh/buildType: branch + lagoon.sh/environment: brancha + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: nginx + lagoon.sh/service-type: nginx-php-persistent + name: nginx-pdb +spec: + minAvailable: 3 + selector: + matchLabels: + lagoon.sh/service: nginx +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 diff --git a/internal/testdata/complex/docker-compose.resource1.yml b/internal/testdata/complex/docker-compose.resource1.yml new file mode 100644 index 00000000..2a57d33f --- /dev/null +++ b/internal/testdata/complex/docker-compose.resource1.yml @@ -0,0 +1,142 @@ +version: '2.3' + +x-example-image-version: &example-image-version ${EXAMPLE_IMAGE_VERSION:-4.x} + +x-project: &project ${PROJECT_NAME:-mysite} + +x-volumes: &default-volumes + volumes: + - .:/app:${VOLUME_FLAGS:-delegated} ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + - ./docroot/sites/default/files:/app/docroot/sites/default/files:${VOLUME_FLAGS:-delegated} ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + +x-environment: &default-environment + LAGOON_PROJECT: *project + DRUPAL_HASH_SALT: fakehashsaltfakehashsaltfakehashsalt + LAGOON_LOCALDEV_URL: ${LOCALDEV_URL:-http://mysite.docker.amazee.io} + LAGOON_ROUTE: ${LOCALDEV_URL:-http://mysite.docker.amazee.io} + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + EXAMPLE_KEY: ${EXAMPLE_KEY:-} + EXAMPLE_IMAGE_VERSION: ${EXAMPLE_IMAGE_VERSION:-latest} + LAGOON_ENVIRONMENT_TYPE: ${LAGOON_ENVIRONMENT_TYPE:-local} + DRUPAL_REFRESH_SEARCHAPI: ${DRUPAL_REFRESH_SEARCHAPI:-} + EXAMPLE_INGRESS_PSK: ${EXAMPLE_INGRESS_PSK:-} + EXAMPLE_INGRESS_HEADER: ${EXAMPLE_INGRESS_HEADER:-} + EXAMPLE_INGRESS_ENABLED: ${EXAMPLE_INGRESS_ENABLED:-} + REDIS_CACHE_PREFIX: "tide_" + DB_ALIAS: ${DB_ALIAS:-bay.production} + +services: + + cli: + build: + context: internal/testdata/complex/docker + dockerfile: .docker/Dockerfile.cli + args: + COMPOSER: ${COMPOSER:-composer.json} + EXAMPLE_IMAGE_VERSION: *example-image-version + image: *project + environment: + <<: *default-environment + <<: *default-volumes + volumes_from: + ### Local overrides to mount host SSH keys. Automatically removed in CI. + - container:amazeeio-ssh-agent ### Local overrides to mount host SSH keys. Automatically removed in CI. + labels: + lagoon.type: cli-persistent + lagoon.persistent: /app/docroot/sites/default/files/ + lagoon.persistent.name: nginx-php + lagoon.persistent.size: 5Gi + + nginx: + build: + context: internal/testdata/complex/docker + dockerfile: .docker/Dockerfile.nginx-drupal + args: + CLI_IMAGE: *project + EXAMPLE_IMAGE_VERSION: *example-image-version + <<: *default-volumes + environment: + <<: *default-environment + depends_on: + - cli + networks: + - amazeeio-network + - default + labels: + lagoon.type: nginx-php-persistent + lagoon.persistent: /app/docroot/sites/default/files/ + lagoon.persistent.size: 5Gi + lagoon.name: nginx-php + lagoon.workloadresource: nginx-php-performance + expose: + - "8080" + php: + build: + context: internal/testdata/complex/docker + dockerfile: .docker/Dockerfile.php + args: + CLI_IMAGE: *project + EXAMPLE_IMAGE_VERSION: *example-image-version + environment: + <<: *default-environment + <<: *default-volumes + depends_on: + - cli + labels: + lagoon.type: nginx-php-persistent + lagoon.persistent: /app/docroot/sites/default/files/ + lagoon.persistent.size: 5Gi + lagoon.name: nginx-php + + mariadb: + image: amazeeio/mariadb-drupal + environment: + <<: *default-environment + ports: + - "3306" # Find port on host with `ahoy info` or `docker-compose port mariadb 3306` + labels: + lagoon.type: mariadb-shared + + redis: + image: amazeeio/redis + labels: + lagoon.type: redis + + elasticsearch: + build: + context: internal/testdata/complex/docker + dockerfile: .docker/Dockerfile.elasticsearch + args: + - ES_TPL=${ES_TPL:-elasticsearch.yml} + environment: + - discovery.type=single-node + labels: + lagoon.type: none + + chrome: + image: selenium/standalone-chrome:3.141.59-oxygen + shm_size: '1gb' + environment: + <<: *default-environment + <<: *default-volumes + depends_on: + - cli + labels: + lagoon.type: none + + clamav: + image: clamav/clamav:${EXAMPLE_IMAGE_VERSION:-4.x} + environment: + <<: *default-environment + ports: + - "3310" + labels: + lagoon.type: none + +networks: + amazeeio-network: + external: true + +volumes: + app: {} + files: {} diff --git a/internal/testdata/complex/lagoon.resource1.yml b/internal/testdata/complex/lagoon.resource1.yml new file mode 100644 index 00000000..58133ca2 --- /dev/null +++ b/internal/testdata/complex/lagoon.resource1.yml @@ -0,0 +1,43 @@ +--- +docker-compose-yaml: internal/testdata/complex/docker-compose.resource1.yml + +project: content-example-com + +environments: + production: + cronjobs: + - name: drush cron + schedule: "*/15 * * * *" + command: 'drush cron' + service: cli + + routes: + - nginx-php: + - "content.example.com": + monitoring-path: "/api/v1" + tls-acme: 'false' + insecure: Allow + master: + cronjobs: + - name: drush cron + schedule: "0 1,4 * * *" + command: 'drush cron' + service: cli + + routes: + - nginx-php: + - "master.content.example.com": + tls-acme: 'false' + insecure: Allow + develop: + cronjobs: + - name: drush cron + schedule: "0 1,4 * * *" + command: 'drush cron' + service: cli + + routes: + - nginx-php: + - "develop.content.example.com": + tls-acme: 'false' + insecure: Allow diff --git a/internal/testdata/complex/resource-templates/resource1/hpas.yaml b/internal/testdata/complex/resource-templates/resource1/hpas.yaml new file mode 100644 index 00000000..0a4e426a --- /dev/null +++ b/internal/testdata/complex/resource-templates/resource1/hpas.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: nginx-php + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: nginx-php-persistent + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: nginx-php + lagoon.sh/service-type: nginx-php-persistent + name: nginx-php-hpa +spec: + maxReplicas: 16 + metrics: + - resource: + name: cpu + target: + averageUtilization: 3000 + type: Utilization + type: Resource + minReplicas: 8 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-php +status: + currentMetrics: null + desiredReplicas: 0 diff --git a/internal/testdata/complex/resource-templates/resource1/pdbs.yaml b/internal/testdata/complex/resource-templates/resource1/pdbs.yaml new file mode 100644 index 00000000..058edd44 --- /dev/null +++ b/internal/testdata/complex/resource-templates/resource1/pdbs.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: nginx-php + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: nginx-php-persistent + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: nginx-php + lagoon.sh/service-type: nginx-php-persistent + name: nginx-php-pdb +spec: + minAvailable: 2 + selector: + matchLabels: + lagoon.sh/service: nginx-php +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 diff --git a/internal/testdata/complex/resource-templates/resource2/hpas.yaml b/internal/testdata/complex/resource-templates/resource2/hpas.yaml new file mode 100644 index 00000000..0a4e426a --- /dev/null +++ b/internal/testdata/complex/resource-templates/resource2/hpas.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: nginx-php + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: nginx-php-persistent + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: nginx-php + lagoon.sh/service-type: nginx-php-persistent + name: nginx-php-hpa +spec: + maxReplicas: 16 + metrics: + - resource: + name: cpu + target: + averageUtilization: 3000 + type: Utilization + type: Resource + minReplicas: 8 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx-php +status: + currentMetrics: null + desiredReplicas: 0 diff --git a/internal/testdata/complex/resource-templates/resource2/pdbs.yaml b/internal/testdata/complex/resource-templates/resource2/pdbs.yaml new file mode 100644 index 00000000..058edd44 --- /dev/null +++ b/internal/testdata/complex/resource-templates/resource2/pdbs.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: nginx-php + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: nginx-php-persistent + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: nginx-php + lagoon.sh/service-type: nginx-php-persistent + name: nginx-php-pdb +spec: + minAvailable: 2 + selector: + matchLabels: + lagoon.sh/service: nginx-php +status: + currentHealthy: 0 + desiredHealthy: 0 + disruptionsAllowed: 0 + expectedPods: 0 diff --git a/internal/testdata/complex/workload.resources1.json b/internal/testdata/complex/workload.resources1.json new file mode 100644 index 00000000..705a99a2 --- /dev/null +++ b/internal/testdata/complex/workload.resources1.json @@ -0,0 +1,28 @@ +{ + "nginx-php-performance": { + "serviceType": "nginx-php-persistent", + "hpa": { + "spec": { + "minReplicas": 8, + "maxReplicas": 16, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 3000 + } + } + } + ] + } + }, + "pdb": { + "spec": { + "minAvailable": 2 + } + } + } +} \ No newline at end of file diff --git a/internal/testdata/complex/workload.resources2.json b/internal/testdata/complex/workload.resources2.json new file mode 100644 index 00000000..705a99a2 --- /dev/null +++ b/internal/testdata/complex/workload.resources2.json @@ -0,0 +1,28 @@ +{ + "nginx-php-performance": { + "serviceType": "nginx-php-persistent", + "hpa": { + "spec": { + "minReplicas": 8, + "maxReplicas": 16, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 3000 + } + } + } + ] + } + }, + "pdb": { + "spec": { + "minAvailable": 2 + } + } + } +} \ No newline at end of file diff --git a/internal/testdata/node/docker-compose.resources1.yml b/internal/testdata/node/docker-compose.resources1.yml new file mode 100644 index 00000000..7d2ddf74 --- /dev/null +++ b/internal/testdata/node/docker-compose.resources1.yml @@ -0,0 +1,21 @@ +version: '2' +services: + node: + networks: + - amazeeio-network + - default + build: + context: internal/testdata/node/docker + dockerfile: node.dockerfile + labels: + lagoon.type: node + lagoon.workloadresource: node + volumes: + - .:/app:delegated + environment: + - LAGOON_LOCALDEV_HTTP_PORT=3000 + - LAGOON_ROUTE=http://node.docker.amazee.io + +networks: + amazeeio-network: + external: true \ No newline at end of file diff --git a/internal/testdata/node/lagoon.resources1.yml b/internal/testdata/node/lagoon.resources1.yml new file mode 100644 index 00000000..193ec450 --- /dev/null +++ b/internal/testdata/node/lagoon.resources1.yml @@ -0,0 +1,10 @@ +docker-compose-yaml: internal/testdata/node/docker-compose.resources1.yml + +environment_variables: + git_sha: "true" + +environments: + main: + routes: + - node: + - example.com diff --git a/internal/testdata/node/resource-templates/resource1/hpas.yaml b/internal/testdata/node/resource-templates/resource1/hpas.yaml new file mode 100644 index 00000000..e69de29b diff --git a/internal/testdata/node/resource-templates/resource1/pdbs.yaml b/internal/testdata/node/resource-templates/resource1/pdbs.yaml new file mode 100644 index 00000000..e69de29b diff --git a/internal/testdata/node/resource-templates/resource2/hpas.yaml b/internal/testdata/node/resource-templates/resource2/hpas.yaml new file mode 100644 index 00000000..e07c216f --- /dev/null +++ b/internal/testdata/node/resource-templates/resource2/hpas.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: node + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: node + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: node + lagoon.sh/service-type: node + name: node-hpa +spec: + maxReplicas: 16 + metrics: + - resource: + name: cpu + target: + averageUtilization: 3000 + type: Utilization + type: Resource + minReplicas: 8 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: node +status: + currentMetrics: null + desiredReplicas: 0 diff --git a/internal/testdata/node/resource-templates/resource2/pdbs.yaml b/internal/testdata/node/resource-templates/resource2/pdbs.yaml new file mode 100644 index 00000000..e69de29b diff --git a/internal/testdata/node/workload.resources1.json b/internal/testdata/node/workload.resources1.json new file mode 100644 index 00000000..d82c8b5f --- /dev/null +++ b/internal/testdata/node/workload.resources1.json @@ -0,0 +1,23 @@ +{ + "node": { + "serviceType": "node", + "hpa": { + "spec": { + "minReplicas": 8, + "maxReplicas": 16, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": { + "type": "Utilization", + "averageUtilization": 3000 + } + } + } + ] + } + } + } +} \ No newline at end of file diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 9e11fe6a..2a8b9794 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -1530,6 +1530,8 @@ LAGOON_SERVICES_YAML_FOLDER="/kubectl-build-deploy/lagoon/service-deployments" mkdir -p $LAGOON_SERVICES_YAML_FOLDER build-deploy-tool template lagoon-services --saved-templates-path ${LAGOON_SERVICES_YAML_FOLDER} --images /kubectl-build-deploy/images.yaml +build-deploy-tool template resource-workloads --saved-templates-path ${LAGOON_SERVICES_YAML_FOLDER} + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "deploymentTemplatingComplete" "Deployment Templating" "false" previousStepEnd=${currentStepEnd}