From 81fb0d020f65bd1f24b0a93a7a9e2660217ce8ff Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Mon, 29 Jul 2024 17:25:39 +1000 Subject: [PATCH] feat: template lagoonenv configmap --- Dockerfile | 4 +- cmd/root.go | 2 + cmd/template_lagoonenv.go | 86 +++++++ cmd/template_lagoonenv_test.go | 240 ++++++++++++++++++ internal/generator/build_data.go | 24 +- internal/generator/buildvalues.go | 6 + internal/generator/generator.go | 20 +- internal/generator/services.go | 6 +- .../lagoonenv/template_configmap.go | 66 +++++ .../lagoonenv/template_configmap_test.go | 93 +++++++ .../test-resources/lagoon-env-1.yaml | 20 ++ .../lagoonenv1/lagoon-env-configmap.yaml | 36 +++ .../lagoonenv2/lagoon-env-configmap.yaml | 52 ++++ internal/testdata/basic/lagoonenv2-creds.json | 24 ++ legacy/build-deploy-docker-compose.sh | 240 ++++++++---------- legacy/scripts/exec-kubectl-dbaas-wait.sh | 24 ++ legacy/scripts/exec-kubectl-mariadb-dbaas.sh | 47 ---- legacy/scripts/exec-kubectl-mongodb-dbaas.sh | 39 --- legacy/scripts/exec-kubectl-postgres-dbaas.sh | 47 ---- 19 files changed, 800 insertions(+), 276 deletions(-) create mode 100644 cmd/template_lagoonenv.go create mode 100644 cmd/template_lagoonenv_test.go create mode 100644 internal/templating/lagoonenv/template_configmap.go create mode 100644 internal/templating/lagoonenv/template_configmap_test.go create mode 100644 internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml create mode 100644 internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml create mode 100644 internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml create mode 100644 internal/testdata/basic/lagoonenv2-creds.json create mode 100644 legacy/scripts/exec-kubectl-dbaas-wait.sh delete mode 100644 legacy/scripts/exec-kubectl-mariadb-dbaas.sh delete mode 100644 legacy/scripts/exec-kubectl-mongodb-dbaas.sh delete mode 100644 legacy/scripts/exec-kubectl-postgres-dbaas.sh diff --git a/Dockerfile b/Dockerfile index 09076ea2..615f34e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,13 +72,11 @@ RUN apk add -U --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing au && apk upgrade --no-cache openssh openssh-keygen openssh-client-common openssh-client-default \ && apk add --no-cache openssl curl jq parallel bash git py-pip skopeo \ && git config --global user.email "lagoon@lagoon.io" && git config --global user.name lagoon \ - && pip install --break-system-packages shyaml yq + && pip install --break-system-packages yq RUN architecture=$(case $(uname -m) in x86_64 | amd64) echo "amd64" ;; aarch64 | arm64 | armv8) echo "arm64" ;; *) echo "amd64" ;; esac) \ && curl -Lo /usr/bin/kubectl https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/${architecture}/kubectl \ && chmod +x /usr/bin/kubectl \ - && curl -Lo /usr/bin/yq3 https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${architecture} \ - && chmod +x /usr/bin/yq3 \ && curl -Lo /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.35.2/yq_linux_${architecture} \ && chmod +x /usr/bin/yq \ && curl -Lo /tmp/helm.tar.gz https://get.helm.sh/helm-${HELM_VERSION}-linux-${architecture}.tar.gz \ diff --git a/cmd/root.go b/cmd/root.go index e88ae3cc..72b7e839 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -153,6 +153,8 @@ func init() { "Ignore missing env_file files (true by default, subject to change).") rootCmd.PersistentFlags().StringP("images", "", "", "JSON representation of service:image reference") + rootCmd.PersistentFlags().StringP("dbaas-creds", "", "", + "JSON representation of dbaas credential references") } // initConfig reads in config file and ENV variables if set. diff --git a/cmd/template_lagoonenv.go b/cmd/template_lagoonenv.go new file mode 100644 index 00000000..b1f29b5c --- /dev/null +++ b/cmd/template_lagoonenv.go @@ -0,0 +1,86 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + generator "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + "github.com/uselagoon/build-deploy-tool/internal/templating/lagoonenv" + "sigs.k8s.io/yaml" +) + +type DBaaSCredRefs []map[string]string + +var lagoonEnvGeneration = &cobra.Command{ + Use: "lagoon-env", + Aliases: []string{"le"}, + Short: "Generate the lagoon-env configmap template for a Lagoon build", + RunE: func(cmd *cobra.Command, args []string) error { + generator, err := generator.GenerateInput(*rootCmd, true) + if err != nil { + return err + } + dbaasCreds, err := rootCmd.PersistentFlags().GetString("dbaas-creds") + if err != nil { + return fmt.Errorf("error reading images flag: %v", err) + } + dbaasCredRefs, err := loadCredsFromFile(dbaasCreds) + if err != nil { + return err + } + dbCreds := map[string]string{} + for _, v := range *dbaasCredRefs { + for k, v1 := range v { + dbCreds[k] = v1 + } + } + generator.DBaaSVariables = dbCreds + return LagoonEnvTemplateGeneration(generator) + }, +} + +func loadCredsFromFile(file string) (*DBaaSCredRefs, error) { + dbaasCredRefs := &DBaaSCredRefs{} + dbaasCredJSON, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("couldn't read file %v: %v", file, err) + } + if err := json.Unmarshal(dbaasCredJSON, dbaasCredRefs); err != nil { + return nil, fmt.Errorf("error unmarshalling images payload: %v", err) + } + return dbaasCredRefs, nil +} + +// LagoonEnvTemplateGeneration . +func LagoonEnvTemplateGeneration(g generator.GeneratorInput, +) error { + lagoonBuild, err := generator.NewGenerator( + g, + ) + if err != nil { + return err + } + savedTemplates := g.SavedTemplatesPath + cm, err := lagoonenv.GenerateLagoonEnvConfigMap(*lagoonBuild.BuildValues) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + cmBytes, err := yaml.Marshal(cm) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + if len(cmBytes) > 0 { + if g.Debug { + fmt.Printf("Templating lagoon-env configmap %s\n", fmt.Sprintf("%s/%s.yaml", savedTemplates, "lagoon-env-configmap")) + } + helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "lagoon-env-configmap"), cmBytes) + } + return nil +} + +func init() { + templateCmd.AddCommand(lagoonEnvGeneration) +} diff --git a/cmd/template_lagoonenv_test.go b/cmd/template_lagoonenv_test.go new file mode 100644 index 00000000..587e1f81 --- /dev/null +++ b/cmd/template_lagoonenv_test.go @@ -0,0 +1,240 @@ +package cmd + +import ( + "fmt" + "os" + "reflect" + "testing" + + "github.com/andreyvit/diff" + "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + "github.com/uselagoon/build-deploy-tool/internal/lagoon" + "github.com/uselagoon/build-deploy-tool/internal/testdata" +) + +func TestLagoonEnvTemplateGeneration(t *testing.T) { + tests := []struct { + name string + description string + args testdata.TestData + templatePath string + want string + dbaasCreds string + vars []helpers.EnvironmentVariable + }{ + { + name: "test1 basic deployment", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/basic/lagoon.yml", + ProjectVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE1", + Value: "myspecialvariable1", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2", + Scope: "runtime", + }, + { + Name: "MY_SPECIAL_VARIABLE3", + Value: "myspecialvariable3", + Scope: "build", + }, + { + Name: "MY_SPECIAL_VARIABLE", + Value: "myspecialvariable", + Scope: "global", + }, + { + Name: "LAGOON_SYSTEM_CORE_VERSION", + Value: "v2.19.0", + Scope: "internal_system", + }, + { + Name: "REGISTRY_PASSWORD", + Value: "myenvvarregistrypassword", + Scope: "container_registry", + }, + }, + EnvVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2-env-override", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE4", + Value: "myspecialvariable4", + Scope: "runtime", + }, + }, + }, true), + templatePath: "testoutput", + want: "internal/testdata/basic/configmap-templates/lagoonenv1", + }, + { + name: "test1 basic deployment with mariadb creds", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/basic/lagoon.yml", + ProjectVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE1", + Value: "myspecialvariable1", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2", + Scope: "runtime", + }, + { + Name: "MY_SPECIAL_VARIABLE3", + Value: "myspecialvariable3", + Scope: "build", + }, + { + Name: "MY_SPECIAL_VARIABLE", + Value: "myspecialvariable", + Scope: "global", + }, + { + Name: "LAGOON_SYSTEM_CORE_VERSION", + Value: "v2.19.0", + Scope: "internal_system", + }, + { + Name: "REGISTRY_PASSWORD", + Value: "myenvvarregistrypassword", + Scope: "container_registry", + }, + }, + EnvVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2-env-override", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE4", + Value: "myspecialvariable4", + Scope: "runtime", + }, + }, + }, true), + dbaasCreds: "internal/testdata/basic/lagoonenv2-creds.json", + templatePath: "testoutput", + want: "internal/testdata/basic/configmap-templates/lagoonenv2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + helpers.UnsetEnvVars(tt.vars) //unset variables before running tests + for _, envVar := range tt.vars { + err := os.Setenv(envVar.Name, envVar.Value) + if err != nil { + t.Errorf("%v", err) + } + } + // set the environment variables from args + savedTemplates := tt.templatePath + generator, err := testdata.SetupEnvironment(*rootCmd, savedTemplates, tt.args) + if err != nil { + t.Errorf("%v", err) + } + + err = os.MkdirAll(savedTemplates, 0755) + if err != nil { + t.Errorf("couldn't create directory %v: %v", savedTemplates, err) + } + + defer os.RemoveAll(savedTemplates) + + ts := dbaasclient.TestDBaaSHTTPServer() + defer ts.Close() + err = os.Setenv("DBAAS_OPERATOR_HTTP", ts.URL) + if err != nil { + t.Errorf("%v", err) + } + dbaasCreds := &DBaaSCredRefs{} + if tt.dbaasCreds != "" { + dbaasCreds, err = loadCredsFromFile(tt.dbaasCreds) + if err != nil { + t.Errorf("%v", err) + } + dbCreds := map[string]string{} + for _, v := range *dbaasCreds { + for k, v1 := range v { + dbCreds[k] = v1 + } + } + generator.DBaaSVariables = dbCreds + } + err = LagoonEnvTemplateGeneration(generator) + if err != nil { + t.Errorf("%v", err) + } + + files, err := os.ReadDir(savedTemplates) + if err != nil { + t.Errorf("couldn't read directory %v: %v", savedTemplates, err) + } + results, err := os.ReadDir(tt.want) + if err != nil { + t.Errorf("couldn't read directory %v: %v", tt.want, err) + } + if len(files) != len(results) { + for _, f := range files { + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + fmt.Println(string(f1)) + } + t.Errorf("number of generated templates doesn't match results %v/%v: %v", len(files), len(results), err) + } + fCount := 0 + for _, f := range files { + for _, r := range results { + if f.Name() == r.Name() { + fCount++ + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + r1, err := os.ReadFile(fmt.Sprintf("%s/%s", tt.want, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + if !reflect.DeepEqual(f1, r1) { + t.Errorf("LagoonEnvTemplateGeneration() = \n%v", diff.LineDiff(string(r1), string(f1))) + } + } + } + } + if fCount != len(files) { + for _, f := range files { + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + fmt.Println(string(f1)) + } + t.Errorf("resulting templates do not match") + } + t.Cleanup(func() { + helpers.UnsetEnvVars(tt.vars) + }) + }) + } +} diff --git a/internal/generator/build_data.go b/internal/generator/build_data.go index efdbb67c..e34fc4cf 100644 --- a/internal/generator/build_data.go +++ b/internal/generator/build_data.go @@ -9,13 +9,15 @@ import ( ) // this creates a bunch of standard environment variables that are injected into the `lagoon-env` configmap normally -func collectBuildVariables(buildValues BuildValues) []lagoon.EnvironmentVariable { +func collectLagoonEnvConfigmapVariables(buildValues BuildValues) []lagoon.EnvironmentVariable { vars := []lagoon.EnvironmentVariable{} vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_PROJECT", Value: buildValues.Project, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ENVIRONMENT", Value: buildValues.Environment, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ENVIRONMENT_TYPE", Value: buildValues.EnvironmentType, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_GIT_SHA", Value: buildValues.GitSHA, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_KUBERNETES", Value: buildValues.Kubernetes, Scope: "runtime"}) + // LAGOON_GIT_SAFE_BRANCH is pointing to the enviornment name, therefore also is filled if this environment + // is created by a PR or Promote workflow. This technically wrong, therefore will be removed vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_GIT_SAFE_BRANCH", Value: buildValues.Environment, Scope: "runtime"}) //deprecated??? (https://github.com/uselagoon/lagoon/blob/1053965321495213591f4c9110f90a9d9dcfc946/images/kubectl-build-deploy-dind/build-deploy-docker-compose.sh#L748) if buildValues.BuildType == "branch" { vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_GIT_BRANCH", Value: buildValues.Branch, Scope: "runtime"}) @@ -26,15 +28,23 @@ func collectBuildVariables(buildValues BuildValues) []lagoon.EnvironmentVariable vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_PR_TITLE", Value: buildValues.PRTitle, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_PR_NUMBER", Value: buildValues.PRNumber, Scope: "runtime"}) } - if buildValues.ActiveEnvironment != "" { - vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ACTIVE_ENVIRONMENT", Value: buildValues.ActiveEnvironment, Scope: "runtime"}) - } - if buildValues.StandbyEnvironment != "" { - vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_STANDBY_ENVIRONMENT", Value: buildValues.StandbyEnvironment, Scope: "runtime"}) - } + // @TODO: check if these would actually be useful, they've never been used by anything before + // commenting out for now + // if buildValues.ActiveEnvironment != "" { + // vars = append( vars, lagoon.EnvironmentVariable{Name: "LAGOON_ACTIVE_ENVIRONMENT", Value: buildValues.ActiveEnvironment, Scope: "runtime"}) + // } + // if buildValues.StandbyEnvironment != "" { + // vars = append( vars, lagoon.EnvironmentVariable{Name: "LAGOON_STANDBY_ENVIRONMENT", Value: buildValues.StandbyEnvironment, Scope: "runtime"}) + // } vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ROUTE", Value: buildValues.Route, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ROUTES", Value: strings.Join(buildValues.Routes, ","), Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_AUTOGENERATED_ROUTES", Value: strings.Join(buildValues.AutogeneratedRoutes, ","), Scope: "runtime"}) + // add the api/token/ssh configuration variables to envvars + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_API_HOST", Value: buildValues.ConfigAPIHost, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_TOKEN_HOST", Value: buildValues.ConfigTokenHost, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_TOKEN_PORT", Value: buildValues.ConfigTokenPort, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_SSH_HOST", Value: buildValues.ConfigSSHHost, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_SSH_PORT", Value: buildValues.ConfigSSHPort, Scope: "runtime"}) return vars } diff --git a/internal/generator/buildvalues.go b/internal/generator/buildvalues.go index 9ba9c12f..faa3e84f 100644 --- a/internal/generator/buildvalues.go +++ b/internal/generator/buildvalues.go @@ -81,6 +81,12 @@ type BuildValues struct { IgnoreImageCache bool `json:"ignoreImageCache"` SSHPrivateKey string `json:"sshPrivateKey"` ForcePullImages []string `json:"forcePullImages"` + ConfigAPIHost string `json:"configAPIHost"` + ConfigTokenHost string `json:"configTokenHost"` + ConfigTokenPort string `json:"configTokenPort"` + ConfigSSHHost string `json:"configSSHHost"` + ConfigSSHPort string `json:"configSSHPort"` + DBaaSVariables map[string]string `json:"dbaasVariables" description:"map of variables provided by dbaas consumers"` } type Resources struct { diff --git a/internal/generator/generator.go b/internal/generator/generator.go index e84f5f44..a6894597 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -66,6 +66,12 @@ type GeneratorInput struct { DynamicDBaaSSecrets []string ImageCacheBuildArgsJSON string SSHPrivateKey string + ConfigAPIHost string + ConfigTokenHost string + ConfigTokenPort string + ConfigSSHHost string + ConfigSSHPort string + DBaaSVariables map[string]string } func NewGenerator( @@ -115,6 +121,16 @@ func NewGenerator( // this is used by CI systems to influence builds, it is rarely used and should probably be abandoned buildValues.IsCI = helpers.GetEnvBool("CI", generator.CI, generator.Debug) + // add dbaas credentials to build values for injection into configmap + buildValues.DBaaSVariables = generator.DBaaSVariables + + // set the lagoon config variables + buildValues.ConfigAPIHost = helpers.GetEnv("LAGOON_CONFIG_API_HOST", generator.ConfigAPIHost, generator.Debug) + buildValues.ConfigTokenHost = helpers.GetEnv("LAGOON_CONFIG_TOKEN_HOST", generator.ConfigTokenHost, generator.Debug) + buildValues.ConfigTokenPort = helpers.GetEnv("LAGOON_CONFIG_TOKEN_PORT", generator.ConfigTokenPort, generator.Debug) + buildValues.ConfigSSHHost = helpers.GetEnv("LAGOON_CONFIG_SSH_HOST", generator.ConfigSSHHost, generator.Debug) + buildValues.ConfigSSHPort = helpers.GetEnv("LAGOON_CONFIG_SSH_PORT", generator.ConfigSSHPort, generator.Debug) + buildValues.ConfigMapSha = configMapSha buildValues.BuildName = buildName buildValues.Kubernetes = kubernetes @@ -122,6 +138,7 @@ func NewGenerator( buildValues.ImageRegistry = imageRegistry buildValues.SourceRepository = sourceRepository buildValues.PromotionSourceEnvironment = promotionSourceEnvironment + // get the image references values from the build images output buildValues.ImageReferences = generator.ImageReferences defaultBackupSchedule := helpers.GetEnv("DEFAULT_BACKUP_SCHEDULE", generator.DefaultBackupSchedule, generator.Debug) @@ -254,7 +271,8 @@ func NewGenerator( json.Unmarshal([]byte(environmentVariables), &envVars) mergedVariables := lagoon.MergeVariables(projectVars, envVars) // collect a bunch of the default LAGOON_X based build variables that are injected into `lagoon-env` and make them available - configVars := collectBuildVariables(buildValues) + configVars := collectLagoonEnvConfigmapVariables(buildValues) + // add the calculated build runtime variables into the existing variable slice // this will later be used to add `runtime|global` scope into the `lagoon-env` configmap buildValues.EnvironmentVariables = lagoon.MergeVariables(mergedVariables, configVars) diff --git a/internal/generator/services.go b/internal/generator/services.go index 33d0ef0f..aeca541b 100644 --- a/internal/generator/services.go +++ b/internal/generator/services.go @@ -590,11 +590,11 @@ func composeToServiceValues( "myservice": "harbor.example.com/example-project/environment-name/myservice@sha256:abcdefg", }, - // in bash, the images are provided from yaml as base64 encoded data to retain formatting - // the command `lagoon-services` decodes and unmarshals it + // in bash, the images are provided from yaml file + // the command `lagoon-services` reads the file build-deploy-tool template lagoon-services \ --saved-templates-path ${LAGOON_SERVICES_YAML_FOLDER} \ - --images $(yq3 r -j /kubectl-build-deploy/images.yaml | jq -M -c | base64 -w0) + --images /kubectl-build-deploy/images.yaml */ } diff --git a/internal/templating/lagoonenv/template_configmap.go b/internal/templating/lagoonenv/template_configmap.go new file mode 100644 index 00000000..6bc887f7 --- /dev/null +++ b/internal/templating/lagoonenv/template_configmap.go @@ -0,0 +1,66 @@ +package lagoonenv + +import ( + "github.com/uselagoon/build-deploy-tool/internal/generator" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GenerateLagoonEnvConfigMap generates the lagoon template to apply. +func GenerateLagoonEnvConfigMap( + buildValues generator.BuildValues, +) (corev1.ConfigMap, error) { + + // add the default labels + labels := map[string]string{ + "app.kubernetes.io/managed-by": "build-deploy-tool", + "app.kubernetes.io/instance": "lagoon-env", + "app.kubernetes.io/name": "lagoon-env", + "lagoon.sh/template": "lagoon-env-0.1.0", + "lagoon.sh/project": buildValues.Project, + "lagoon.sh/environment": buildValues.Environment, + "lagoon.sh/environmentType": buildValues.EnvironmentType, + "lagoon.sh/buildType": buildValues.BuildType, + } + + // add the default annotations + annotations := map[string]string{} + + // add any additional labels + if buildValues.BuildType == "branch" { + annotations["lagoon.sh/branch"] = buildValues.Branch + } else if buildValues.BuildType == "pullrequest" { + annotations["lagoon.sh/prNumber"] = buildValues.PRNumber + annotations["lagoon.sh/prHeadBranch"] = buildValues.PRHeadBranch + annotations["lagoon.sh/prBaseBranch"] = buildValues.PRBaseBranch + } + + variables := map[string]string{} + + // add variables from the project/environment/build created variables + for _, v := range buildValues.EnvironmentVariables { + if v.Scope == "global" || v.Scope == "runtime" { + variables[v.Name] = v.Value + } + } + + // add dbaas variables to lagoon-env + for k, v := range buildValues.DBaaSVariables { + variables[k] = v + } + + lagoonEnv := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "lagoon-env", + Labels: labels, + Annotations: annotations, + }, + Data: variables, + } + + return lagoonEnv, nil +} diff --git a/internal/templating/lagoonenv/template_configmap_test.go b/internal/templating/lagoonenv/template_configmap_test.go new file mode 100644 index 00000000..cc238d5c --- /dev/null +++ b/internal/templating/lagoonenv/template_configmap_test.go @@ -0,0 +1,93 @@ +package lagoonenv + +import ( + "os" + "reflect" + "testing" + + "github.com/andreyvit/diff" + "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/lagoon" + "sigs.k8s.io/yaml" +) + +func TestGenerateLagoonEnvConfigMap(t *testing.T) { + type args struct { + buildValues generator.BuildValues + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test1", + args: args{ + buildValues: generator.BuildValues{ + Project: "example-project", + Environment: "environment-name", + EnvironmentType: "production", + Namespace: "myexample-project-environment-name", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "environment-name", + EnvironmentVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE1", + Value: "myspecialvariable1", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2", + Scope: "runtime", + }, + { + Name: "MY_SPECIAL_VARIABLE3", + Value: "myspecialvariable3", + Scope: "build", + }, + { + Name: "MY_SPECIAL_VARIABLE", + Value: "myspecialvariable", + Scope: "global", + }, + { + Name: "LAGOON_SYSTEM_CORE_VERSION", + Value: "v2.19.0", + Scope: "internal_system", + }, + { + Name: "REGISTRY_PASSWORD", + Value: "myenvvarregistrypassword", + Scope: "container_registry", + }, + }, + }, + }, + want: "test-resources/lagoon-env-1.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GenerateLagoonEnvConfigMap(tt.args.buildValues) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateLagoonEnvConfigMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + r1, err := os.ReadFile(tt.want) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + cm, err := yaml.Marshal(got) + if err != nil { + t.Errorf("couldn't generate template %v", err) + } + if !reflect.DeepEqual(string(cm), string(r1)) { + t.Errorf("GenerateLagoonEnvConfigMap() = \n%v", diff.LineDiff(string(r1), string(cm))) + } + }) + } +} diff --git a/internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml b/internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml new file mode 100644 index 00000000..3278abaa --- /dev/null +++ b/internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +data: + MY_SPECIAL_VARIABLE: myspecialvariable + MY_SPECIAL_VARIABLE1: myspecialvariable1 + MY_SPECIAL_VARIABLE2: myspecialvariable2 +kind: ConfigMap +metadata: + annotations: + lagoon.sh/branch: environment-name + creationTimestamp: null + labels: + app.kubernetes.io/instance: lagoon-env + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: lagoon-env + lagoon.sh/buildType: branch + lagoon.sh/environment: environment-name + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/template: lagoon-env-0.1.0 + name: lagoon-env diff --git a/internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml b/internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml new file mode 100644 index 00000000..906ad04c --- /dev/null +++ b/internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +data: + LAGOON_AUTOGENERATED_ROUTES: "" + LAGOON_CONFIG_API_HOST: "" + LAGOON_CONFIG_SSH_HOST: "" + LAGOON_CONFIG_SSH_PORT: "" + LAGOON_CONFIG_TOKEN_HOST: "" + LAGOON_CONFIG_TOKEN_PORT: "" + LAGOON_ENVIRONMENT: main + LAGOON_ENVIRONMENT_TYPE: production + LAGOON_GIT_BRANCH: main + LAGOON_GIT_SAFE_BRANCH: main + LAGOON_GIT_SHA: abcdefg123456 + LAGOON_KUBERNETES: remote-cluster1 + LAGOON_PROJECT: example-project + LAGOON_ROUTE: "" + LAGOON_ROUTES: "" + MY_SPECIAL_VARIABLE: myspecialvariable + MY_SPECIAL_VARIABLE1: myspecialvariable1 + MY_SPECIAL_VARIABLE2: myspecialvariable2-env-override + MY_SPECIAL_VARIABLE4: myspecialvariable4 +kind: ConfigMap +metadata: + annotations: + lagoon.sh/branch: main + creationTimestamp: null + labels: + app.kubernetes.io/instance: lagoon-env + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: lagoon-env + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/template: lagoon-env-0.1.0 + name: lagoon-env diff --git a/internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml b/internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml new file mode 100644 index 00000000..d35eeca1 --- /dev/null +++ b/internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +data: + LAGOON_AUTOGENERATED_ROUTES: "" + LAGOON_CONFIG_API_HOST: "" + LAGOON_CONFIG_SSH_HOST: "" + LAGOON_CONFIG_SSH_PORT: "" + LAGOON_CONFIG_TOKEN_HOST: "" + LAGOON_CONFIG_TOKEN_PORT: "" + LAGOON_ENVIRONMENT: main + LAGOON_ENVIRONMENT_TYPE: production + LAGOON_GIT_BRANCH: main + LAGOON_GIT_SAFE_BRANCH: main + LAGOON_GIT_SHA: abcdefg123456 + LAGOON_KUBERNETES: remote-cluster1 + LAGOON_PROJECT: example-project + LAGOON_ROUTE: "" + LAGOON_ROUTES: "" + MARIADB_DATABASE: example-project-main_LMq2Q + MARIADB_HOST: mariadb-abcdef + MARIADB_PASSWORD: juD9RzjCEKbOYucpI5jVqGmr + MARIADB_PORT: "3306" + MARIADB_USERNAME: example-project-main_fO2Fo + MARIADB2_DATABASE: example-project-main_df23s + MARIADB2_HOST: mariadb2-abcdef + MARIADB2_PASSWORD: juD9RzjCEKbOYucpI5jVqGmr + MARIADB2_PORT: "3306" + MARIADB2_USERNAME: example-project-main_f3d1o + MARIADB3_DATABASE: example-project-main_sa241 + MARIADB3_HOST: mariadb3-abcdef + MARIADB3_PASSWORD: juD9RzjCEKbOYucpI5jVqGmr + MARIADB3_PORT: "3306" + MARIADB3_READREPLICA_HOSTS: readreplica-mariadb3-efg-321abc,readreplica-mariadb3-abc123-efg + MARIADB3_USERNAME: example-project-main_as24 + MY_SPECIAL_VARIABLE: myspecialvariable + MY_SPECIAL_VARIABLE1: myspecialvariable1 + MY_SPECIAL_VARIABLE2: myspecialvariable2-env-override + MY_SPECIAL_VARIABLE4: myspecialvariable4 +kind: ConfigMap +metadata: + annotations: + lagoon.sh/branch: main + creationTimestamp: null + labels: + app.kubernetes.io/instance: lagoon-env + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: lagoon-env + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/template: lagoon-env-0.1.0 + name: lagoon-env diff --git a/internal/testdata/basic/lagoonenv2-creds.json b/internal/testdata/basic/lagoonenv2-creds.json new file mode 100644 index 00000000..a2793f6c --- /dev/null +++ b/internal/testdata/basic/lagoonenv2-creds.json @@ -0,0 +1,24 @@ +[ + { + "MARIADB_HOST": "mariadb-abcdef", + "MARIADB_USERNAME": "example-project-main_fO2Fo", + "MARIADB_PASSWORD": "juD9RzjCEKbOYucpI5jVqGmr", + "MARIADB_DATABASE": "example-project-main_LMq2Q", + "MARIADB_PORT": "3306" + }, + { + "MARIADB3_HOST": "mariadb3-abcdef", + "MARIADB3_USERNAME": "example-project-main_as24", + "MARIADB3_PASSWORD": "juD9RzjCEKbOYucpI5jVqGmr", + "MARIADB3_DATABASE": "example-project-main_sa241", + "MARIADB3_PORT": "3306", + "MARIADB3_READREPLICA_HOSTS": "readreplica-mariadb3-efg-321abc,readreplica-mariadb3-abc123-efg" + }, + { + "MARIADB2_HOST": "mariadb2-abcdef", + "MARIADB2_USERNAME": "example-project-main_f3d1o", + "MARIADB2_PASSWORD": "juD9RzjCEKbOYucpI5jVqGmr", + "MARIADB2_DATABASE": "example-project-main_df23s", + "MARIADB2_PORT": "3306" + } +] diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index bab3e08c..7b4f2f99 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -162,7 +162,7 @@ if [ ! -z "$(featureFlag IMAGECACHE_REGISTRY)" ]; then fi # Load path of docker-compose that should be used -DOCKER_COMPOSE_YAML=($(cat .lagoon.yml | shyaml get-value docker-compose-yaml)) +DOCKER_COMPOSE_YAML=($(cat .lagoon.yml | yq -o json | jq -r '.docker-compose-yaml')) echo "Updating lagoon-yaml configmap with a pre-deploy version of the .lagoon.yml file" if kubectl -n ${NAMESPACE} get configmap lagoon-yaml &> /dev/null; then @@ -328,7 +328,7 @@ fi # # export LAGOON_GIT_SHA=`git rev-parse HEAD` # -INJECT_GIT_SHA=$(cat .lagoon.yml | shyaml get-value environment_variables.git_sha false) +INJECT_GIT_SHA=$(cat .lagoon.yml | yq -o json | jq -r '.environment_variables.git_sha // false') if [ "$INJECT_GIT_SHA" == "true" ] then # export this so the build-deploy-tool can read it @@ -342,10 +342,10 @@ currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "lagoonYmlValidation" ".lagoon.yml Validation" "false" previousStepEnd=${currentStepEnd} beginBuildStep "Configure Variables" "configuringVariables" -DEPLOY_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.deploy-type default) +DEPLOY_TYPE=$(cat .lagoon.yml | yq -o json | jq -r '.environments.'\"${BRANCH//.\//.}\"'."deploy-type" // "default"') # Load all Services that are defined -COMPOSE_SERVICES=($(cat $DOCKER_COMPOSE_YAML | shyaml keys services)) +COMPOSE_SERVICES=($(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services | keys_unsorted | .[]')) ############################################## ### CACHE IMAGE LIST GENERATION @@ -376,6 +376,10 @@ declare -A IMAGES_PUSH declare -A IMAGES_PROMOTE # this array stores the hashes of the built images declare -A IMAGE_HASHES +# this array stores the dbaas consumer specs +declare -A MARIADB_DBAAS_CONSUMER_SPECS +declare -A POSTGRES_DBAAS_CONSUMER_SPECS +declare -A MONGODB_DBAAS_CONSUMER_SPECS HELM_ARGUMENTS=() . /kubectl-build-deploy/scripts/kubectl-get-cluster-capabilities.sh @@ -417,16 +421,16 @@ DBAAS=($(build-deploy-tool identify dbaas)) for COMPOSE_SERVICE in "${COMPOSE_SERVICES[@]}" do # The name of the service can be overridden, if not we use the actual servicename - SERVICE_NAME=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.name default) + SERVICE_NAME=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$COMPOSE_SERVICE\"'.labels."lagoon.name" // "default"') if [ "$SERVICE_NAME" == "default" ]; then SERVICE_NAME=$COMPOSE_SERVICE fi # Load the servicetype. If it's "none" we will not care about this service at all - SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.type custom) + SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$COMPOSE_SERVICE\"'.labels."lagoon.type" // "custom"') # Allow the servicetype to be overriden by environment in .lagoon.yml - ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.types.$SERVICE_NAME false) + ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | yq -o json | jq -r '.environments.'\"${BRANCH//./\\.}\"'.types.'\"$SERVICE_NAME\"' // false') if [ ! $ENVIRONMENT_SERVICE_TYPE_OVERRIDE == "false" ]; then SERVICE_TYPE=$ENVIRONMENT_SERVICE_TYPE_OVERRIDE fi @@ -461,7 +465,7 @@ do # For DeploymentConfigs with multiple Services inside (like nginx-php), we allow to define the service type of within the # deploymentconfig via lagoon.deployment.servicetype. If this is not set we use the Compose Service Name - DEPLOYMENT_SERVICETYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.deployment\\.servicetype default) + DEPLOYMENT_SERVICETYPE=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$COMPOSE_SERVICE\"'.labels."lagoon.deployment.servicetype" // "default"') if [ "$DEPLOYMENT_SERVICETYPE" == "default" ]; then DEPLOYMENT_SERVICETYPE=$COMPOSE_SERVICE fi @@ -711,23 +715,23 @@ YAML_FOLDER="/kubectl-build-deploy/lagoon/services-routes" mkdir -p $YAML_FOLDER # BC for routes.insecure, which is now called routes.autogenerate.insecure -BC_ROUTES_AUTOGENERATE_INSECURE=$(cat .lagoon.yml | shyaml get-value routes.insecure false) +BC_ROUTES_AUTOGENERATE_INSECURE=$(cat .lagoon.yml | yq -o json | jq -r '.routes.insecure \\ false') if [ ! $BC_ROUTES_AUTOGENERATE_INSECURE == "false" ]; then echo "=== routes.insecure is now defined in routes.autogenerate.insecure, pleae update your .lagoon.yml file" # update the .lagoon.yml with the new location for build-deploy-tool to read - yq3 write -i -- .lagoon.yml 'routes.autogenerate.insecure' $BC_ROUTES_AUTOGENERATE_INSECURE + yq -i '.routes.autogenerate.insecure = "'${BC_ROUTES_AUTOGENERATE_INSECURE}'"' .lagoon.yml fi touch /kubectl-build-deploy/values.yaml -yq3 write -i -- /kubectl-build-deploy/values.yaml 'project' $PROJECT -yq3 write -i -- /kubectl-build-deploy/values.yaml 'environment' $ENVIRONMENT -yq3 write -i -- /kubectl-build-deploy/values.yaml 'environmentType' $ENVIRONMENT_TYPE -yq3 write -i -- /kubectl-build-deploy/values.yaml 'namespace' $NAMESPACE -yq3 write -i -- /kubectl-build-deploy/values.yaml 'gitSha' $LAGOON_GIT_SHA -yq3 write -i -- /kubectl-build-deploy/values.yaml 'buildType' $BUILD_TYPE -yq3 write -i -- /kubectl-build-deploy/values.yaml 'kubernetes' $KUBERNETES -yq3 write -i -- /kubectl-build-deploy/values.yaml 'lagoonVersion' $LAGOON_VERSION +yq -i '.project = "'${PROJECT}'"' /kubectl-build-deploy/values.yaml +yq -i '.environmentType = "'${ENVIRONMENT}'"' /kubectl-build-deploy/values.yaml +yq -i '.environmentType = "'${ENVIRONMENT_TYPE}'"' /kubectl-build-deploy/values.yaml +yq -i '.namespace = "'${NAMESPACE}'"' /kubectl-build-deploy/values.yaml +yq -i '.gitSha = "'${LAGOON_GIT_SHA}'"' /kubectl-build-deploy/values.yaml +yq -i '.buildType = "'${BUILD_TYPE}'"' /kubectl-build-deploy/values.yaml +yq -i '.kubernetes = "'${KUBERNETES}'"' /kubectl-build-deploy/values.yaml +yq -i '.lagoonVersion = "'${LAGOON_VERSION}'"' /kubectl-build-deploy/values.yaml # check for ROOTLESS_WORKLOAD feature flag, disabled by default if [ "${SCC_CHECK}" != "false" ]; then @@ -735,44 +739,21 @@ if [ "${SCC_CHECK}" != "false" ]; then # this applies it to all deployments in this environment because we don't isolate by service type its applied to all OPENSHIFT_SUPPLEMENTAL_GROUP=$(kubectl get namespace ${NAMESPACE} -o json | jq -r '.metadata.annotations."openshift.io/sa.scc.supplemental-groups"' | cut -c -10) echo "Setting openshift fsGroup to ${OPENSHIFT_SUPPLEMENTAL_GROUP}" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'podSecurityContext.fsGroup' $OPENSHIFT_SUPPLEMENTAL_GROUP + yq -i '.podSecurityContext.fsGroup = "'${OPENSHIFT_SUPPLEMENTAL_GROUP}'"' /kubectl-build-deploy/values.yaml fi -echo -e "\ -LAGOON_PROJECT=${PROJECT}\n\ -LAGOON_ENVIRONMENT=${ENVIRONMENT}\n\ -LAGOON_ENVIRONMENT_TYPE=${ENVIRONMENT_TYPE}\n\ -LAGOON_GIT_SHA=${LAGOON_GIT_SHA}\n\ -LAGOON_KUBERNETES=${KUBERNETES}\n\ -" >> /kubectl-build-deploy/values.env - -# DEPRECATED: will be removed with Lagoon 3.0.0 -# LAGOON_GIT_SAFE_BRANCH is pointing to the enviornment name, therefore also is filled if this environment -# is created by a PR or Promote workflow. This technically wrong, therefore will be removed -echo -e "\ -LAGOON_GIT_SAFE_BRANCH=${ENVIRONMENT}\n\ -" >> /kubectl-build-deploy/values.env -if [ "$BUILD_TYPE" == "branch" ]; then - yq3 write -i -- /kubectl-build-deploy/values.yaml 'branch' $BRANCH - echo -e "\ -LAGOON_GIT_BRANCH=${BRANCH}\n\ -" >> /kubectl-build-deploy/values.env + +if [ "$BUILD_TYPE" == "branch" ]; then + yq -i '.branch = "'${BRANCH}'"' /kubectl-build-deploy/values.yaml fi if [ "$BUILD_TYPE" == "pullrequest" ]; then - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prHeadBranch' "$PR_HEAD_BRANCH" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prBaseBranch' "$PR_BASE_BRANCH" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prTitle' "$PR_TITLE" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prNumber' "$PR_NUMBER" - - echo -e "\ -LAGOON_PR_HEAD_BRANCH=${PR_HEAD_BRANCH}\n\ -LAGOON_PR_BASE_BRANCH=${PR_BASE_BRANCH}\n\ -LAGOON_PR_TITLE=${PR_TITLE}\n\ -LAGOON_PR_NUMBER=${PR_NUMBER}\n\ -" >> /kubectl-build-deploy/values.env + yq -i '.prHeadBranch = "'${PR_HEAD_BRANCH}'"' /kubectl-build-deploy/values.yaml + yq -i '.prBaseBranch = "'${PR_BASE_BRANCH}'"' /kubectl-build-deploy/values.yaml + yq -i '.prTitle = "'${PR_TITLE}'"' /kubectl-build-deploy/values.yaml + yq -i '.prNumber = "'${PR_NUMBER}'"' /kubectl-build-deploy/values.yaml fi currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" @@ -805,14 +786,14 @@ FASTLY_API_SECRET_PREFIX="fastly-api-" FASTLY_API_SECRETS_COUNTER=0 FASTLY_API_SECRETS=() -if [ -n "$(cat .lagoon.yml | shyaml keys fastly.api-secrets.$FASTLY_API_SECRETS_COUNTER 2> /dev/null)" ]; then - while [ -n "$(cat .lagoon.yml | shyaml get-value fastly.api-secrets.$FASTLY_API_SECRETS_COUNTER 2> /dev/null)" ]; do - FASTLY_API_SECRET_NAME=$FASTLY_API_SECRET_PREFIX$(cat .lagoon.yml | shyaml get-value fastly.api-secrets.$FASTLY_API_SECRETS_COUNTER.name 2> /dev/null) +if [ "$(cat .lagoon.yml | yq -o json | jq -r '.fastly."api-secrets"['$FASTLY_API_SECRETS_COUNTER']')" != "null" ]; then + while [ "$(cat .lagoon.yml | yq -o json | jq -r '.fastly."api-secrets"['$FASTLY_API_SECRETS_COUNTER']')" != "null" ]; do + FASTLY_API_SECRET_NAME=$FASTLY_API_SECRET_PREFIX$(cat .lagoon.yml | yq -o json | jq -r '.fastly."api-secrets"['$FASTLY_API_SECRETS_COUNTER'].name' 2> /dev/null) if [ -z "$FASTLY_API_SECRET_NAME" ]; then echo -e "A fastly api secret was defined in the .lagoon.yml file, but no name could be found the .lagoon.yml\n\nPlease check if the name has been set correctly." exit 1 fi - FASTLY_API_TOKEN_VALUE=$(cat .lagoon.yml | shyaml get-value fastly.api-secrets.$FASTLY_API_SECRETS_COUNTER.apiTokenVariableName false) + FASTLY_API_TOKEN_VALUE=$(cat .lagoon.yml | yq -o json | jq -r '.fastly."api-secrets"['$FASTLY_API_SECRETS_COUNTER'].apiTokenVariableName // false') if [[ $FASTLY_API_TOKEN_VALUE == "false" ]]; then echo "No 'apiTokenVariableName' defined for fastly secret $FASTLY_API_SECRET_NAME"; exit 1; fi @@ -834,7 +815,7 @@ if [ -n "$(cat .lagoon.yml | shyaml keys fastly.api-secrets.$FASTLY_API_SECRETS_ exit 1 fi fi - FASTLY_API_PLATFORMTLS_CONFIGURATION=$(cat .lagoon.yml | shyaml get-value fastly.api-secrets.$FASTLY_API_SECRETS_COUNTER.platformTLSConfiguration "") + FASTLY_API_PLATFORMTLS_CONFIGURATION=$(cat .lagoon.yml | yq -o json | jq -r '.fastly."api-secrets"['$FASTLY_API_SECRETS_COUNTER'].platformTLSConfiguration // ""') if [ -z "$FASTLY_API_PLATFORMTLS_CONFIGURATION" ]; then echo -e "A fastly api secret was defined in the .lagoon.yml file, but no platform tls configuration id could be found in the .lagoon.yml\n\nPlease check if the platform tls configuration id has been set correctly." exit 1 @@ -1185,74 +1166,9 @@ fi # Get list of autogenerated routes AUTOGENERATED_ROUTES=$(kubectl -n ${NAMESPACE} get ingress --sort-by='{.metadata.name}' -l "lagoon.sh/autogenerated=true" -o=go-template --template='{{range $indexItems, $ingress := .items}}{{if $indexItems}},{{end}}{{$tls := .spec.tls}}{{range $indexRule, $rule := .spec.rules}}{{if $indexRule}},{{end}}{{if $tls}}https://{{else}}http://{{end}}{{.host}}{{end}}{{end}}') -yq3 write -i -- /kubectl-build-deploy/values.yaml 'route' "$ROUTE" -yq3 write -i -- /kubectl-build-deploy/values.yaml 'routes' "$ROUTES" -yq3 write -i -- /kubectl-build-deploy/values.yaml 'autogeneratedRoutes' "$AUTOGENERATED_ROUTES" - - -# Add in Lagoon core api and ssh-portal details, if available -if [ ! -z "$LAGOON_CONFIG_API_HOST" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_API_HOST="${LAGOON_CONFIG_API_HOST}") - echo -e "LAGOON_CONFIG_API_HOST=${LAGOON_CONFIG_API_HOST}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_TOKEN_HOST" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_TOKEN_HOST="${LAGOON_CONFIG_TOKEN_HOST}") - echo -e "LAGOON_CONFIG_TOKEN_HOST=${LAGOON_CONFIG_TOKEN_HOST}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_TOKEN_PORT" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_TOKEN_PORT="${LAGOON_CONFIG_TOKEN_PORT}") - echo -e "LAGOON_CONFIG_TOKEN_PORT=${LAGOON_CONFIG_TOKEN_PORT}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_SSH_HOST" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_SSH_HOST="${LAGOON_CONFIG_SSH_HOST}") - echo -e "LAGOON_CONFIG_SSH_HOST=${LAGOON_CONFIG_SSH_HOST}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_SSH_PORT" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_SSH_PORT="${LAGOON_CONFIG_SSH_PORT}") - echo -e "LAGOON_CONFIG_SSH_PORT=${LAGOON_CONFIG_SSH_PORT}\n" >> /kubectl-build-deploy/values.env -fi - -echo -e "\ -LAGOON_ROUTE=${ROUTE}\n\ -LAGOON_ROUTES=${ROUTES}\n\ -LAGOON_AUTOGENERATED_ROUTES=${AUTOGENERATED_ROUTES}\n\ -" >> /kubectl-build-deploy/values.env - -# Generate a Config Map with project wide env variables -kubectl -n ${NAMESPACE} create configmap lagoon-env -o yaml --dry-run=client --from-env-file=/kubectl-build-deploy/values.env | kubectl apply -n ${NAMESPACE} -f - - -# Add environment variables from lagoon API -if [ ! -z "$LAGOON_PROJECT_VARIABLES" ]; then - HAS_PROJECT_RUNTIME_VARS=$(echo $LAGOON_PROJECT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") )') - - if [ ! "$HAS_PROJECT_RUNTIME_VARS" = "[]" ]; then - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":$(echo $LAGOON_PROJECT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") ) | map( { (.name) : .value } ) | add | tostring')}" - fi -fi -if [ ! -z "$LAGOON_ENVIRONMENT_VARIABLES" ]; then - HAS_ENVIRONMENT_RUNTIME_VARS=$(echo $LAGOON_ENVIRONMENT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") )') - - if [ ! "$HAS_ENVIRONMENT_RUNTIME_VARS" = "[]" ]; then - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":$(echo $LAGOON_ENVIRONMENT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") ) | map( { (.name) : .value } ) | add | tostring')}" - fi -fi - -if [ "$BUILD_TYPE" == "pullrequest" ]; then - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"LAGOON_PR_HEAD_BRANCH\":\"${PR_HEAD_BRANCH}\", \"LAGOON_PR_BASE_BRANCH\":\"${PR_BASE_BRANCH}\", \"LAGOON_PR_TITLE\":$(echo $PR_TITLE | jq -R)}}" -fi +yq -i '.route = "'${ROUTE}'"' /kubectl-build-deploy/values.yaml +yq -i '.routes = "'${ROUTES}'"' /kubectl-build-deploy/values.yaml +yq -i '.autogeneratedRoutes = "'${AUTOGENERATED_ROUTES}'"' /kubectl-build-deploy/values.yaml # loop through created DBAAS templates DBAAS=($(build-deploy-tool identify dbaas)) @@ -1272,19 +1188,25 @@ do mariadb-dbaas) # remove the image from images to pull unset IMAGES_PULL[$SERVICE_NAME] - . /kubectl-build-deploy/scripts/exec-kubectl-mariadb-dbaas.sh + CONSUMER_TYPE="mariadbconsumer" + . /kubectl-build-deploy/scripts/exec-kubectl-dbaas-wait.sh + MARIADB_DBAAS_CONSUMER_SPECS["${SERVICE_NAME}"]=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o json | jq -r '.spec | @base64') ;; postgres-dbaas) # remove the image from images to pull unset IMAGES_PULL[$SERVICE_NAME] - . /kubectl-build-deploy/scripts/exec-kubectl-postgres-dbaas.sh + CONSUMER_TYPE="postgresqlconsumer" + . /kubectl-build-deploy/scripts/exec-kubectl-dbaas-wait.sh + POSTGRES_DBAAS_CONSUMER_SPECS["${SERVICE_NAME}"]=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o json | jq -r '.spec | @base64') ;; mongodb-dbaas) # remove the image from images to pull unset IMAGES_PULL[$SERVICE_NAME] - . /kubectl-build-deploy/scripts/exec-kubectl-mongodb-dbaas.sh + CONSUMER_TYPE="mongodbconsumer" + . /kubectl-build-deploy/scripts/exec-kubectl-dbaas-wait.sh + MONGODB_DBAAS_CONSUMER_SPECS["${SERVICE_NAME}"]=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o json | jq -r '.spec | @base64') ;; *) @@ -1293,6 +1215,66 @@ do esac done +# convert specs into credential dump for ingestion by build-deploy-tool +DBAAS_VARIABLES="[]" +for SERVICE_NAME in "${!MARIADB_DBAAS_CONSUMER_SPECS[@]}" +do + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + DB_HOST=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.primary') + DB_USER=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.username') + DB_PASSWORD=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.password') + DB_NAME=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.database') + DB_PORT=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.port') + DB_CONSUMER='{"'${SERVICE_NAME_UPPERCASE}'_HOST":"'${DB_HOST}'", "'${SERVICE_NAME_UPPERCASE}'_USERNAME":"'${DB_USER}'","'${SERVICE_NAME_UPPERCASE}'_PASSWORD":"'${DB_PASSWORD}'","'${SERVICE_NAME_UPPERCASE}'_DATABASE":"'${DB_NAME}'","'${SERVICE_NAME_UPPERCASE}'_PORT":"'${DB_PORT}'"}' + if DB_READREPLICA_HOSTS=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.replicas | .[]' 2>/dev/null); then + if [ "$DB_READREPLICA_HOSTS" != "null" ]; then + DB_READREPLICA_HOSTS=$(echo "$DB_READREPLICA_HOSTS" | sed 's/^\|$//g' | paste -sd, -) + DB_CONSUMER=$(echo "${DB_CONSUMER}" | jq '. + {"'${SERVICE_NAME_UPPERCASE}'_READREPLICA_HOSTS":"'${DB_READREPLICA_HOSTS}'"}') + fi + fi + DBAAS_VARIABLES=$(echo "$DBAAS_VARIABLES" | jq '. + '$(echo "$DB_CONSUMER" | jq -sMrc)'') +done + +for SERVICE_NAME in "${!POSTGRES_DBAAS_CONSUMER_SPECS[@]}" +do + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + DB_HOST=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.primary') + DB_USER=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.username') + DB_PASSWORD=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.password') + DB_NAME=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.database') + DB_PORT=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.port') + DB_CONSUMER='{"'${SERVICE_NAME_UPPERCASE}'_HOST":"'${DB_HOST}'", "'${SERVICE_NAME_UPPERCASE}'_USERNAME":"'${DB_USER}'","'${SERVICE_NAME_UPPERCASE}'_PASSWORD":"'${DB_PASSWORD}'","'${SERVICE_NAME_UPPERCASE}'_DATABASE":"'${DB_NAME}'","'${SERVICE_NAME_UPPERCASE}'_PORT":"'${DB_PORT}'"}' + if DB_READREPLICA_HOSTS=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.replicas | .[]' 2>/dev/null); then + if [ "$DB_READREPLICA_HOSTS" != "null" ]; then + DB_READREPLICA_HOSTS=$(echo "$DB_READREPLICA_HOSTS" | sed 's/^\|$//g' | paste -sd, -) + DB_CONSUMER=$(echo "${DB_CONSUMER}" | jq '. + {"'${SERVICE_NAME_UPPERCASE}'_READREPLICA_HOSTS":"'${DB_READREPLICA_HOSTS}'"}') + fi + fi + DBAAS_VARIABLES=$(echo "$DBAAS_VARIABLES" | jq '. + '$(echo "$DB_CONSUMER" | jq -sMrc)'') +done + +for SERVICE_NAME in "${!MONGODB_DBAAS_CONSUMER_SPECS[@]}" +do + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + DB_HOST=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.primary') + DB_USER=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.username') + DB_PASSWORD=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.password') + DB_NAME=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.database') + DB_PORT=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.port') + DB_AUTHSOURCE=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.auth.source') + DB_AUTHMECHANISM=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.auth.mechanism') + DB_AUTHTLS=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.auth.tls') + DB_CONSUMER='{"'${SERVICE_NAME_UPPERCASE}'_HOST":"'${DB_HOST}'", "'${SERVICE_NAME_UPPERCASE}'_USERNAME":"'${DB_USER}'", "'${SERVICE_NAME_UPPERCASE}'_PASSWORD":"'${DB_PASSWORD}'", "'${SERVICE_NAME_UPPERCASE}'_DATABASE":"'${DB_NAME}'", "'${SERVICE_NAME_UPPERCASE}'_PORT":"'${DB_PORT}'", "'${SERVICE_NAME_UPPERCASE}'_AUTHSOURCE":"'${DB_AUTHSOURCE}'", "'${SERVICE_NAME_UPPERCASE}'_AUTHMECHANISM":"'${DB_AUTHMECHANISM}'", "'${SERVICE_NAME_UPPERCASE}'_AUTHTLS":"'${DB_AUTHTLS}'"}' + DBAAS_VARIABLES=$(echo "$DBAAS_VARIABLES" | jq '. + '$(echo "$DB_CONSUMER" | jq -sMrc)'') +done +echo "$DBAAS_VARIABLES" | jq -Mr > /kubectl-build-deploy/dbaas-creds.json + +# Generate a Config Map with project wide env variables +LAGOON_ENV_YAML_FOLDER="/kubectl-build-deploy/lagoon/lagoon-env" +mkdir -p $LAGOON_ENV_YAML_FOLDER +build-deploy-tool template lagoon-env --saved-templates-path ${LAGOON_ENV_YAML_FOLDER} --dbaas-creds /kubectl-build-deploy/dbaas-creds.json +kubectl apply -n ${NAMESPACE} -f ${LAGOON_ENV_YAML_FOLDER}/lagoon-env-configmap.yaml + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "updateConfigmapComplete" "Update Configmap" "false" previousStepEnd=${currentStepEnd} @@ -1302,11 +1284,11 @@ beginBuildStep "Image Push to Registry" "pushingImages" ### REDEPLOY DEPLOYMENTS IF CONFIG MAP CHANGES ############################################## -CONFIG_MAP_SHA=$(kubectl -n ${NAMESPACE} get configmap lagoon-env -o yaml | shyaml get-value data | sha256sum | awk '{print $1}') +CONFIG_MAP_SHA=$(kubectl -n ${NAMESPACE} get configmap lagoon-env -o yaml | yq -M '.data' | sha256sum | awk '{print $1}') export CONFIG_MAP_SHA # write the configmap to the values file so when we `exec-kubectl-resources-with-images.sh` the deployments will get the value of the config map # which will cause a change in the deployment and trigger a rollout if only the configmap has changed -yq3 write -i -- /kubectl-build-deploy/values.yaml 'configMapSha' $CONFIG_MAP_SHA +yq -i '.configMapSha = "'${CONFIG_MAP_SHA}'"' /kubectl-build-deploy/values.yaml ############################################## ### PUSH IMAGES TO REGISTRY @@ -1513,10 +1495,10 @@ do SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[0]} SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[1]} - SERVICE_ROLLOUT_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.${SERVICE_NAME}.labels.lagoon\\.rollout deployment) + SERVICE_ROLLOUT_TYPE=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$SERVICE_NAME\"'.labels."lagoon.rollout" // "deployment"') # Allow the rollout type to be overriden by environment in .lagoon.yml - ENVIRONMENT_SERVICE_ROLLOUT_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.rollouts.${SERVICE_NAME} false) + ENVIRONMENT_SERVICE_ROLLOUT_TYPE=$(cat .lagoon.yml | yq -o json | jq -r '.environments.'\"${BRANCH//./\\.}\"'.rollouts.'\"$SERVICE_NAME\"' // false') if [ ! $ENVIRONMENT_SERVICE_ROLLOUT_TYPE == "false" ]; then SERVICE_ROLLOUT_TYPE=$ENVIRONMENT_SERVICE_ROLLOUT_TYPE fi diff --git a/legacy/scripts/exec-kubectl-dbaas-wait.sh b/legacy/scripts/exec-kubectl-dbaas-wait.sh new file mode 100644 index 00000000..c3b64984 --- /dev/null +++ b/legacy/scripts/exec-kubectl-dbaas-wait.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# The operator can sometimes take a bit, wait until the details are available +# We added a timeout of 5 minutes (60 retries) before exit +OPERATOR_COUNTER=1 +OPERATOR_TIMEOUT=60 +# use the secret name from the consumer to prevent credential clash +until [ "$(kubectl -n ${NAMESPACE} get ${CONSUMER_TYPE}/${SERVICE_NAME} -o json | jq -r '.spec.consumer.database')" != "null" ]; +do +if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then + consumer_failed=$(kubectl -n ${NAMESPACE} get ${CONSUMER_TYPE}/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') + if [ "${consumer_failed}" == "true" ]; then + echo "Failed to provision a database. Contact your support team to investigate." + exit 1 + fi + let OPERATOR_COUNTER=OPERATOR_COUNTER+1 + echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" + sleep 5 +else + echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" + exit 1 +fi +done + diff --git a/legacy/scripts/exec-kubectl-mariadb-dbaas.sh b/legacy/scripts/exec-kubectl-mariadb-dbaas.sh deleted file mode 100644 index 87a8525f..00000000 --- a/legacy/scripts/exec-kubectl-mariadb-dbaas.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# The operator can sometimes take a bit, wait until the details are available -# We added a timeout of 5 minutes (60 retries) before exit -OPERATOR_COUNTER=1 -OPERATOR_TIMEOUT=60 -# use the secret name from the consumer to prevent credential clash -until kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database -do -if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then - consumer_failed=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') - if [ "${consumer_failed}" == "true" ]; then - echo "Failed to provision a database. Contact your support team to investigate." - exit 1 - fi - let OPERATOR_COUNTER=OPERATOR_COUNTER+1 - echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" - sleep 5 -else - echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" - exit 1 -fi -done - -# Grab the details from the consumer spec -DB_HOST=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary) -DB_USER=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username) -DB_PASSWORD=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.password) -DB_NAME=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database) -DB_PORT=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.port) - -# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker -kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}" - -# only add the DB_READREPLICA_HOSTS variable if it exists in the consumer spec -# since the operator can support multiple replica hosts being defined, we should comma seperate them here -if DB_READREPLICA_HOSTS=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.replicas); then - DB_READREPLICA_HOSTS=$(echo $DB_READREPLICA_HOSTS | cut -c 3- | rev | cut -c 1- | rev | sed 's/^\|$//g' | paste -sd, -) - yq3 write -i -- /kubectl-build-deploy/${SERVICE_NAME}-values.yaml 'readReplicaHosts' $DB_READREPLICA_HOSTS - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_READREPLICA_HOSTS\":\"${DB_READREPLICA_HOSTS}\"}}" -fi diff --git a/legacy/scripts/exec-kubectl-mongodb-dbaas.sh b/legacy/scripts/exec-kubectl-mongodb-dbaas.sh deleted file mode 100644 index fc4211bf..00000000 --- a/legacy/scripts/exec-kubectl-mongodb-dbaas.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# The operator can sometimes take a bit, wait until the details are available -# We added a timeout of 5 minutes (60 retries) before exit -OPERATOR_COUNTER=1 -OPERATOR_TIMEOUT=60 -# use the secret name from the consumer to prevent credential clash -until kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database -do -if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then - consumer_failed=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') - if [ "${consumer_failed}" == "true" ]; then - echo "Failed to provision a database. Contact your support team to investigate." - exit 1 - fi - let OPERATOR_COUNTER=OPERATOR_COUNTER+1 - echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" - sleep 5 -else - echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" - exit 1 -fi -done - -# Grab the details from the consumer spec -DB_HOST=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary) -DB_USER=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username) -DB_PASSWORD=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.password) -DB_NAME=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database) -DB_PORT=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.port) -DB_AUTHSOURCE=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.auth.source) -DB_AUTHMECHANISM=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.auth.mechanism) -DB_AUTHTLS=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.auth.tls) - -# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker -kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\", \"${SERVICE_NAME_UPPERCASE}_AUTHSOURCE\":\"${DB_AUTHSOURCE}\", \"${SERVICE_NAME_UPPERCASE}_AUTHMECHANISM\":\"${DB_AUTHMECHANISM}\", \"${SERVICE_NAME_UPPERCASE}_AUTHTLS\":\"${DB_AUTHTLS}\" }}" diff --git a/legacy/scripts/exec-kubectl-postgres-dbaas.sh b/legacy/scripts/exec-kubectl-postgres-dbaas.sh deleted file mode 100644 index 54b4dd29..00000000 --- a/legacy/scripts/exec-kubectl-postgres-dbaas.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# The operator can sometimes take a bit, wait until the details are available -# We added a timeout of 5 minutes (60 retries) before exit -OPERATOR_COUNTER=1 -OPERATOR_TIMEOUT=60 -# use the secret name from the consumer to prevent credential clash -until kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database -do -if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then - consumer_failed=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') - if [ "${consumer_failed}" == "true" ]; then - echo "Failed to provision a database. Contact your support team to investigate." - exit 1 - fi - let OPERATOR_COUNTER=OPERATOR_COUNTER+1 - echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" - sleep 5 -else - echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" - exit 1 -fi -done - -# Grab the details from the consumer spec -DB_HOST=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary) -DB_USER=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username) -DB_PASSWORD=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.password) -DB_NAME=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database) -DB_PORT=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.port) - -# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker -kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}" - -# only add the DB_READREPLICA_HOSTS variable if it exists in the consumer spec -# since the operator can support multiple replica hosts being defined, we should comma seperate them here -if DB_READREPLICA_HOSTS=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.replicas); then - DB_READREPLICA_HOSTS=$(echo $DB_READREPLICA_HOSTS | cut -c 3- | rev | cut -c 1- | rev | sed 's/^\|$//g' | paste -sd, -) - yq3 write -i -- /kubectl-build-deploy/${SERVICE_NAME}-values.yaml 'readReplicaHosts' $DB_READREPLICA_HOSTS - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_READREPLICA_HOSTS\":\"${DB_READREPLICA_HOSTS}\"}}" -fi