From 1b57fb0b6fd396206b875de58bb5c302c5b8b289 Mon Sep 17 00:00:00 2001 From: shreddedbacon <b@benjackson.email> Date: Mon, 29 Jul 2024 17:25:39 +1000 Subject: [PATCH] feat: template lagoonenv configmap --- Dockerfile | 4 +- cmd/root.go | 2 + cmd/template_lagoonenv.go | 103 ++++++ cmd/template_lagoonenv_test.go | 240 +++++++++++++ internal/generator/build_data.go | 24 +- internal/generator/buildvalues.go | 6 + internal/generator/generator.go | 33 +- .../lagoonenv/template_configmap.go | 66 ++++ .../lagoonenv/template_configmap_test.go | 93 +++++ .../test-resources/lagoon-env-1.yaml | 20 ++ .../lagoonenv1/lagoon-env-configmap.yaml | 36 ++ .../lagoonenv2/lagoon-env-configmap.yaml | 52 +++ internal/testdata/basic/lagoonenv2-creds.json | 24 ++ legacy/build-deploy-docker-compose.sh | 318 +++++++++--------- legacy/scripts/exec-kubectl-dbaas-wait.sh | 24 ++ legacy/scripts/exec-kubectl-mariadb-dbaas.sh | 47 --- legacy/scripts/exec-kubectl-mongodb-dbaas.sh | 39 --- legacy/scripts/exec-kubectl-postgres-dbaas.sh | 47 --- 18 files changed, 866 insertions(+), 312 deletions(-) create mode 100644 cmd/template_lagoonenv.go create mode 100644 cmd/template_lagoonenv_test.go create mode 100644 internal/templating/lagoonenv/template_configmap.go create mode 100644 internal/templating/lagoonenv/template_configmap_test.go create mode 100644 internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml create mode 100644 internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml create mode 100644 internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml create mode 100644 internal/testdata/basic/lagoonenv2-creds.json create mode 100644 legacy/scripts/exec-kubectl-dbaas-wait.sh delete mode 100644 legacy/scripts/exec-kubectl-mariadb-dbaas.sh delete mode 100644 legacy/scripts/exec-kubectl-mongodb-dbaas.sh delete mode 100644 legacy/scripts/exec-kubectl-postgres-dbaas.sh diff --git a/Dockerfile b/Dockerfile index 56433b36..a7b13ffa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,13 +78,11 @@ RUN apk add -U --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing au && apk upgrade --no-cache openssh openssh-keygen openssh-client-common openssh-client-default \ && apk add --no-cache openssl curl jq parallel bash git py-pip skopeo \ && git config --global user.email "lagoon@lagoon.io" && git config --global user.name lagoon \ - && pip install --break-system-packages shyaml yq + && pip install --break-system-packages yq RUN architecture=$(case $(uname -m) in x86_64 | amd64) echo "amd64" ;; aarch64 | arm64 | armv8) echo "arm64" ;; *) echo "amd64" ;; esac) \ && curl -Lo /usr/bin/kubectl https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/${architecture}/kubectl \ && chmod +x /usr/bin/kubectl \ - && curl -Lo /usr/bin/yq3 https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${architecture} \ - && chmod +x /usr/bin/yq3 \ && curl -Lo /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.35.2/yq_linux_${architecture} \ && chmod +x /usr/bin/yq \ && curl -Lo /tmp/helm.tar.gz https://get.helm.sh/helm-${HELM_VERSION}-linux-${architecture}.tar.gz \ diff --git a/cmd/root.go b/cmd/root.go index e88ae3cc..72b7e839 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -153,6 +153,8 @@ func init() { "Ignore missing env_file files (true by default, subject to change).") rootCmd.PersistentFlags().StringP("images", "", "", "JSON representation of service:image reference") + rootCmd.PersistentFlags().StringP("dbaas-creds", "", "", + "JSON representation of dbaas credential references") } // initConfig reads in config file and ENV variables if set. diff --git a/cmd/template_lagoonenv.go b/cmd/template_lagoonenv.go new file mode 100644 index 00000000..975b30bc --- /dev/null +++ b/cmd/template_lagoonenv.go @@ -0,0 +1,103 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + generator "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + "github.com/uselagoon/build-deploy-tool/internal/templating/lagoonenv" + "sigs.k8s.io/yaml" +) + +type DBaaSCredRefs []map[string]string + +var lagoonEnvGeneration = &cobra.Command{ + Use: "lagoon-env", + Aliases: []string{"le"}, + Short: "Generate the lagoon-env configmap template for a Lagoon build", + RunE: func(cmd *cobra.Command, args []string) error { + generator, err := generator.GenerateInput(*rootCmd, true) + if err != nil { + return err + } + routes, err := cmd.Flags().GetString("routes") + if err != nil { + return fmt.Errorf("error reading routes flag: %v", err) + } + dbaasCreds, err := rootCmd.PersistentFlags().GetString("dbaas-creds") + if err != nil { + return fmt.Errorf("error reading images flag: %v", err) + } + dbaasCredRefs, err := loadCredsFromFile(dbaasCreds) + if err != nil { + return err + } + dbCreds := map[string]string{} + for _, v := range *dbaasCredRefs { + for k, v1 := range v { + dbCreds[k] = v1 + } + } + generator.DBaaSVariables = dbCreds + return LagoonEnvTemplateGeneration(generator, routes) + }, +} + +func loadCredsFromFile(file string) (*DBaaSCredRefs, error) { + dbaasCredRefs := &DBaaSCredRefs{} + dbaasCredJSON, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("couldn't read file %v: %v", file, err) + } + if err := json.Unmarshal(dbaasCredJSON, dbaasCredRefs); err != nil { + return nil, fmt.Errorf("error unmarshalling images payload: %v", err) + } + return dbaasCredRefs, nil +} + +// LagoonEnvTemplateGeneration . +func LagoonEnvTemplateGeneration( + g generator.GeneratorInput, + routes string, +) error { + lagoonBuild, err := generator.NewGenerator( + g, + ) + if err != nil { + return err + } + savedTemplates := g.SavedTemplatesPath + // if the routes have been passed from the command line, use them instead. we do this since lagoon currently doesn't enforce route state to match + // what is in the `.lagoon.yml` file, so there may be items that exist in the cluster that don't exist in yaml + // eventually once route state enforcement is enforced, or the tool can reconcile what is in the cluster itself rather than in bash + // then this can be removed + // https://github.com/uselagoon/build-deploy-tool/blob/f527a89ad5efb46e19a2f59d9ff3ffbff541e2a2/legacy/build-deploy-docker-compose.sh#L1090 + if routes != "" { + lagoonBuild.BuildValues.Routes = strings.Split(routes, ",") + } + cm, err := lagoonenv.GenerateLagoonEnvConfigMap(*lagoonBuild.BuildValues) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + cmBytes, err := yaml.Marshal(cm) + if err != nil { + return fmt.Errorf("couldn't generate template: %v", err) + } + if len(cmBytes) > 0 { + if g.Debug { + fmt.Printf("Templating lagoon-env configmap %s\n", fmt.Sprintf("%s/%s.yaml", savedTemplates, "lagoon-env-configmap")) + } + helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "lagoon-env-configmap"), cmBytes) + } + return nil +} + +func init() { + templateCmd.AddCommand(lagoonEnvGeneration) + lagoonEnvGeneration.Flags().StringP("routes", "R", "", + "The routes from the environment") +} diff --git a/cmd/template_lagoonenv_test.go b/cmd/template_lagoonenv_test.go new file mode 100644 index 00000000..619905db --- /dev/null +++ b/cmd/template_lagoonenv_test.go @@ -0,0 +1,240 @@ +package cmd + +import ( + "fmt" + "os" + "reflect" + "testing" + + "github.com/andreyvit/diff" + "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" + "github.com/uselagoon/build-deploy-tool/internal/helpers" + "github.com/uselagoon/build-deploy-tool/internal/lagoon" + "github.com/uselagoon/build-deploy-tool/internal/testdata" +) + +func TestLagoonEnvTemplateGeneration(t *testing.T) { + tests := []struct { + name string + description string + args testdata.TestData + templatePath string + want string + dbaasCreds string + vars []helpers.EnvironmentVariable + }{ + { + name: "test1 basic deployment", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/basic/lagoon.yml", + ProjectVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE1", + Value: "myspecialvariable1", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2", + Scope: "runtime", + }, + { + Name: "MY_SPECIAL_VARIABLE3", + Value: "myspecialvariable3", + Scope: "build", + }, + { + Name: "MY_SPECIAL_VARIABLE", + Value: "myspecialvariable", + Scope: "global", + }, + { + Name: "LAGOON_SYSTEM_CORE_VERSION", + Value: "v2.19.0", + Scope: "internal_system", + }, + { + Name: "REGISTRY_PASSWORD", + Value: "myenvvarregistrypassword", + Scope: "container_registry", + }, + }, + EnvVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2-env-override", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE4", + Value: "myspecialvariable4", + Scope: "runtime", + }, + }, + }, true), + templatePath: "testoutput", + want: "internal/testdata/basic/configmap-templates/lagoonenv1", + }, + { + name: "test1 basic deployment with mariadb creds", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/basic/lagoon.yml", + ProjectVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE1", + Value: "myspecialvariable1", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2", + Scope: "runtime", + }, + { + Name: "MY_SPECIAL_VARIABLE3", + Value: "myspecialvariable3", + Scope: "build", + }, + { + Name: "MY_SPECIAL_VARIABLE", + Value: "myspecialvariable", + Scope: "global", + }, + { + Name: "LAGOON_SYSTEM_CORE_VERSION", + Value: "v2.19.0", + Scope: "internal_system", + }, + { + Name: "REGISTRY_PASSWORD", + Value: "myenvvarregistrypassword", + Scope: "container_registry", + }, + }, + EnvVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2-env-override", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE4", + Value: "myspecialvariable4", + Scope: "runtime", + }, + }, + }, true), + dbaasCreds: "internal/testdata/basic/lagoonenv2-creds.json", + templatePath: "testoutput", + want: "internal/testdata/basic/configmap-templates/lagoonenv2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + helpers.UnsetEnvVars(tt.vars) //unset variables before running tests + for _, envVar := range tt.vars { + err := os.Setenv(envVar.Name, envVar.Value) + if err != nil { + t.Errorf("%v", err) + } + } + // set the environment variables from args + savedTemplates := tt.templatePath + generator, err := testdata.SetupEnvironment(*rootCmd, savedTemplates, tt.args) + if err != nil { + t.Errorf("%v", err) + } + + err = os.MkdirAll(savedTemplates, 0755) + if err != nil { + t.Errorf("couldn't create directory %v: %v", savedTemplates, err) + } + + defer os.RemoveAll(savedTemplates) + + ts := dbaasclient.TestDBaaSHTTPServer() + defer ts.Close() + err = os.Setenv("DBAAS_OPERATOR_HTTP", ts.URL) + if err != nil { + t.Errorf("%v", err) + } + dbaasCreds := &DBaaSCredRefs{} + if tt.dbaasCreds != "" { + dbaasCreds, err = loadCredsFromFile(tt.dbaasCreds) + if err != nil { + t.Errorf("%v", err) + } + dbCreds := map[string]string{} + for _, v := range *dbaasCreds { + for k, v1 := range v { + dbCreds[k] = v1 + } + } + generator.DBaaSVariables = dbCreds + } + err = LagoonEnvTemplateGeneration(generator, "") + if err != nil { + t.Errorf("%v", err) + } + + files, err := os.ReadDir(savedTemplates) + if err != nil { + t.Errorf("couldn't read directory %v: %v", savedTemplates, err) + } + results, err := os.ReadDir(tt.want) + if err != nil { + t.Errorf("couldn't read directory %v: %v", tt.want, err) + } + if len(files) != len(results) { + for _, f := range files { + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + fmt.Println(string(f1)) + } + t.Errorf("number of generated templates doesn't match results %v/%v: %v", len(files), len(results), err) + } + fCount := 0 + for _, f := range files { + for _, r := range results { + if f.Name() == r.Name() { + fCount++ + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + r1, err := os.ReadFile(fmt.Sprintf("%s/%s", tt.want, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + if !reflect.DeepEqual(f1, r1) { + t.Errorf("LagoonEnvTemplateGeneration() = \n%v", diff.LineDiff(string(r1), string(f1))) + } + } + } + } + if fCount != len(files) { + for _, f := range files { + f1, err := os.ReadFile(fmt.Sprintf("%s/%s", savedTemplates, f.Name())) + if err != nil { + t.Errorf("couldn't read file %v: %v", savedTemplates, err) + } + fmt.Println(string(f1)) + } + t.Errorf("resulting templates do not match") + } + t.Cleanup(func() { + helpers.UnsetEnvVars(tt.vars) + }) + }) + } +} diff --git a/internal/generator/build_data.go b/internal/generator/build_data.go index efdbb67c..e34fc4cf 100644 --- a/internal/generator/build_data.go +++ b/internal/generator/build_data.go @@ -9,13 +9,15 @@ import ( ) // this creates a bunch of standard environment variables that are injected into the `lagoon-env` configmap normally -func collectBuildVariables(buildValues BuildValues) []lagoon.EnvironmentVariable { +func collectLagoonEnvConfigmapVariables(buildValues BuildValues) []lagoon.EnvironmentVariable { vars := []lagoon.EnvironmentVariable{} vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_PROJECT", Value: buildValues.Project, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ENVIRONMENT", Value: buildValues.Environment, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ENVIRONMENT_TYPE", Value: buildValues.EnvironmentType, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_GIT_SHA", Value: buildValues.GitSHA, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_KUBERNETES", Value: buildValues.Kubernetes, Scope: "runtime"}) + // LAGOON_GIT_SAFE_BRANCH is pointing to the enviornment name, therefore also is filled if this environment + // is created by a PR or Promote workflow. This technically wrong, therefore will be removed vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_GIT_SAFE_BRANCH", Value: buildValues.Environment, Scope: "runtime"}) //deprecated??? (https://github.com/uselagoon/lagoon/blob/1053965321495213591f4c9110f90a9d9dcfc946/images/kubectl-build-deploy-dind/build-deploy-docker-compose.sh#L748) if buildValues.BuildType == "branch" { vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_GIT_BRANCH", Value: buildValues.Branch, Scope: "runtime"}) @@ -26,15 +28,23 @@ func collectBuildVariables(buildValues BuildValues) []lagoon.EnvironmentVariable vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_PR_TITLE", Value: buildValues.PRTitle, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_PR_NUMBER", Value: buildValues.PRNumber, Scope: "runtime"}) } - if buildValues.ActiveEnvironment != "" { - vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ACTIVE_ENVIRONMENT", Value: buildValues.ActiveEnvironment, Scope: "runtime"}) - } - if buildValues.StandbyEnvironment != "" { - vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_STANDBY_ENVIRONMENT", Value: buildValues.StandbyEnvironment, Scope: "runtime"}) - } + // @TODO: check if these would actually be useful, they've never been used by anything before + // commenting out for now + // if buildValues.ActiveEnvironment != "" { + // vars = append( vars, lagoon.EnvironmentVariable{Name: "LAGOON_ACTIVE_ENVIRONMENT", Value: buildValues.ActiveEnvironment, Scope: "runtime"}) + // } + // if buildValues.StandbyEnvironment != "" { + // vars = append( vars, lagoon.EnvironmentVariable{Name: "LAGOON_STANDBY_ENVIRONMENT", Value: buildValues.StandbyEnvironment, Scope: "runtime"}) + // } vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ROUTE", Value: buildValues.Route, Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_ROUTES", Value: strings.Join(buildValues.Routes, ","), Scope: "runtime"}) vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_AUTOGENERATED_ROUTES", Value: strings.Join(buildValues.AutogeneratedRoutes, ","), Scope: "runtime"}) + // add the api/token/ssh configuration variables to envvars + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_API_HOST", Value: buildValues.ConfigAPIHost, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_TOKEN_HOST", Value: buildValues.ConfigTokenHost, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_TOKEN_PORT", Value: buildValues.ConfigTokenPort, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_SSH_HOST", Value: buildValues.ConfigSSHHost, Scope: "runtime"}) + vars = append(vars, lagoon.EnvironmentVariable{Name: "LAGOON_CONFIG_SSH_PORT", Value: buildValues.ConfigSSHPort, Scope: "runtime"}) return vars } diff --git a/internal/generator/buildvalues.go b/internal/generator/buildvalues.go index bc1a20c9..530aa29b 100644 --- a/internal/generator/buildvalues.go +++ b/internal/generator/buildvalues.go @@ -82,6 +82,12 @@ type BuildValues struct { ForcePullImages []string `json:"forcePullImages"` Volumes []ComposeVolume `json:"volumes,omitempty" description:"stores any additional persistent volume definitions"` PodAntiAffinity bool `json:"podAntiAffinity"` + ConfigAPIHost string `json:"configAPIHost"` + ConfigTokenHost string `json:"configTokenHost"` + ConfigTokenPort string `json:"configTokenPort"` + ConfigSSHHost string `json:"configSSHHost"` + ConfigSSHPort string `json:"configSSHPort"` + DBaaSVariables map[string]string `json:"dbaasVariables" description:"map of variables provided by dbaas consumers"` } type Resources struct { diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 0809f129..b623cbfa 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -65,6 +65,12 @@ type GeneratorInput struct { DynamicDBaaSSecrets []string ImageCacheBuildArgsJSON string SSHPrivateKey string + ConfigAPIHost string + ConfigTokenHost string + ConfigTokenPort string + ConfigSSHHost string + ConfigSSHPort string + DBaaSVariables map[string]string } func NewGenerator( @@ -113,6 +119,16 @@ func NewGenerator( // this is used by CI systems to influence builds, it is rarely used and should probably be abandoned buildValues.IsCI = helpers.GetEnvBool("CI", generator.CI, generator.Debug) + // add dbaas credentials to build values for injection into configmap + buildValues.DBaaSVariables = generator.DBaaSVariables + + // set the lagoon config variables + buildValues.ConfigAPIHost = helpers.GetEnv("LAGOON_CONFIG_API_HOST", generator.ConfigAPIHost, generator.Debug) + buildValues.ConfigTokenHost = helpers.GetEnv("LAGOON_CONFIG_TOKEN_HOST", generator.ConfigTokenHost, generator.Debug) + buildValues.ConfigTokenPort = helpers.GetEnv("LAGOON_CONFIG_TOKEN_PORT", generator.ConfigTokenPort, generator.Debug) + buildValues.ConfigSSHHost = helpers.GetEnv("LAGOON_CONFIG_SSH_HOST", generator.ConfigSSHHost, generator.Debug) + buildValues.ConfigSSHPort = helpers.GetEnv("LAGOON_CONFIG_SSH_PORT", generator.ConfigSSHPort, generator.Debug) + buildValues.ConfigMapSha = configMapSha buildValues.BuildName = buildName buildValues.Kubernetes = kubernetes @@ -120,6 +136,7 @@ func NewGenerator( buildValues.ImageRegistry = imageRegistry buildValues.SourceRepository = sourceRepository buildValues.PromotionSourceEnvironment = promotionSourceEnvironment + // get the image references values from the build images output buildValues.ImageReferences = generator.ImageReferences defaultBackupSchedule := helpers.GetEnv("DEFAULT_BACKUP_SCHEDULE", generator.DefaultBackupSchedule, generator.Debug) @@ -249,12 +266,9 @@ func NewGenerator( envVars := []lagoon.EnvironmentVariable{} json.Unmarshal([]byte(projectVariables), &projectVars) json.Unmarshal([]byte(environmentVariables), &envVars) - mergedVariables := lagoon.MergeVariables(projectVars, envVars) - // collect a bunch of the default LAGOON_X based build variables that are injected into `lagoon-env` and make them available - configVars := collectBuildVariables(buildValues) - // add the calculated build runtime variables into the existing variable slice - // this will later be used to add `runtime|global` scope into the `lagoon-env` configmap - buildValues.EnvironmentVariables = lagoon.MergeVariables(mergedVariables, configVars) + + // set the environment variables to all the known merged variables so far + buildValues.EnvironmentVariables = lagoon.MergeVariables(projectVars, envVars) // if the core version is provided from the API, set the buildvalues LagoonVersion to this instead lagoonCoreVersion, _ := lagoon.GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, buildValues.EnvironmentVariables) @@ -455,6 +469,13 @@ func NewGenerator( } /* end route generation configuration */ + // collect a bunch of the default LAGOON_X based build variables that are injected into `lagoon-env` and make them available + configVars := collectLagoonEnvConfigmapVariables(buildValues) + + // add the calculated build runtime variables into the existing variable slice + // this will later be used to add `runtime|global` scope into the `lagoon-env` configmap + buildValues.EnvironmentVariables = lagoon.MergeVariables(buildValues.EnvironmentVariables, configVars) + // finally return the generator values, this should be a mostly complete version of the resulting data needed for a build // another step will collect the current or known state of a build. // the output of the generator and the output of that state collector will eventually replace a lot of the legacy BASH script diff --git a/internal/templating/lagoonenv/template_configmap.go b/internal/templating/lagoonenv/template_configmap.go new file mode 100644 index 00000000..6bc887f7 --- /dev/null +++ b/internal/templating/lagoonenv/template_configmap.go @@ -0,0 +1,66 @@ +package lagoonenv + +import ( + "github.com/uselagoon/build-deploy-tool/internal/generator" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GenerateLagoonEnvConfigMap generates the lagoon template to apply. +func GenerateLagoonEnvConfigMap( + buildValues generator.BuildValues, +) (corev1.ConfigMap, error) { + + // add the default labels + labels := map[string]string{ + "app.kubernetes.io/managed-by": "build-deploy-tool", + "app.kubernetes.io/instance": "lagoon-env", + "app.kubernetes.io/name": "lagoon-env", + "lagoon.sh/template": "lagoon-env-0.1.0", + "lagoon.sh/project": buildValues.Project, + "lagoon.sh/environment": buildValues.Environment, + "lagoon.sh/environmentType": buildValues.EnvironmentType, + "lagoon.sh/buildType": buildValues.BuildType, + } + + // add the default annotations + annotations := map[string]string{} + + // add any additional labels + if buildValues.BuildType == "branch" { + annotations["lagoon.sh/branch"] = buildValues.Branch + } else if buildValues.BuildType == "pullrequest" { + annotations["lagoon.sh/prNumber"] = buildValues.PRNumber + annotations["lagoon.sh/prHeadBranch"] = buildValues.PRHeadBranch + annotations["lagoon.sh/prBaseBranch"] = buildValues.PRBaseBranch + } + + variables := map[string]string{} + + // add variables from the project/environment/build created variables + for _, v := range buildValues.EnvironmentVariables { + if v.Scope == "global" || v.Scope == "runtime" { + variables[v.Name] = v.Value + } + } + + // add dbaas variables to lagoon-env + for k, v := range buildValues.DBaaSVariables { + variables[k] = v + } + + lagoonEnv := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "lagoon-env", + Labels: labels, + Annotations: annotations, + }, + Data: variables, + } + + return lagoonEnv, nil +} diff --git a/internal/templating/lagoonenv/template_configmap_test.go b/internal/templating/lagoonenv/template_configmap_test.go new file mode 100644 index 00000000..cc238d5c --- /dev/null +++ b/internal/templating/lagoonenv/template_configmap_test.go @@ -0,0 +1,93 @@ +package lagoonenv + +import ( + "os" + "reflect" + "testing" + + "github.com/andreyvit/diff" + "github.com/uselagoon/build-deploy-tool/internal/generator" + "github.com/uselagoon/build-deploy-tool/internal/lagoon" + "sigs.k8s.io/yaml" +) + +func TestGenerateLagoonEnvConfigMap(t *testing.T) { + type args struct { + buildValues generator.BuildValues + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test1", + args: args{ + buildValues: generator.BuildValues{ + Project: "example-project", + Environment: "environment-name", + EnvironmentType: "production", + Namespace: "myexample-project-environment-name", + BuildType: "branch", + LagoonVersion: "v2.x.x", + Kubernetes: "generator.local", + Branch: "environment-name", + EnvironmentVariables: []lagoon.EnvironmentVariable{ + { + Name: "MY_SPECIAL_VARIABLE1", + Value: "myspecialvariable1", + Scope: "global", + }, + { + Name: "MY_SPECIAL_VARIABLE2", + Value: "myspecialvariable2", + Scope: "runtime", + }, + { + Name: "MY_SPECIAL_VARIABLE3", + Value: "myspecialvariable3", + Scope: "build", + }, + { + Name: "MY_SPECIAL_VARIABLE", + Value: "myspecialvariable", + Scope: "global", + }, + { + Name: "LAGOON_SYSTEM_CORE_VERSION", + Value: "v2.19.0", + Scope: "internal_system", + }, + { + Name: "REGISTRY_PASSWORD", + Value: "myenvvarregistrypassword", + Scope: "container_registry", + }, + }, + }, + }, + want: "test-resources/lagoon-env-1.yaml", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GenerateLagoonEnvConfigMap(tt.args.buildValues) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateLagoonEnvConfigMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + r1, err := os.ReadFile(tt.want) + if err != nil { + t.Errorf("couldn't read file %v: %v", tt.want, err) + } + cm, err := yaml.Marshal(got) + if err != nil { + t.Errorf("couldn't generate template %v", err) + } + if !reflect.DeepEqual(string(cm), string(r1)) { + t.Errorf("GenerateLagoonEnvConfigMap() = \n%v", diff.LineDiff(string(r1), string(cm))) + } + }) + } +} diff --git a/internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml b/internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml new file mode 100644 index 00000000..3278abaa --- /dev/null +++ b/internal/templating/lagoonenv/test-resources/lagoon-env-1.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +data: + MY_SPECIAL_VARIABLE: myspecialvariable + MY_SPECIAL_VARIABLE1: myspecialvariable1 + MY_SPECIAL_VARIABLE2: myspecialvariable2 +kind: ConfigMap +metadata: + annotations: + lagoon.sh/branch: environment-name + creationTimestamp: null + labels: + app.kubernetes.io/instance: lagoon-env + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: lagoon-env + lagoon.sh/buildType: branch + lagoon.sh/environment: environment-name + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/template: lagoon-env-0.1.0 + name: lagoon-env diff --git a/internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml b/internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml new file mode 100644 index 00000000..4c90e747 --- /dev/null +++ b/internal/testdata/basic/configmap-templates/lagoonenv1/lagoon-env-configmap.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +data: + LAGOON_AUTOGENERATED_ROUTES: https://node-example-project-main.example.com + LAGOON_CONFIG_API_HOST: "" + LAGOON_CONFIG_SSH_HOST: "" + LAGOON_CONFIG_SSH_PORT: "" + LAGOON_CONFIG_TOKEN_HOST: "" + LAGOON_CONFIG_TOKEN_PORT: "" + LAGOON_ENVIRONMENT: main + LAGOON_ENVIRONMENT_TYPE: production + LAGOON_GIT_BRANCH: main + LAGOON_GIT_SAFE_BRANCH: main + LAGOON_GIT_SHA: abcdefg123456 + LAGOON_KUBERNETES: remote-cluster1 + LAGOON_PROJECT: example-project + LAGOON_ROUTE: https://example.com + LAGOON_ROUTES: https://node-example-project-main.example.com,https://example.com + MY_SPECIAL_VARIABLE: myspecialvariable + MY_SPECIAL_VARIABLE1: myspecialvariable1 + MY_SPECIAL_VARIABLE2: myspecialvariable2-env-override + MY_SPECIAL_VARIABLE4: myspecialvariable4 +kind: ConfigMap +metadata: + annotations: + lagoon.sh/branch: main + creationTimestamp: null + labels: + app.kubernetes.io/instance: lagoon-env + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: lagoon-env + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/template: lagoon-env-0.1.0 + name: lagoon-env diff --git a/internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml b/internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml new file mode 100644 index 00000000..c9d919f8 --- /dev/null +++ b/internal/testdata/basic/configmap-templates/lagoonenv2/lagoon-env-configmap.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +data: + LAGOON_AUTOGENERATED_ROUTES: https://node-example-project-main.example.com + LAGOON_CONFIG_API_HOST: "" + LAGOON_CONFIG_SSH_HOST: "" + LAGOON_CONFIG_SSH_PORT: "" + LAGOON_CONFIG_TOKEN_HOST: "" + LAGOON_CONFIG_TOKEN_PORT: "" + LAGOON_ENVIRONMENT: main + LAGOON_ENVIRONMENT_TYPE: production + LAGOON_GIT_BRANCH: main + LAGOON_GIT_SAFE_BRANCH: main + LAGOON_GIT_SHA: abcdefg123456 + LAGOON_KUBERNETES: remote-cluster1 + LAGOON_PROJECT: example-project + LAGOON_ROUTE: https://example.com + LAGOON_ROUTES: https://node-example-project-main.example.com,https://example.com + MARIADB_DATABASE: example-project-main_LMq2Q + MARIADB_HOST: mariadb-abcdef + MARIADB_PASSWORD: juD9RzjCEKbOYucpI5jVqGmr + MARIADB_PORT: "3306" + MARIADB_USERNAME: example-project-main_fO2Fo + MARIADB2_DATABASE: example-project-main_df23s + MARIADB2_HOST: mariadb2-abcdef + MARIADB2_PASSWORD: juD9RzjCEKbOYucpI5jVqGmr + MARIADB2_PORT: "3306" + MARIADB2_USERNAME: example-project-main_f3d1o + MARIADB3_DATABASE: example-project-main_sa241 + MARIADB3_HOST: mariadb3-abcdef + MARIADB3_PASSWORD: juD9RzjCEKbOYucpI5jVqGmr + MARIADB3_PORT: "3306" + MARIADB3_READREPLICA_HOSTS: readreplica-mariadb3-efg-321abc,readreplica-mariadb3-abc123-efg + MARIADB3_USERNAME: example-project-main_as24 + MY_SPECIAL_VARIABLE: myspecialvariable + MY_SPECIAL_VARIABLE1: myspecialvariable1 + MY_SPECIAL_VARIABLE2: myspecialvariable2-env-override + MY_SPECIAL_VARIABLE4: myspecialvariable4 +kind: ConfigMap +metadata: + annotations: + lagoon.sh/branch: main + creationTimestamp: null + labels: + app.kubernetes.io/instance: lagoon-env + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: lagoon-env + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/template: lagoon-env-0.1.0 + name: lagoon-env diff --git a/internal/testdata/basic/lagoonenv2-creds.json b/internal/testdata/basic/lagoonenv2-creds.json new file mode 100644 index 00000000..a2793f6c --- /dev/null +++ b/internal/testdata/basic/lagoonenv2-creds.json @@ -0,0 +1,24 @@ +[ + { + "MARIADB_HOST": "mariadb-abcdef", + "MARIADB_USERNAME": "example-project-main_fO2Fo", + "MARIADB_PASSWORD": "juD9RzjCEKbOYucpI5jVqGmr", + "MARIADB_DATABASE": "example-project-main_LMq2Q", + "MARIADB_PORT": "3306" + }, + { + "MARIADB3_HOST": "mariadb3-abcdef", + "MARIADB3_USERNAME": "example-project-main_as24", + "MARIADB3_PASSWORD": "juD9RzjCEKbOYucpI5jVqGmr", + "MARIADB3_DATABASE": "example-project-main_sa241", + "MARIADB3_PORT": "3306", + "MARIADB3_READREPLICA_HOSTS": "readreplica-mariadb3-efg-321abc,readreplica-mariadb3-abc123-efg" + }, + { + "MARIADB2_HOST": "mariadb2-abcdef", + "MARIADB2_USERNAME": "example-project-main_f3d1o", + "MARIADB2_PASSWORD": "juD9RzjCEKbOYucpI5jVqGmr", + "MARIADB2_DATABASE": "example-project-main_df23s", + "MARIADB2_PORT": "3306" + } +] diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 60d88357..115d1757 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -161,8 +161,19 @@ if [ ! -z "$(featureFlag IMAGECACHE_REGISTRY)" ]; then [[ $last_char != "/" ]] && IMAGECACHE_REGISTRY="$IMAGECACHE_REGISTRY/"; : fi -# Load path of docker-compose that should be used -DOCKER_COMPOSE_YAML=($(cat .lagoon.yml | shyaml get-value docker-compose-yaml)) +set +e +currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" +patchBuildStep "${buildStartTime}" "${buildStartTime}" "${currentStepEnd}" "${NAMESPACE}" "initialSetup" "Initial Environment Setup" "false" +previousStepEnd=${currentStepEnd} + +# Validate `lagoon.yml` first to try detect any errors here first +beginBuildStep ".lagoon.yml Validation" "lagoonYmlValidation" +############################################## +### RUN lagoon-yml validation against the final data which may have overrides +### from .lagoon.override.yml file or LAGOON_YAML_OVERRIDE environment variable +############################################## +lyvOutput=$(bash -c 'build-deploy-tool validate lagoon-yml; exit $?' 2>&1) +lyvExit=$? echo "Updating lagoon-yaml configmap with a pre-deploy version of the .lagoon.yml file" if kubectl -n ${NAMESPACE} get configmap lagoon-yaml &> /dev/null; then @@ -181,6 +192,39 @@ if kubectl -n ${NAMESPACE} get configmap lagoon-yaml &> /dev/null; then # create it kubectl -n ${NAMESPACE} create configmap lagoon-yaml --from-file=pre-deploy=.lagoon.yml fi + +if [ "${lyvExit}" != "0" ]; then + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" + patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "lagoonYmlValidationError" ".lagoon.yml Validation" "false" + previousStepEnd=${currentStepEnd} + echo " +############################################## +Warning! +There are issues with your .lagoon.yml file that must be fixed. +Refer to the .lagoon.yml docs for the correct syntax +https://docs.lagoon.sh/using-lagoon-the-basics/lagoon-yml/ +############################################## +" + echo "${lyvOutput}" + echo " +##############################################" + exit 1 +fi + +# The attempt to valid the `docker-compose.yaml` file +beginBuildStep "Docker Compose Validation" "dockerComposeValidation" + +# Load path of docker-compose that should be used +DOCKER_COMPOSE_YAML=($(cat .lagoon.yml | yq -o json | jq -r '."docker-compose-yaml"')) + +DOCKER_COMPOSE_WARNING_COUNT=0 +############################################## +### RUN docker compose config check against the provided docker-compose file +### use the `build-validate` built in validater to run over the provided docker-compose file +############################################## +dccOutput=$(bash -c 'build-deploy-tool validate docker-compose --docker-compose '${DOCKER_COMPOSE_YAML}'; exit $?' 2>&1) +dccExit=$? + echo "Updating docker-compose-yaml configmap with a pre-deploy version of the docker-compose.yml file" if kubectl -n ${NAMESPACE} get configmap docker-compose-yaml &> /dev/null; then # replace it @@ -199,18 +243,6 @@ if kubectl -n ${NAMESPACE} get configmap docker-compose-yaml &> /dev/null; then kubectl -n ${NAMESPACE} create configmap docker-compose-yaml --from-file=pre-deploy=${DOCKER_COMPOSE_YAML} fi -set +e -currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" -patchBuildStep "${buildStartTime}" "${buildStartTime}" "${currentStepEnd}" "${NAMESPACE}" "initialSetup" "Initial Environment Setup" "false" -previousStepEnd=${currentStepEnd} -beginBuildStep "Docker Compose Validation" "dockerComposeValidation" -DOCKER_COMPOSE_WARNING_COUNT=0 -############################################## -### RUN docker compose config check against the provided docker-compose file -### use the `build-validate` built in validater to run over the provided docker-compose file -############################################## -dccOutput=$(bash -c 'build-deploy-tool validate docker-compose --docker-compose '${DOCKER_COMPOSE_YAML}'; exit $?' 2>&1) -dccExit=$? if [ "${dccExit}" != "0" ]; then currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "dockerComposeValidationError" "Docker Compose Validation" "false" @@ -280,32 +312,6 @@ else patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "dockerComposeValidation" "Docker Compose Validation" "false" previousStepEnd=${currentStepEnd} fi - -beginBuildStep ".lagoon.yml Validation" "lagoonYmlValidation" -############################################## -### RUN lagoon-yml validation against the final data which may have overrides -### from .lagoon.override.yml file or LAGOON_YAML_OVERRIDE environment variable -############################################## -lyvOutput=$(bash -c 'build-deploy-tool validate lagoon-yml; exit $?' 2>&1) -lyvExit=$? - -if [ "${lyvExit}" != "0" ]; then - currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" - patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "lagoonYmlValidationError" ".lagoon.yml Validation" "false" - previousStepEnd=${currentStepEnd} - echo " -############################################## -Warning! -There are issues with your .lagoon.yml file that must be fixed. -Refer to the .lagoon.yml docs for the correct syntax -${LAGOON_FEATURE_FLAG_DEFAULT_DOCUMENTATION_URL}/using-lagoon-the-basics/lagoon-yml/ -############################################## -" - echo "${lyvOutput}" - echo " -##############################################" - exit 1 -fi set -e # Validate .lagoon.yml only, no overrides. lagoon-linter still has checks that @@ -328,7 +334,7 @@ fi # # export LAGOON_GIT_SHA=`git rev-parse HEAD` # -INJECT_GIT_SHA=$(cat .lagoon.yml | shyaml get-value environment_variables.git_sha false) +INJECT_GIT_SHA=$(cat .lagoon.yml | yq -o json | jq -r '.environment_variables.git_sha // false') if [ "$INJECT_GIT_SHA" == "true" ] then # export this so the build-deploy-tool can read it @@ -342,10 +348,10 @@ currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "lagoonYmlValidation" ".lagoon.yml Validation" "false" previousStepEnd=${currentStepEnd} beginBuildStep "Configure Variables" "configuringVariables" -DEPLOY_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.deploy-type default) +DEPLOY_TYPE=$(cat .lagoon.yml | yq -o json | jq -r '.environments.'\"${BRANCH//.\//.}\"'."deploy-type" // "default"') # Load all Services that are defined -COMPOSE_SERVICES=($(cat $DOCKER_COMPOSE_YAML | shyaml keys services)) +COMPOSE_SERVICES=($(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services | keys_unsorted | .[]')) ############################################## ### CACHE IMAGE LIST GENERATION @@ -376,6 +382,10 @@ declare -A IMAGES_PUSH declare -A IMAGES_PROMOTE # this array stores the hashes of the built images declare -A IMAGE_HASHES +# this array stores the dbaas consumer specs +declare -A MARIADB_DBAAS_CONSUMER_SPECS +declare -A POSTGRES_DBAAS_CONSUMER_SPECS +declare -A MONGODB_DBAAS_CONSUMER_SPECS HELM_ARGUMENTS=() . /kubectl-build-deploy/scripts/kubectl-get-cluster-capabilities.sh @@ -417,16 +427,16 @@ DBAAS=($(build-deploy-tool identify dbaas)) for COMPOSE_SERVICE in "${COMPOSE_SERVICES[@]}" do # The name of the service can be overridden, if not we use the actual servicename - SERVICE_NAME=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.name default) + SERVICE_NAME=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$COMPOSE_SERVICE\"'.labels."lagoon.name" // "default"') if [ "$SERVICE_NAME" == "default" ]; then SERVICE_NAME=$COMPOSE_SERVICE fi # Load the servicetype. If it's "none" we will not care about this service at all - SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.type custom) + SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$COMPOSE_SERVICE\"'.labels."lagoon.type" // "custom"') # Allow the servicetype to be overriden by environment in .lagoon.yml - ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.types.$SERVICE_NAME false) + ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | yq -o json | jq -r '.environments.'\"${BRANCH//./\\.}\"'.types.'\"$SERVICE_NAME\"' // false') if [ ! $ENVIRONMENT_SERVICE_TYPE_OVERRIDE == "false" ]; then SERVICE_TYPE=$ENVIRONMENT_SERVICE_TYPE_OVERRIDE fi @@ -461,7 +471,7 @@ do # For DeploymentConfigs with multiple Services inside (like nginx-php), we allow to define the service type of within the # deploymentconfig via lagoon.deployment.servicetype. If this is not set we use the Compose Service Name - DEPLOYMENT_SERVICETYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.deployment\\.servicetype default) + DEPLOYMENT_SERVICETYPE=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$COMPOSE_SERVICE\"'.labels."lagoon.deployment.servicetype" // "default"') if [ "$DEPLOYMENT_SERVICETYPE" == "default" ]; then DEPLOYMENT_SERVICETYPE=$COMPOSE_SERVICE fi @@ -713,23 +723,23 @@ YAML_FOLDER="/kubectl-build-deploy/lagoon/services-routes" mkdir -p $YAML_FOLDER # BC for routes.insecure, which is now called routes.autogenerate.insecure -BC_ROUTES_AUTOGENERATE_INSECURE=$(cat .lagoon.yml | shyaml get-value routes.insecure false) +BC_ROUTES_AUTOGENERATE_INSECURE=$(cat .lagoon.yml | yq -o json | jq -r '.routes.insecure // false') if [ ! $BC_ROUTES_AUTOGENERATE_INSECURE == "false" ]; then echo "=== routes.insecure is now defined in routes.autogenerate.insecure, pleae update your .lagoon.yml file" # update the .lagoon.yml with the new location for build-deploy-tool to read - yq3 write -i -- .lagoon.yml 'routes.autogenerate.insecure' $BC_ROUTES_AUTOGENERATE_INSECURE + yq -i '.routes.autogenerate.insecure = "'${BC_ROUTES_AUTOGENERATE_INSECURE}'"' .lagoon.yml fi touch /kubectl-build-deploy/values.yaml -yq3 write -i -- /kubectl-build-deploy/values.yaml 'project' $PROJECT -yq3 write -i -- /kubectl-build-deploy/values.yaml 'environment' $ENVIRONMENT -yq3 write -i -- /kubectl-build-deploy/values.yaml 'environmentType' $ENVIRONMENT_TYPE -yq3 write -i -- /kubectl-build-deploy/values.yaml 'namespace' $NAMESPACE -yq3 write -i -- /kubectl-build-deploy/values.yaml 'gitSha' $LAGOON_GIT_SHA -yq3 write -i -- /kubectl-build-deploy/values.yaml 'buildType' $BUILD_TYPE -yq3 write -i -- /kubectl-build-deploy/values.yaml 'kubernetes' $KUBERNETES -yq3 write -i -- /kubectl-build-deploy/values.yaml 'lagoonVersion' $LAGOON_VERSION +yq -i '.project = "'${PROJECT}'"' /kubectl-build-deploy/values.yaml +yq -i '.environmentType = "'${ENVIRONMENT}'"' /kubectl-build-deploy/values.yaml +yq -i '.environmentType = "'${ENVIRONMENT_TYPE}'"' /kubectl-build-deploy/values.yaml +yq -i '.namespace = "'${NAMESPACE}'"' /kubectl-build-deploy/values.yaml +yq -i '.gitSha = "'${LAGOON_GIT_SHA}'"' /kubectl-build-deploy/values.yaml +yq -i '.buildType = "'${BUILD_TYPE}'"' /kubectl-build-deploy/values.yaml +yq -i '.kubernetes = "'${KUBERNETES}'"' /kubectl-build-deploy/values.yaml +yq -i '.lagoonVersion = "'${LAGOON_VERSION}'"' /kubectl-build-deploy/values.yaml # check for ROOTLESS_WORKLOAD feature flag, disabled by default if [ "${SCC_CHECK}" != "false" ]; then @@ -737,44 +747,21 @@ if [ "${SCC_CHECK}" != "false" ]; then # this applies it to all deployments in this environment because we don't isolate by service type its applied to all OPENSHIFT_SUPPLEMENTAL_GROUP=$(kubectl get namespace ${NAMESPACE} -o json | jq -r '.metadata.annotations."openshift.io/sa.scc.supplemental-groups"' | cut -c -10) echo "Setting openshift fsGroup to ${OPENSHIFT_SUPPLEMENTAL_GROUP}" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'podSecurityContext.fsGroup' $OPENSHIFT_SUPPLEMENTAL_GROUP + yq -i '.podSecurityContext.fsGroup = "'${OPENSHIFT_SUPPLEMENTAL_GROUP}'"' /kubectl-build-deploy/values.yaml fi -echo -e "\ -LAGOON_PROJECT=${PROJECT}\n\ -LAGOON_ENVIRONMENT=${ENVIRONMENT}\n\ -LAGOON_ENVIRONMENT_TYPE=${ENVIRONMENT_TYPE}\n\ -LAGOON_GIT_SHA=${LAGOON_GIT_SHA}\n\ -LAGOON_KUBERNETES=${KUBERNETES}\n\ -" >> /kubectl-build-deploy/values.env - -# DEPRECATED: will be removed with Lagoon 3.0.0 -# LAGOON_GIT_SAFE_BRANCH is pointing to the enviornment name, therefore also is filled if this environment -# is created by a PR or Promote workflow. This technically wrong, therefore will be removed -echo -e "\ -LAGOON_GIT_SAFE_BRANCH=${ENVIRONMENT}\n\ -" >> /kubectl-build-deploy/values.env -if [ "$BUILD_TYPE" == "branch" ]; then - yq3 write -i -- /kubectl-build-deploy/values.yaml 'branch' $BRANCH - echo -e "\ -LAGOON_GIT_BRANCH=${BRANCH}\n\ -" >> /kubectl-build-deploy/values.env + +if [ "$BUILD_TYPE" == "branch" ]; then + yq -i '.branch = "'${BRANCH}'"' /kubectl-build-deploy/values.yaml fi if [ "$BUILD_TYPE" == "pullrequest" ]; then - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prHeadBranch' "$PR_HEAD_BRANCH" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prBaseBranch' "$PR_BASE_BRANCH" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prTitle' "$PR_TITLE" - yq3 write -i -- /kubectl-build-deploy/values.yaml 'prNumber' "$PR_NUMBER" - - echo -e "\ -LAGOON_PR_HEAD_BRANCH=${PR_HEAD_BRANCH}\n\ -LAGOON_PR_BASE_BRANCH=${PR_BASE_BRANCH}\n\ -LAGOON_PR_TITLE=${PR_TITLE}\n\ -LAGOON_PR_NUMBER=${PR_NUMBER}\n\ -" >> /kubectl-build-deploy/values.env + yq -i '.prHeadBranch = "'${PR_HEAD_BRANCH}'"' /kubectl-build-deploy/values.yaml + yq -i '.prBaseBranch = "'${PR_BASE_BRANCH}'"' /kubectl-build-deploy/values.yaml + yq -i '.prTitle = "'${PR_TITLE}'"' /kubectl-build-deploy/values.yaml + yq -i '.prNumber = "'${PR_NUMBER}'"' /kubectl-build-deploy/values.yaml fi currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" @@ -1084,74 +1071,9 @@ fi # Get list of autogenerated routes AUTOGENERATED_ROUTES=$(kubectl -n ${NAMESPACE} get ingress --sort-by='{.metadata.name}' -l "lagoon.sh/autogenerated=true" -o=go-template --template='{{range $indexItems, $ingress := .items}}{{if $indexItems}},{{end}}{{$tls := .spec.tls}}{{range $indexRule, $rule := .spec.rules}}{{if $indexRule}},{{end}}{{if $tls}}https://{{else}}http://{{end}}{{.host}}{{end}}{{end}}') -yq3 write -i -- /kubectl-build-deploy/values.yaml 'route' "$ROUTE" -yq3 write -i -- /kubectl-build-deploy/values.yaml 'routes' "$ROUTES" -yq3 write -i -- /kubectl-build-deploy/values.yaml 'autogeneratedRoutes' "$AUTOGENERATED_ROUTES" - - -# Add in Lagoon core api and ssh-portal details, if available -if [ ! -z "$LAGOON_CONFIG_API_HOST" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_API_HOST="${LAGOON_CONFIG_API_HOST}") - echo -e "LAGOON_CONFIG_API_HOST=${LAGOON_CONFIG_API_HOST}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_TOKEN_HOST" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_TOKEN_HOST="${LAGOON_CONFIG_TOKEN_HOST}") - echo -e "LAGOON_CONFIG_TOKEN_HOST=${LAGOON_CONFIG_TOKEN_HOST}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_TOKEN_PORT" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_TOKEN_PORT="${LAGOON_CONFIG_TOKEN_PORT}") - echo -e "LAGOON_CONFIG_TOKEN_PORT=${LAGOON_CONFIG_TOKEN_PORT}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_SSH_HOST" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_SSH_HOST="${LAGOON_CONFIG_SSH_HOST}") - echo -e "LAGOON_CONFIG_SSH_HOST=${LAGOON_CONFIG_SSH_HOST}\n" >> /kubectl-build-deploy/values.env -fi - -if [ ! -z "$LAGOON_CONFIG_SSH_PORT" ]; then - BUILD_ARGS+=(--build-arg LAGOON_CONFIG_SSH_PORT="${LAGOON_CONFIG_SSH_PORT}") - echo -e "LAGOON_CONFIG_SSH_PORT=${LAGOON_CONFIG_SSH_PORT}\n" >> /kubectl-build-deploy/values.env -fi - -echo -e "\ -LAGOON_ROUTE=${ROUTE}\n\ -LAGOON_ROUTES=${ROUTES}\n\ -LAGOON_AUTOGENERATED_ROUTES=${AUTOGENERATED_ROUTES}\n\ -" >> /kubectl-build-deploy/values.env - -# Generate a Config Map with project wide env variables -kubectl -n ${NAMESPACE} create configmap lagoon-env -o yaml --dry-run=client --from-env-file=/kubectl-build-deploy/values.env | kubectl apply -n ${NAMESPACE} -f - - -# Add environment variables from lagoon API -if [ ! -z "$LAGOON_PROJECT_VARIABLES" ]; then - HAS_PROJECT_RUNTIME_VARS=$(echo $LAGOON_PROJECT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") )') - - if [ ! "$HAS_PROJECT_RUNTIME_VARS" = "[]" ]; then - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":$(echo $LAGOON_PROJECT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") ) | map( { (.name) : .value } ) | add | tostring')}" - fi -fi -if [ ! -z "$LAGOON_ENVIRONMENT_VARIABLES" ]; then - HAS_ENVIRONMENT_RUNTIME_VARS=$(echo $LAGOON_ENVIRONMENT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") )') - - if [ ! "$HAS_ENVIRONMENT_RUNTIME_VARS" = "[]" ]; then - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":$(echo $LAGOON_ENVIRONMENT_VARIABLES | jq -r 'map( select(.scope == "runtime" or .scope == "global") ) | map( { (.name) : .value } ) | add | tostring')}" - fi -fi - -if [ "$BUILD_TYPE" == "pullrequest" ]; then - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"LAGOON_PR_HEAD_BRANCH\":\"${PR_HEAD_BRANCH}\", \"LAGOON_PR_BASE_BRANCH\":\"${PR_BASE_BRANCH}\", \"LAGOON_PR_TITLE\":$(echo $PR_TITLE | jq -R)}}" -fi +yq -i '.route = "'${ROUTE}'"' /kubectl-build-deploy/values.yaml +yq -i '.routes = "'${ROUTES}'"' /kubectl-build-deploy/values.yaml +yq -i '.autogeneratedRoutes = "'${AUTOGENERATED_ROUTES}'"' /kubectl-build-deploy/values.yaml # loop through created DBAAS templates DBAAS=($(build-deploy-tool identify dbaas)) @@ -1171,19 +1093,25 @@ do mariadb-dbaas) # remove the image from images to pull unset IMAGES_PULL[$SERVICE_NAME] - . /kubectl-build-deploy/scripts/exec-kubectl-mariadb-dbaas.sh + CONSUMER_TYPE="mariadbconsumer" + . /kubectl-build-deploy/scripts/exec-kubectl-dbaas-wait.sh + MARIADB_DBAAS_CONSUMER_SPECS["${SERVICE_NAME}"]=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o json | jq -r '.spec | @base64') ;; postgres-dbaas) # remove the image from images to pull unset IMAGES_PULL[$SERVICE_NAME] - . /kubectl-build-deploy/scripts/exec-kubectl-postgres-dbaas.sh + CONSUMER_TYPE="postgresqlconsumer" + . /kubectl-build-deploy/scripts/exec-kubectl-dbaas-wait.sh + POSTGRES_DBAAS_CONSUMER_SPECS["${SERVICE_NAME}"]=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o json | jq -r '.spec | @base64') ;; mongodb-dbaas) # remove the image from images to pull unset IMAGES_PULL[$SERVICE_NAME] - . /kubectl-build-deploy/scripts/exec-kubectl-mongodb-dbaas.sh + CONSUMER_TYPE="mongodbconsumer" + . /kubectl-build-deploy/scripts/exec-kubectl-dbaas-wait.sh + MONGODB_DBAAS_CONSUMER_SPECS["${SERVICE_NAME}"]=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o json | jq -r '.spec | @base64') ;; *) @@ -1192,6 +1120,70 @@ do esac done +# convert specs into credential dump for ingestion by build-deploy-tool +DBAAS_VARIABLES="[]" +for SERVICE_NAME in "${!MARIADB_DBAAS_CONSUMER_SPECS[@]}" +do + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + DB_HOST=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.primary') + DB_USER=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.username') + DB_PASSWORD=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.password') + DB_NAME=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.database') + DB_PORT=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.port') + DB_CONSUMER='{"'${SERVICE_NAME_UPPERCASE}'_HOST":"'${DB_HOST}'", "'${SERVICE_NAME_UPPERCASE}'_USERNAME":"'${DB_USER}'","'${SERVICE_NAME_UPPERCASE}'_PASSWORD":"'${DB_PASSWORD}'","'${SERVICE_NAME_UPPERCASE}'_DATABASE":"'${DB_NAME}'","'${SERVICE_NAME_UPPERCASE}'_PORT":"'${DB_PORT}'"}' + if DB_READREPLICA_HOSTS=$(echo ${MARIADB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.replicas | .[]' 2>/dev/null); then + if [ "$DB_READREPLICA_HOSTS" != "null" ]; then + DB_READREPLICA_HOSTS=$(echo "$DB_READREPLICA_HOSTS" | sed 's/^\|$//g' | paste -sd, -) + DB_CONSUMER=$(echo "${DB_CONSUMER}" | jq '. + {"'${SERVICE_NAME_UPPERCASE}'_READREPLICA_HOSTS":"'${DB_READREPLICA_HOSTS}'"}') + fi + fi + DBAAS_VARIABLES=$(echo "$DBAAS_VARIABLES" | jq '. + '$(echo "$DB_CONSUMER" | jq -sMrc)'') +done + +for SERVICE_NAME in "${!POSTGRES_DBAAS_CONSUMER_SPECS[@]}" +do + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + DB_HOST=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.primary') + DB_USER=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.username') + DB_PASSWORD=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.password') + DB_NAME=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.database') + DB_PORT=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.port') + DB_CONSUMER='{"'${SERVICE_NAME_UPPERCASE}'_HOST":"'${DB_HOST}'", "'${SERVICE_NAME_UPPERCASE}'_USERNAME":"'${DB_USER}'","'${SERVICE_NAME_UPPERCASE}'_PASSWORD":"'${DB_PASSWORD}'","'${SERVICE_NAME_UPPERCASE}'_DATABASE":"'${DB_NAME}'","'${SERVICE_NAME_UPPERCASE}'_PORT":"'${DB_PORT}'"}' + if DB_READREPLICA_HOSTS=$(echo ${POSTGRES_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.replicas | .[]' 2>/dev/null); then + if [ "$DB_READREPLICA_HOSTS" != "null" ]; then + DB_READREPLICA_HOSTS=$(echo "$DB_READREPLICA_HOSTS" | sed 's/^\|$//g' | paste -sd, -) + DB_CONSUMER=$(echo "${DB_CONSUMER}" | jq '. + {"'${SERVICE_NAME_UPPERCASE}'_READREPLICA_HOSTS":"'${DB_READREPLICA_HOSTS}'"}') + fi + fi + DBAAS_VARIABLES=$(echo "$DBAAS_VARIABLES" | jq '. + '$(echo "$DB_CONSUMER" | jq -sMrc)'') +done + +for SERVICE_NAME in "${!MONGODB_DBAAS_CONSUMER_SPECS[@]}" +do + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + DB_HOST=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.services.primary') + DB_USER=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.username') + DB_PASSWORD=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.password') + DB_NAME=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .consumer.database') + DB_PORT=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.port') + DB_AUTHSOURCE=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.auth.source') + DB_AUTHMECHANISM=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.auth.mechanism') + DB_AUTHTLS=$(echo ${MONGODB_DBAAS_CONSUMER_SPECS["$SERVICE_NAME"]} | jq -Rr '@base64d | fromjson | .provider.auth.tls') + DB_CONSUMER='{"'${SERVICE_NAME_UPPERCASE}'_HOST":"'${DB_HOST}'", "'${SERVICE_NAME_UPPERCASE}'_USERNAME":"'${DB_USER}'", "'${SERVICE_NAME_UPPERCASE}'_PASSWORD":"'${DB_PASSWORD}'", "'${SERVICE_NAME_UPPERCASE}'_DATABASE":"'${DB_NAME}'", "'${SERVICE_NAME_UPPERCASE}'_PORT":"'${DB_PORT}'", "'${SERVICE_NAME_UPPERCASE}'_AUTHSOURCE":"'${DB_AUTHSOURCE}'", "'${SERVICE_NAME_UPPERCASE}'_AUTHMECHANISM":"'${DB_AUTHMECHANISM}'", "'${SERVICE_NAME_UPPERCASE}'_AUTHTLS":"'${DB_AUTHTLS}'"}' + DBAAS_VARIABLES=$(echo "$DBAAS_VARIABLES" | jq '. + '$(echo "$DB_CONSUMER" | jq -sMrc)'') +done +echo "$DBAAS_VARIABLES" | jq -Mr > /kubectl-build-deploy/dbaas-creds.json + +# Generate the lagoon-env configmap +LAGOON_ENV_YAML_FOLDER="/kubectl-build-deploy/lagoon/lagoon-env" +mkdir -p $LAGOON_ENV_YAML_FOLDER +# for now, pass the `--routes` flag to the template command so that the routes from the cluster are used in the `lagoon-env` configmap LAGOON_ROUTES as this is how it used to be +# since this tool currently has no kube scrape, and the ones the tool generates are only the ones it knows about currently +# we have to source them this way for now. In the future though, we'll be able to omit this flag and remove it from the tool +# also would be part of https://github.com/uselagoon/build-deploy-tool/blob/f527a89ad5efb46e19a2f59d9ff3ffbff541e2a2/legacy/build-deploy-docker-compose.sh#L1090 +build-deploy-tool template lagoon-env --saved-templates-path ${LAGOON_ENV_YAML_FOLDER} --dbaas-creds /kubectl-build-deploy/dbaas-creds.json --routes "${ROUTES}" +kubectl apply -n ${NAMESPACE} -f ${LAGOON_ENV_YAML_FOLDER}/lagoon-env-configmap.yaml + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "updateConfigmapComplete" "Update Configmap" "false" previousStepEnd=${currentStepEnd} @@ -1201,11 +1193,11 @@ beginBuildStep "Image Push to Registry" "pushingImages" ### REDEPLOY DEPLOYMENTS IF CONFIG MAP CHANGES ############################################## -CONFIG_MAP_SHA=$(kubectl -n ${NAMESPACE} get configmap lagoon-env -o yaml | shyaml get-value data | sha256sum | awk '{print $1}') +CONFIG_MAP_SHA=$(kubectl -n ${NAMESPACE} get configmap lagoon-env -o yaml | yq -M '.data' | sha256sum | awk '{print $1}') export CONFIG_MAP_SHA # write the configmap to the values file so when we `exec-kubectl-resources-with-images.sh` the deployments will get the value of the config map # which will cause a change in the deployment and trigger a rollout if only the configmap has changed -yq3 write -i -- /kubectl-build-deploy/values.yaml 'configMapSha' $CONFIG_MAP_SHA +yq -i '.configMapSha = "'${CONFIG_MAP_SHA}'"' /kubectl-build-deploy/values.yaml ############################################## ### PUSH IMAGES TO REGISTRY @@ -1468,10 +1460,10 @@ do SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[0]} SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[1]} - SERVICE_ROLLOUT_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.${SERVICE_NAME}.labels.lagoon\\.rollout deployment) + SERVICE_ROLLOUT_TYPE=$(cat $DOCKER_COMPOSE_YAML | yq -o json | jq -r '.services.'\"$SERVICE_NAME\"'.labels."lagoon.rollout" // "deployment"') # Allow the rollout type to be overriden by environment in .lagoon.yml - ENVIRONMENT_SERVICE_ROLLOUT_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.rollouts.${SERVICE_NAME} false) + ENVIRONMENT_SERVICE_ROLLOUT_TYPE=$(cat .lagoon.yml | yq -o json | jq -r '.environments.'\"${BRANCH//./\\.}\"'.rollouts.'\"$SERVICE_NAME\"' // false') if [ ! $ENVIRONMENT_SERVICE_ROLLOUT_TYPE == "false" ]; then SERVICE_ROLLOUT_TYPE=$ENVIRONMENT_SERVICE_ROLLOUT_TYPE fi diff --git a/legacy/scripts/exec-kubectl-dbaas-wait.sh b/legacy/scripts/exec-kubectl-dbaas-wait.sh new file mode 100644 index 00000000..c3b64984 --- /dev/null +++ b/legacy/scripts/exec-kubectl-dbaas-wait.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# The operator can sometimes take a bit, wait until the details are available +# We added a timeout of 5 minutes (60 retries) before exit +OPERATOR_COUNTER=1 +OPERATOR_TIMEOUT=60 +# use the secret name from the consumer to prevent credential clash +until [ "$(kubectl -n ${NAMESPACE} get ${CONSUMER_TYPE}/${SERVICE_NAME} -o json | jq -r '.spec.consumer.database')" != "null" ]; +do +if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then + consumer_failed=$(kubectl -n ${NAMESPACE} get ${CONSUMER_TYPE}/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') + if [ "${consumer_failed}" == "true" ]; then + echo "Failed to provision a database. Contact your support team to investigate." + exit 1 + fi + let OPERATOR_COUNTER=OPERATOR_COUNTER+1 + echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" + sleep 5 +else + echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" + exit 1 +fi +done + diff --git a/legacy/scripts/exec-kubectl-mariadb-dbaas.sh b/legacy/scripts/exec-kubectl-mariadb-dbaas.sh deleted file mode 100644 index 87a8525f..00000000 --- a/legacy/scripts/exec-kubectl-mariadb-dbaas.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# The operator can sometimes take a bit, wait until the details are available -# We added a timeout of 5 minutes (60 retries) before exit -OPERATOR_COUNTER=1 -OPERATOR_TIMEOUT=60 -# use the secret name from the consumer to prevent credential clash -until kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database -do -if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then - consumer_failed=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') - if [ "${consumer_failed}" == "true" ]; then - echo "Failed to provision a database. Contact your support team to investigate." - exit 1 - fi - let OPERATOR_COUNTER=OPERATOR_COUNTER+1 - echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" - sleep 5 -else - echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" - exit 1 -fi -done - -# Grab the details from the consumer spec -DB_HOST=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary) -DB_USER=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username) -DB_PASSWORD=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.password) -DB_NAME=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database) -DB_PORT=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.port) - -# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker -kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}" - -# only add the DB_READREPLICA_HOSTS variable if it exists in the consumer spec -# since the operator can support multiple replica hosts being defined, we should comma seperate them here -if DB_READREPLICA_HOSTS=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.replicas); then - DB_READREPLICA_HOSTS=$(echo $DB_READREPLICA_HOSTS | cut -c 3- | rev | cut -c 1- | rev | sed 's/^\|$//g' | paste -sd, -) - yq3 write -i -- /kubectl-build-deploy/${SERVICE_NAME}-values.yaml 'readReplicaHosts' $DB_READREPLICA_HOSTS - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_READREPLICA_HOSTS\":\"${DB_READREPLICA_HOSTS}\"}}" -fi diff --git a/legacy/scripts/exec-kubectl-mongodb-dbaas.sh b/legacy/scripts/exec-kubectl-mongodb-dbaas.sh deleted file mode 100644 index fc4211bf..00000000 --- a/legacy/scripts/exec-kubectl-mongodb-dbaas.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# The operator can sometimes take a bit, wait until the details are available -# We added a timeout of 5 minutes (60 retries) before exit -OPERATOR_COUNTER=1 -OPERATOR_TIMEOUT=60 -# use the secret name from the consumer to prevent credential clash -until kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database -do -if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then - consumer_failed=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') - if [ "${consumer_failed}" == "true" ]; then - echo "Failed to provision a database. Contact your support team to investigate." - exit 1 - fi - let OPERATOR_COUNTER=OPERATOR_COUNTER+1 - echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" - sleep 5 -else - echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" - exit 1 -fi -done - -# Grab the details from the consumer spec -DB_HOST=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary) -DB_USER=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username) -DB_PASSWORD=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.password) -DB_NAME=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database) -DB_PORT=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.port) -DB_AUTHSOURCE=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.auth.source) -DB_AUTHMECHANISM=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.auth.mechanism) -DB_AUTHTLS=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.auth.tls) - -# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker -kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\", \"${SERVICE_NAME_UPPERCASE}_AUTHSOURCE\":\"${DB_AUTHSOURCE}\", \"${SERVICE_NAME_UPPERCASE}_AUTHMECHANISM\":\"${DB_AUTHMECHANISM}\", \"${SERVICE_NAME_UPPERCASE}_AUTHTLS\":\"${DB_AUTHTLS}\" }}" diff --git a/legacy/scripts/exec-kubectl-postgres-dbaas.sh b/legacy/scripts/exec-kubectl-postgres-dbaas.sh deleted file mode 100644 index 54b4dd29..00000000 --- a/legacy/scripts/exec-kubectl-postgres-dbaas.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# The operator can sometimes take a bit, wait until the details are available -# We added a timeout of 5 minutes (60 retries) before exit -OPERATOR_COUNTER=1 -OPERATOR_TIMEOUT=60 -# use the secret name from the consumer to prevent credential clash -until kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database -do -if [ $OPERATOR_COUNTER -lt $OPERATOR_TIMEOUT ]; then - consumer_failed=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o json | jq -r '.metadata.annotations."dbaas.amazee.io/failed"') - if [ "${consumer_failed}" == "true" ]; then - echo "Failed to provision a database. Contact your support team to investigate." - exit 1 - fi - let OPERATOR_COUNTER=OPERATOR_COUNTER+1 - echo "Service for ${SERVICE_NAME} not available yet, waiting for 5 secs" - sleep 5 -else - echo "Timeout of $OPERATOR_TIMEOUT for ${SERVICE_NAME} creation reached" - exit 1 -fi -done - -# Grab the details from the consumer spec -DB_HOST=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary) -DB_USER=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username) -DB_PASSWORD=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.password) -DB_NAME=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.database) -DB_PORT=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.provider.port) - -# Add credentials to our configmap, prefixed with the name of the servicename of this servicebroker -kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\"}}" - -# only add the DB_READREPLICA_HOSTS variable if it exists in the consumer spec -# since the operator can support multiple replica hosts being defined, we should comma seperate them here -if DB_READREPLICA_HOSTS=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.replicas); then - DB_READREPLICA_HOSTS=$(echo $DB_READREPLICA_HOSTS | cut -c 3- | rev | cut -c 1- | rev | sed 's/^\|$//g' | paste -sd, -) - yq3 write -i -- /kubectl-build-deploy/${SERVICE_NAME}-values.yaml 'readReplicaHosts' $DB_READREPLICA_HOSTS - kubectl patch \ - -n ${NAMESPACE} \ - configmap lagoon-env \ - -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_READREPLICA_HOSTS\":\"${DB_READREPLICA_HOSTS}\"}}" -fi