From b655c496a921486c830398040145c64ef124020f Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Wed, 25 Sep 2024 09:16:40 +1000 Subject: [PATCH] refactor: dont update dbaas crds after they are provisioned --- cmd/template_dbaas.go | 6 +- internal/templating/template_dbaas.go | 10 +-- internal/templating/template_dbaas_test.go | 9 +- .../dbaas-1/{dbaas.yaml => mariadb.yaml} | 0 .../dbaas-2/{dbaas.yaml => mariadb.yaml} | 0 .../dbaas-templates/dbaas-3/mariadb.yaml | 26 ++++++ .../dbaas-3/{dbaas.yaml => mariadb2.yaml} | 26 ------ .../{dbaas.yaml => mariadb-10-11.yaml} | 0 .../node/dbaas-templates/dbaas-1/dbaas.yaml | 90 ------------------- .../node/dbaas-templates/dbaas-1/mongo.yaml | 30 +++++++ .../dbaas.yaml => dbaas-1/mongo2.yaml} | 30 ------- .../node/dbaas-templates/dbaas-1/mongo3.yaml | 30 +++++++ .../node/dbaas-templates/dbaas-2/mongo2.yaml | 30 +++++++ .../node/dbaas-templates/dbaas-2/mongo3.yaml | 30 +++++++ legacy/build-deploy-docker-compose.sh | 47 +++++++++- 15 files changed, 205 insertions(+), 159 deletions(-) rename internal/testdata/complex/dbaas-templates/dbaas-1/{dbaas.yaml => mariadb.yaml} (100%) rename internal/testdata/complex/dbaas-templates/dbaas-2/{dbaas.yaml => mariadb.yaml} (100%) create mode 100644 internal/testdata/complex/dbaas-templates/dbaas-3/mariadb.yaml rename internal/testdata/complex/dbaas-templates/dbaas-3/{dbaas.yaml => mariadb2.yaml} (50%) rename internal/testdata/complex/dbaas-templates/dbaas-4/{dbaas.yaml => mariadb-10-11.yaml} (100%) delete mode 100644 internal/testdata/node/dbaas-templates/dbaas-1/dbaas.yaml create mode 100644 internal/testdata/node/dbaas-templates/dbaas-1/mongo.yaml rename internal/testdata/node/dbaas-templates/{dbaas-2/dbaas.yaml => dbaas-1/mongo2.yaml} (50%) create mode 100644 internal/testdata/node/dbaas-templates/dbaas-1/mongo3.yaml create mode 100644 internal/testdata/node/dbaas-templates/dbaas-2/mongo2.yaml create mode 100644 internal/testdata/node/dbaas-templates/dbaas-2/mongo3.yaml diff --git a/cmd/template_dbaas.go b/cmd/template_dbaas.go index 9b34a2da..aadf286b 100644 --- a/cmd/template_dbaas.go +++ b/cmd/template_dbaas.go @@ -41,10 +41,10 @@ func DBaaSTemplateGeneration(g generator.GeneratorInput, if err != nil { return fmt.Errorf("couldn't generate template: %v", err) } - if len(templateYAML) > 0 { - helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, "dbaas"), templateYAML) + for db, tpl := range templateYAML { + helpers.WriteTemplateFile(fmt.Sprintf("%s/%s.yaml", savedTemplates, db), tpl) if g.Debug { - fmt.Printf("Templating dbaas consumers to %s\n", fmt.Sprintf("%s/%s.yaml", savedTemplates, "dbaas")) + fmt.Printf("Templating dbaas consumers to %s\n", fmt.Sprintf("%s/%s.yaml", savedTemplates, db)) } } return nil diff --git a/internal/templating/template_dbaas.go b/internal/templating/template_dbaas.go index 208ba8d7..34e97fc6 100644 --- a/internal/templating/template_dbaas.go +++ b/internal/templating/template_dbaas.go @@ -218,16 +218,16 @@ func GenerateDBaaSTemplate( return &dbaasTemplates, nil } -func TemplateConsumers(dbaas *DBaaSTemplates) ([]byte, error) { +func TemplateConsumers(dbaas *DBaaSTemplates) (map[string][]byte, error) { separator := []byte("---\n") - var templateYAML []byte + templateYAML := map[string][]byte{} for _, db := range dbaas.MariaDB { dbBytes, err := yaml.Marshal(db) if err != nil { return nil, fmt.Errorf("couldn't generate template: %v", err) } restoreResult := append(separator[:], dbBytes[:]...) - templateYAML = append(templateYAML, restoreResult[:]...) + templateYAML[db.Name] = append(templateYAML[db.Name], restoreResult[:]...) } for _, db := range dbaas.MongoDB { dbBytes, err := yaml.Marshal(db) @@ -235,7 +235,7 @@ func TemplateConsumers(dbaas *DBaaSTemplates) ([]byte, error) { return nil, fmt.Errorf("couldn't generate template: %v", err) } restoreResult := append(separator[:], dbBytes[:]...) - templateYAML = append(templateYAML, restoreResult[:]...) + templateYAML[db.Name] = append(templateYAML[db.Name], restoreResult[:]...) } for _, db := range dbaas.PostgreSQL { dbBytes, err := yaml.Marshal(db) @@ -243,7 +243,7 @@ func TemplateConsumers(dbaas *DBaaSTemplates) ([]byte, error) { return nil, fmt.Errorf("couldn't generate template: %v", err) } restoreResult := append(separator[:], dbBytes[:]...) - templateYAML = append(templateYAML, restoreResult[:]...) + templateYAML[db.Name] = append(templateYAML[db.Name], restoreResult[:]...) } return templateYAML, nil } diff --git a/internal/templating/template_dbaas_test.go b/internal/templating/template_dbaas_test.go index 3bc31332..cd4a7511 100644 --- a/internal/templating/template_dbaas_test.go +++ b/internal/templating/template_dbaas_test.go @@ -161,6 +161,7 @@ func TestGenerateDBaaSTemplate(t *testing.T) { t.Errorf("GenerateDBaaSTemplate() error = %v, wantErr %v", err, tt.wantErr) return } + r1, err := os.ReadFile(tt.want) if err != nil { t.Errorf("couldn't read file %v: %v", tt.want, err) @@ -169,8 +170,12 @@ func TestGenerateDBaaSTemplate(t *testing.T) { if err != nil { t.Errorf("couldn't generate template: %v", err) } - if !reflect.DeepEqual(string(templateYAML), string(r1)) { - t.Errorf("GenerateDBaaSTemplate() = \n%v", diff.LineDiff(string(r1), string(templateYAML))) + var tplYAML []byte + for _, tpl := range templateYAML { + tplYAML = append(tplYAML, tpl...) + } + if !reflect.DeepEqual(string(tplYAML), string(r1)) { + t.Errorf("GenerateDBaaSTemplate() = \n%v", diff.LineDiff(string(r1), string(tplYAML))) } }) } diff --git a/internal/testdata/complex/dbaas-templates/dbaas-1/dbaas.yaml b/internal/testdata/complex/dbaas-templates/dbaas-1/mariadb.yaml similarity index 100% rename from internal/testdata/complex/dbaas-templates/dbaas-1/dbaas.yaml rename to internal/testdata/complex/dbaas-templates/dbaas-1/mariadb.yaml diff --git a/internal/testdata/complex/dbaas-templates/dbaas-2/dbaas.yaml b/internal/testdata/complex/dbaas-templates/dbaas-2/mariadb.yaml similarity index 100% rename from internal/testdata/complex/dbaas-templates/dbaas-2/dbaas.yaml rename to internal/testdata/complex/dbaas-templates/dbaas-2/mariadb.yaml diff --git a/internal/testdata/complex/dbaas-templates/dbaas-3/mariadb.yaml b/internal/testdata/complex/dbaas-templates/dbaas-3/mariadb.yaml new file mode 100644 index 00000000..7c206c80 --- /dev/null +++ b/internal/testdata/complex/dbaas-templates/dbaas-3/mariadb.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mariadb.amazee.io/v1 +kind: MariaDBConsumer +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: mariadb + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: mariadb-dbaas + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: mariadb + lagoon.sh/service-type: mariadb-dbaas + lagoon.sh/template: mariadb-dbaas-0.1.0 + name: mariadb +spec: + consumer: + services: {} + environment: production + provider: {} +status: {} diff --git a/internal/testdata/complex/dbaas-templates/dbaas-3/dbaas.yaml b/internal/testdata/complex/dbaas-templates/dbaas-3/mariadb2.yaml similarity index 50% rename from internal/testdata/complex/dbaas-templates/dbaas-3/dbaas.yaml rename to internal/testdata/complex/dbaas-templates/dbaas-3/mariadb2.yaml index b16ae4d4..2b0e0349 100644 --- a/internal/testdata/complex/dbaas-templates/dbaas-3/dbaas.yaml +++ b/internal/testdata/complex/dbaas-templates/dbaas-3/mariadb2.yaml @@ -1,32 +1,6 @@ --- apiVersion: mariadb.amazee.io/v1 kind: MariaDBConsumer -metadata: - annotations: - lagoon.sh/branch: main - lagoon.sh/version: v2.7.x - creationTimestamp: null - labels: - app.kubernetes.io/instance: mariadb - app.kubernetes.io/managed-by: build-deploy-tool - app.kubernetes.io/name: mariadb-dbaas - lagoon.sh/buildType: branch - lagoon.sh/environment: main - lagoon.sh/environmentType: production - lagoon.sh/project: example-project - lagoon.sh/service: mariadb - lagoon.sh/service-type: mariadb-dbaas - lagoon.sh/template: mariadb-dbaas-0.1.0 - name: mariadb -spec: - consumer: - services: {} - environment: production - provider: {} -status: {} ---- -apiVersion: mariadb.amazee.io/v1 -kind: MariaDBConsumer metadata: annotations: lagoon.sh/branch: main diff --git a/internal/testdata/complex/dbaas-templates/dbaas-4/dbaas.yaml b/internal/testdata/complex/dbaas-templates/dbaas-4/mariadb-10-11.yaml similarity index 100% rename from internal/testdata/complex/dbaas-templates/dbaas-4/dbaas.yaml rename to internal/testdata/complex/dbaas-templates/dbaas-4/mariadb-10-11.yaml diff --git a/internal/testdata/node/dbaas-templates/dbaas-1/dbaas.yaml b/internal/testdata/node/dbaas-templates/dbaas-1/dbaas.yaml deleted file mode 100644 index 8835d7dd..00000000 --- a/internal/testdata/node/dbaas-templates/dbaas-1/dbaas.yaml +++ /dev/null @@ -1,90 +0,0 @@ ---- -apiVersion: mongodb.amazee.io/v1 -kind: MongoDBConsumer -metadata: - annotations: - lagoon.sh/branch: main - lagoon.sh/version: v2.7.x - creationTimestamp: null - labels: - app.kubernetes.io/instance: mongo - app.kubernetes.io/managed-by: build-deploy-tool - app.kubernetes.io/name: mongodb-dbaas - lagoon.sh/buildType: branch - lagoon.sh/environment: main - lagoon.sh/environmentType: production - lagoon.sh/project: example-project - lagoon.sh/service: mongo - lagoon.sh/service-type: mongodb-dbaas - lagoon.sh/template: mongodb-dbaas-0.1.0 - name: mongo -spec: - consumer: - auth: - tls: false - services: {} - environment: production - provider: - auth: - tls: false -status: {} ---- -apiVersion: mongodb.amazee.io/v1 -kind: MongoDBConsumer -metadata: - annotations: - lagoon.sh/branch: main - lagoon.sh/version: v2.7.x - creationTimestamp: null - labels: - app.kubernetes.io/instance: mongo2 - app.kubernetes.io/managed-by: build-deploy-tool - app.kubernetes.io/name: mongodb-dbaas - lagoon.sh/buildType: branch - lagoon.sh/environment: main - lagoon.sh/environmentType: production - lagoon.sh/project: example-project - lagoon.sh/service: mongo2 - lagoon.sh/service-type: mongodb-dbaas - lagoon.sh/template: mongodb-dbaas-0.1.0 - name: mongo2 -spec: - consumer: - auth: - tls: false - services: {} - environment: production - provider: - auth: - tls: false -status: {} ---- -apiVersion: mongodb.amazee.io/v1 -kind: MongoDBConsumer -metadata: - annotations: - lagoon.sh/branch: main - lagoon.sh/version: v2.7.x - creationTimestamp: null - labels: - app.kubernetes.io/instance: mongo3 - app.kubernetes.io/managed-by: build-deploy-tool - app.kubernetes.io/name: mongodb-dbaas - lagoon.sh/buildType: branch - lagoon.sh/environment: main - lagoon.sh/environmentType: production - lagoon.sh/project: example-project - lagoon.sh/service: mongo3 - lagoon.sh/service-type: mongodb-dbaas - lagoon.sh/template: mongodb-dbaas-0.1.0 - name: mongo3 -spec: - consumer: - auth: - tls: false - services: {} - environment: production - provider: - auth: - tls: false -status: {} diff --git a/internal/testdata/node/dbaas-templates/dbaas-1/mongo.yaml b/internal/testdata/node/dbaas-templates/dbaas-1/mongo.yaml new file mode 100644 index 00000000..a1c6b7cb --- /dev/null +++ b/internal/testdata/node/dbaas-templates/dbaas-1/mongo.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.amazee.io/v1 +kind: MongoDBConsumer +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: mongo + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: mongodb-dbaas + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: mongo + lagoon.sh/service-type: mongodb-dbaas + lagoon.sh/template: mongodb-dbaas-0.1.0 + name: mongo +spec: + consumer: + auth: + tls: false + services: {} + environment: production + provider: + auth: + tls: false +status: {} diff --git a/internal/testdata/node/dbaas-templates/dbaas-2/dbaas.yaml b/internal/testdata/node/dbaas-templates/dbaas-1/mongo2.yaml similarity index 50% rename from internal/testdata/node/dbaas-templates/dbaas-2/dbaas.yaml rename to internal/testdata/node/dbaas-templates/dbaas-1/mongo2.yaml index c42f53ca..e0fe9b9e 100644 --- a/internal/testdata/node/dbaas-templates/dbaas-2/dbaas.yaml +++ b/internal/testdata/node/dbaas-templates/dbaas-1/mongo2.yaml @@ -28,33 +28,3 @@ spec: auth: tls: false status: {} ---- -apiVersion: mongodb.amazee.io/v1 -kind: MongoDBConsumer -metadata: - annotations: - lagoon.sh/branch: main - lagoon.sh/version: v2.7.x - creationTimestamp: null - labels: - app.kubernetes.io/instance: mongo3 - app.kubernetes.io/managed-by: build-deploy-tool - app.kubernetes.io/name: mongodb-dbaas - lagoon.sh/buildType: branch - lagoon.sh/environment: main - lagoon.sh/environmentType: production - lagoon.sh/project: example-project - lagoon.sh/service: mongo3 - lagoon.sh/service-type: mongodb-dbaas - lagoon.sh/template: mongodb-dbaas-0.1.0 - name: mongo3 -spec: - consumer: - auth: - tls: false - services: {} - environment: production - provider: - auth: - tls: false -status: {} diff --git a/internal/testdata/node/dbaas-templates/dbaas-1/mongo3.yaml b/internal/testdata/node/dbaas-templates/dbaas-1/mongo3.yaml new file mode 100644 index 00000000..fb817f08 --- /dev/null +++ b/internal/testdata/node/dbaas-templates/dbaas-1/mongo3.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.amazee.io/v1 +kind: MongoDBConsumer +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: mongo3 + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: mongodb-dbaas + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: mongo3 + lagoon.sh/service-type: mongodb-dbaas + lagoon.sh/template: mongodb-dbaas-0.1.0 + name: mongo3 +spec: + consumer: + auth: + tls: false + services: {} + environment: production + provider: + auth: + tls: false +status: {} diff --git a/internal/testdata/node/dbaas-templates/dbaas-2/mongo2.yaml b/internal/testdata/node/dbaas-templates/dbaas-2/mongo2.yaml new file mode 100644 index 00000000..e0fe9b9e --- /dev/null +++ b/internal/testdata/node/dbaas-templates/dbaas-2/mongo2.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.amazee.io/v1 +kind: MongoDBConsumer +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: mongo2 + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: mongodb-dbaas + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: mongo2 + lagoon.sh/service-type: mongodb-dbaas + lagoon.sh/template: mongodb-dbaas-0.1.0 + name: mongo2 +spec: + consumer: + auth: + tls: false + services: {} + environment: production + provider: + auth: + tls: false +status: {} diff --git a/internal/testdata/node/dbaas-templates/dbaas-2/mongo3.yaml b/internal/testdata/node/dbaas-templates/dbaas-2/mongo3.yaml new file mode 100644 index 00000000..fb817f08 --- /dev/null +++ b/internal/testdata/node/dbaas-templates/dbaas-2/mongo3.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.amazee.io/v1 +kind: MongoDBConsumer +metadata: + annotations: + lagoon.sh/branch: main + lagoon.sh/version: v2.7.x + creationTimestamp: null + labels: + app.kubernetes.io/instance: mongo3 + app.kubernetes.io/managed-by: build-deploy-tool + app.kubernetes.io/name: mongodb-dbaas + lagoon.sh/buildType: branch + lagoon.sh/environment: main + lagoon.sh/environmentType: production + lagoon.sh/project: example-project + lagoon.sh/service: mongo3 + lagoon.sh/service-type: mongodb-dbaas + lagoon.sh/template: mongodb-dbaas-0.1.0 + name: mongo3 +spec: + consumer: + auth: + tls: false + services: {} + environment: production + provider: + auth: + tls: false +status: {} diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 9a3f510b..bdc8fac8 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -896,10 +896,51 @@ LAGOON_DBAAS_YAML_FOLDER="/kubectl-build-deploy/lagoon/dbaas" mkdir -p $LAGOON_DBAAS_YAML_FOLDER build-deploy-tool template dbaas --saved-templates-path ${LAGOON_DBAAS_YAML_FOLDER} -# apply dbaas +# apply dbaas only if it doesn't already exist, this prevents environment spec changes from occuring on the dbaas which can make deprovisioning difficult +# changing the spec is the only modification that can be performed, but the current dbaas operator does nothing with this change +# this causes issues with deprovisioning if the spec is changed after it is created because the current dbaas operator doesn't know what to do +# additionally, migrations of these types of resources are mostly manual if they are done, and the manual operations +# change these resources. this blocking operation will also prevent issues after migrations take place from the consumer spec being modified +# once the new dbaas controller is ready with the new CRD types, this functionality won't be required anymore as the entire provisioning mechanism is different if [ -n "$(ls -A $LAGOON_DBAAS_YAML_FOLDER/ 2>/dev/null)" ]; then - find $LAGOON_DBAAS_YAML_FOLDER -type f -exec cat {} \; - kubectl apply -n ${NAMESPACE} -f $LAGOON_DBAAS_YAML_FOLDER/ + DBAAS=($(build-deploy-tool identify dbaas)) + for DBAAS_ENTRY in "${DBAAS[@]}" + do + IFS=':' read -ra DBAAS_ENTRY_SPLIT <<< "$DBAAS_ENTRY" + + SERVICE_NAME=${DBAAS_ENTRY_SPLIT[0]} + SERVICE_TYPE=${DBAAS_ENTRY_SPLIT[1]} + SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + if [[ "$SERVICE_TYPE}" =~ "-single" ]]; then + # skip to next if this type is a single + continue + fi + CONSUMER_NAME=mariadbconsumer + case "$SERVICE_TYPE" in + mariadb-dbaas) + CONSUMER_NAME=mariadbconsumer + ;; + + postgres-dbaas) + CONSUMER_NAME=postgresqlconsumer + ;; + + mongodb-dbaas) + CONSUMER_NAME=mongodbconsumer + ;; + esac + cat $LAGOON_DBAAS_YAML_FOLDER/${SERVICE_NAME}.yaml + # check if the dbaas consumer has already been created + if ! kubectl -n ${NAMESPACE} get ${CONSUMER_NAME} ${SERVICE_NAME} &> /dev/null; then + # only apply this dbaas if it doesn't exist already + kubectl apply -n ${NAMESPACE} -f $LAGOON_DBAAS_YAML_FOLDER/${SERVICE_NAME}.yaml + else + # drop a note that it is already created + echo "> DBaaS ${CONSUMER_NAME} ${SERVICE_NAME} already created" + # maybe update labels still by clearing out the spec? + cat $LAGOON_DBAAS_YAML_FOLDER/${SERVICE_NAME}.yaml | yq eval 'del(.spec)' | kubectl apply -n ${NAMESPACE} -f - + fi + done fi currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"