From 29c77dedc7851b7ec98780204d638d312f104719 Mon Sep 17 00:00:00 2001 From: George Barbarosie Date: Sun, 16 Jun 2024 21:47:19 +0100 Subject: [PATCH] proposed changes for next chart version: - remove db provider split code, transition from zalando - update cnpg db image to 16.1-cron - make s3 serverName default to -db-pg16 suffix --- charts/etrip/templates/cron.yml | 36 --- charts/etrip/templates/database-zalando.yml | 231 ------------------ .../{database-cnpg.yml => database.yml} | 19 -- charts/etrip/templates/deploy.yml | 60 +---- charts/etrip/values.yaml | 79 +----- 5 files changed, 4 insertions(+), 421 deletions(-) delete mode 100644 charts/etrip/templates/database-zalando.yml rename charts/etrip/templates/{database-cnpg.yml => database.yml} (86%) diff --git a/charts/etrip/templates/cron.yml b/charts/etrip/templates/cron.yml index 650c725..7efa13c 100644 --- a/charts/etrip/templates/cron.yml +++ b/charts/etrip/templates/cron.yml @@ -1,15 +1,6 @@ {{- define "etripCronjob" }} {{- $db := mergeOverwrite .global.Values.db ( .global.Values.db.operator_install | default dict ) -}} {{- $clusterName := include "clusterName" .global -}} -{{- $createSecretsVolume := false }} -{{- $createSecretsVolume = or $createSecretsVolume (eq $db.provider "cnpg") }} -{{- $createSecretsVolume = or $createSecretsVolume ( - and .global.Values.etrip.searchlogs.enabled - .global.Values.etrip.searchlogs.certificate ) }} -{{- $createSecretsVolumeMount := $createSecretsVolume }} -{{- $createSecretsVolume = or $createSecretsVolume ( - and .global.Values.elogger.enabled - .global.Values.elogger.db.certificate ) }} kind: CronJob apiVersion: batch/v1 metadata: @@ -46,34 +37,12 @@ spec: subPath: etrip - mountPath: /tmp/elogger name: elogger - {{- if $createSecretsVolumeMount }} - mountPath: /secrets name: secrets readOnly: true - {{- end }} env: - name: ETRIP_CONFIG value: /config - {{- if eq $db.provider "zalando" }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: etrip. {{- $clusterName -}} .credentials - key: password - - name: ETRIP_DSN - value: >- - user=etrip - password=$(DB_PASSWORD) - host={{ $clusterName }} - {{- if gt (int .global.Values.replicas.db) 1 }} - - name: ETRIP_DSN_RO - value: >- - user=etrip - password=$(DB_PASSWORD) - host={{ $clusterName }}-repl - connect_timeout=1 - {{- end }} - {{- else if eq $db.provider "cnpg" }} - name: ETRIP_DSN value: >- user=etrip @@ -88,7 +57,6 @@ spec: sslcert=/secrets/app-tls/postgresql.crt sslkey=/secrets/app-tls/postgresql.key sslrootcert=/secrets/app-tls/root.crt - {{- end }} {{- with .global.Values.etrip.searchlogs -}}{{- if .enabled }} {{- if not .certificate }} - name: SEARCHLOGS_DBPASSWORD @@ -192,11 +160,9 @@ spec: {{- end }} - name: elogger emptyDir: {} - {{- if $createSecretsVolume }} - name: secrets projected: sources: - {{- if eq $db.provider "cnpg" }} - secret: name: {{ .global.Release.Name }}-db-app-tls items: @@ -207,7 +173,6 @@ spec: mode: 416 - key: ca.crt path: app-tls/root.crt - {{- end }} {{- with .global.Values.etrip.searchlogs }} {{- if and .enabled .certificate }} - secret: @@ -234,7 +199,6 @@ spec: - key: ca.crt path: elogger-tls/root.crt {{- end }}{{ end }} - {{- end }} {{- end }} {{- range .Values.cron -}} {{- include "etripCronjob" (dict "cron" . "global" $ ) }} diff --git a/charts/etrip/templates/database-zalando.yml b/charts/etrip/templates/database-zalando.yml deleted file mode 100644 index 405f22f..0000000 --- a/charts/etrip/templates/database-zalando.yml +++ /dev/null @@ -1,231 +0,0 @@ -{{- $db := mergeOverwrite .Values.db ( .Values.db.operator_install | default dict ) -}} -{{- if or (eq $db.provider "zalando") (eq $db.bootstrap.mode "transition") -}} -{{- $clusterName := include "clusterName" $ -}} -apiVersion: "acid.zalan.do/v1" -kind: postgresql -metadata: - name: {{ $clusterName }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" $ | nindent 4 }} - {{- if $db.exporter.enabled }} - exporter: "true" - {{- end }}{{/* $db.exporter.enabled */}} - {{- if $db.controller }} - annotations: - acid.zalan.do/controller: {{ $db.controller }} - {{- end }}{{/* $db.controller */}} -spec: - teamId: {{ .Chart.Name }} - volume: {{ toYaml $db.dbVolume | nindent 4 }} - numberOfInstances: {{ .Values.replicas.db }} - users: - etrip: [] - databases: - etrip: etrip - postgresql: - version: "11" # postgres >12 does not support OIDs - parameters: {{ toYaml $db.dbParameters | nindent 6}} - {{- with $db.resources }} - resources: {{ toYaml . | nindent 4 }} - {{- end }}{{/* $db.resources */}} - {{- with $db.clone }} - {{- if not .namespace }} - clone: {{ toYaml . | nindent 4 }} - {{- else }} - clone: - cluster: {{ $.Release.Name }}-source - {{- end }}{{/* not .namespace */}} - {{- end }}{{/* $db.clone */}} - {{- with $db.standby }} - {{- if .enabled }} - standby: - s3_wal_path: s3://{{ .bucket }}/spilo/{{ .name | default $clusterName }}/{{ .uid }}/wal/11 - {{- end }}{{/* .enabled */}} - {{- end }}{{/* $db.standby */}} - {{- if $db.exporter.enabled }} - sidecars: - - name: exporter - image: {{ $db.exporter.image }} - ports: - - name: http-prom - containerPort: 9187 - protocol: TCP - env: - - name: DATA_SOURCE_NAME - value: postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@localhost/postgres - {{- if len $db.exporter.extraQueries }} - - name: PG_EXPORTER_EXTEND_QUERY_PATH - value: /queries.yaml - additionalVolumes: - - name: queries - mountPath: /queries.yaml - subPath: queries.yaml - volumeSource: - name: queries - configMap: - name: {{ $clusterName }}-queries - targetContainers: - - exporter ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ $clusterName }}-queries - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" $ | nindent 4 }} -data: - queries.yaml: | - {{- toYaml $db.exporter.extraQueries | nindent 4 }} - {{- end }}{{/* extraQueries */}} - {{- end }}{{/* $db.exporter.enabled */}} -{{- if and $db.clone $db.refresh }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-refresh-sa - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" $ | nindent 4 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-refresh-role - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" $ | nindent 4 }} -rules: -- apiGroups: [ acid.zalan.do ] - resources: [ postgresqls ] - resourceNames: [ {{ $clusterName }} ] - verbs: [ get, delete ] -- apiGroups: [ acid.zalan.do ] - resources: [ postgresqls ] - verbs: [ create ] -- apiGroups: [ apps ] - resources: [ deployments ] - resourceNames: [ {{ .Release.Name }}-web ] - verbs: [ get, patch ] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-refresh-rolebinding - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" $ | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-refresh-role -subjects: -- kind: ServiceAccount - name: {{ .Release.Name }}-refresh-sa ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ .Release.Name }}-refresh-db - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" $ | nindent 4 }} -spec: - schedule: {{ $db.refresh }} - concurrencyPolicy: Forbid - jobTemplate: - metadata: - labels: - cronjob-name: refresh-db - {{- include "selectorLabels" . | nindent 8 }} - spec: - template: - metadata: - labels: - cronjob-name: refresh-db - {{- include "selectorLabels" . | nindent 12 }} - spec: - restartPolicy: Never - serviceAccountName: {{ .Release.Name }}-refresh-sa - containers: - - name: kubectl - command: [ /bin/sh ] - args: - - -c - - | - kubectl get postgresql $ETRIP_CLUSTER -o yaml > /tmp/temp.yaml - kubectl delete -f /tmp/temp.yaml - sleep 5 - kubectl create -f /tmp/temp.yaml - sleep 5 - while [ "$(kubectl get -f /tmp/temp.yaml -o jsonpath='{.status.PostgresClusterStatus}')" != 'Running' ]; do - echo "Waiting for ready state" - sleep 5 - done - kubectl rollout restart deploy/${ETRIP_RELEASE}-web - image: bitnami/kubectl - env: - - name: ETRIP_RELEASE - value: {{ .Release.Name }} - - name: ETRIP_CLUSTER - value: {{ $clusterName }} -{{- end }}{{/* and $db.clone $db.refresh */}} -{{- if $db.clone }} -{{- if $db.clone.namespace }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: standby.{{ .Release.Name }}-source.credentials - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" . | nindent 4 }} - annotations: - reflector.v1.k8s.emberstack.com/reflects: "{{- $db.clone.namespace -}} /standby. - {{- $db.clone.cluster | default $clusterName -}} .credentials" -type: Opaque ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-source - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" . | nindent 4 }} -spec: - type: ExternalName - externalName: {{ $db.clone.cluster | default $clusterName -}} . - {{- $db.clone.namespace -}} .svc.cluster.local - ports: - - name: postgres - port: 5432 - protocol: TCP - targetPort: 5432 -{{- end }}{{/* $db.clone.namespace */}} -{{- end }}{{/* $db.clone */}} -{{- if $db.allowCloneFrom }} ---- -apiVersion: v1 -kind: Secret -metadata: - - name: standby.{{ $clusterName }}.credentials - namespace: {{ .Release.Namespace }} - labels: - {{- include "labels" . | nindent 4 }} - annotations: - reflector.v1.k8s.emberstack.com/reflection-allowed: "true" - reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces: {{ $db.allowCloneFrom | quote }} -type: Opaque -data: - username: {{ "standby" | b64enc }} - {{- $secret := (lookup "v1" "Secret" .Release.Namespace (printf "standby.%s.credentials" $clusterName) ) }} - {{- if $secret }} - password: {{ $secret.data.password }} - {{- else }} - password: {{ randAlphaNum 64 | b64enc }} - {{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/etrip/templates/database-cnpg.yml b/charts/etrip/templates/database.yml similarity index 86% rename from charts/etrip/templates/database-cnpg.yml rename to charts/etrip/templates/database.yml index 8e89fac..ce79ca8 100644 --- a/charts/etrip/templates/database-cnpg.yml +++ b/charts/etrip/templates/database.yml @@ -1,5 +1,4 @@ {{- $db := mergeOverwrite .Values.db ( .Values.db.operator_install | default dict ) -}} -{{- if or (eq $db.provider "cnpg") (eq $db.bootstrap.mode "transition") -}} {{- if $db.generateIssuer }} apiVersion: cert-manager.io/v1 kind: Certificate @@ -144,27 +143,10 @@ kind: Cluster metadata: name: {{ .Release.Name }}-db namespace: {{ .Release.Namespace }} - {{- /* Fencing mode */}} - {{- if and (eq $db.bootstrap.mode "transition") (eq $db.bootstrap.transitionPhase "standby_fence") }} - annotations: - cnpg.io/fencedInstances: '["*"]' - {{- end }} labels: {{- include "labels" $ | nindent 4 }} spec: - {{- /* - CNPG bug: while transitioning, the multitude of reconciliation errors caused - by zalando incompatibilities before cleanup scripts get applied confuse the - CNPG operator and cause the cluster to get stuck in an unrecoverable state - as far as the operator can see. This is recoverable only if no other - instances are spawned while the first one gets cleaned up. After transition - phase is complete (transitionPhase set to full or mode switched away from - "transition") we can allow more than one instance to be spawned. */}} - {{- if and (eq $db.bootstrap.mode "transition") $db.bootstrap.transitionPhase }} - instances: 1 - {{- else }} instances: {{ .Values.replicas.db }} - {{- end }} imageName: {{ $db.image.registry -}} / {{- $db.image.name -}} : {{- $db.image.tag }} @@ -223,4 +205,3 @@ spec: schedule: {{ .schedule | quote }} backupOwnerReference: self {{- end }}{{ end }}{{/* s3 enabled, schedule, backup */}} -{{- end }} diff --git a/charts/etrip/templates/deploy.yml b/charts/etrip/templates/deploy.yml index fd59210..4198880 100644 --- a/charts/etrip/templates/deploy.yml +++ b/charts/etrip/templates/deploy.yml @@ -1,14 +1,5 @@ {{- $db := mergeOverwrite .Values.db ( .Values.db.operator_install | default dict ) -}} {{- $clusterName := include "clusterName" $ -}} -{{- $createSecretsVolume := false }} -{{- $createSecretsVolume = or $createSecretsVolume (eq $db.provider "cnpg") }} -{{- $createSecretsVolume = or $createSecretsVolume ( - and .Values.etrip.searchlogs.enabled - .Values.etrip.searchlogs.certificate ) }} -{{- $createSecretsVolumeMount := $createSecretsVolume -}} -{{- $createSecretsVolume = or $createSecretsVolume ( - and .Values.elogger.enabled - .Values.elogger.db.certificate) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -40,14 +31,7 @@ spec: fsGroup: 33 initContainers: {{- /* no migrations if standby mode */}} - {{- if or - (and (eq $db.provider "zalando") ( not $db.standby.enabled )) - (and (eq $db.provider "cnpg") ( not - (or $db.bootstrap.standby - (and (eq $db.bootstrap.mode "transition") - (eq $db.bootstrap.transitionPhase "standby")) - ))) - }} + {{- if not $db.bootstrap.standby }} - name: migrations {{- include "etripImage" . | nindent 8 }} command: [ "/var/www/localhost/migrate" ] @@ -57,26 +41,12 @@ spec: subPath: etrip - mountPath: /tmp/elogger name: elogger - {{- if $createSecretsVolumeMount }} - mountPath: /secrets name: secrets readOnly: true - {{- end }} env: - name: ETRIP_CONFIG value: /config - {{- if eq $db.provider "zalando" }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: etrip. {{- $clusterName -}} .credentials - key: password - - name: ETRIP_DSN - value: >- - user=etrip - password=$(DB_PASSWORD) - host={{ $clusterName }} - {{- else if eq $db.provider "cnpg" }} - name: ETRIP_DSN value: >- user=etrip @@ -84,7 +54,6 @@ spec: sslcert=/secrets/app-tls/postgresql.crt sslkey=/secrets/app-tls/postgresql.key sslrootcert=/secrets/app-tls/root.crt - {{- end }} resources: {{- toYaml .Values.resources.migrate | nindent 10 }} {{- end }} @@ -119,34 +88,12 @@ spec: - mountPath: /frontends name: frontends {{- end}} - {{- if $createSecretsVolumeMount }} - mountPath: /secrets name: secrets readOnly: true - {{- end }} env: - name: ETRIP_CONFIG value: /config - {{- if eq $db.provider "zalando" }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: etrip. {{- $clusterName -}} .credentials - key: password - - name: ETRIP_DSN - value: >- - user=etrip - password=$(DB_PASSWORD) - host={{ $clusterName }} - {{- if gt (int .Values.replicas.db) 1 }} - - name: ETRIP_DSN_RO - value: >- - user=etrip - password=$(DB_PASSWORD) - host={{ $clusterName }}-repl - connect_timeout=1 - {{- end }} - {{- else if eq $db.provider "cnpg" }} - name: ETRIP_DSN value: >- user=etrip @@ -161,7 +108,6 @@ spec: sslcert=/secrets/app-tls/postgresql.crt sslkey=/secrets/app-tls/postgresql.key sslrootcert=/secrets/app-tls/root.crt - {{- end }} {{- if .Values.etrip.use_redis }} - name: REDIS_SECRET valueFrom: @@ -346,11 +292,9 @@ spec: - key: b2b-{{$b2bkey}}.json path: etrip/b2b-{{$b2bkey}}.json {{ end }} - {{- if $createSecretsVolume }} - name: secrets projected: sources: - {{- if eq $db.provider "cnpg" }} - secret: name: {{ .Release.Name }}-db-app-tls items: @@ -361,7 +305,6 @@ spec: mode: 416 - key: ca.crt path: app-tls/root.crt - {{- end }} {{- with .Values.etrip.searchlogs }} {{- if and .enabled .certificate }} - secret: @@ -388,7 +331,6 @@ spec: - key: ca.crt path: elogger-tls/root.crt {{- end }}{{ end }} - {{- end }} - name: elogger emptyDir: {} {{- if .Values.frontends.enabled}} diff --git a/charts/etrip/values.yaml b/charts/etrip/values.yaml index 651fbfb..0acfa72 100644 --- a/charts/etrip/values.yaml +++ b/charts/etrip/values.yaml @@ -814,8 +814,6 @@ cron: [] # resources: {} db: - provider: zalando # cnpg or zalando - dbParameters: max_connections: "100" shared_buffers: 1GB @@ -851,7 +849,7 @@ db: pullPolicy: IfNotPresent registry: ghcr.io/tripsolutions name: postgresql - tag: 11.20-cron + tag: 16.1-cron ## CNPG specific: generateIssuer: true @@ -895,27 +893,6 @@ db: encryption: AES256 {{- end }}{{/* define "etrip.db.template.objectStore" */}} - {{- if eq .db.bootstrap.mode "transition" }} - {{/* transition mode is very particular so we have a dedicated section */}} - {{- $phase := .db.bootstrap.transitionPhase }} - bootstrap: - pg_basebackup: - database: etrip - source: zalando - {{- if or (eq $phase "standby") (eq $phase "standby_fence") }} - replica: - enabled: true - source: zalando - {{- end }}{{/* $phase is standby */}} - externalClusters: - - name: zalando - connectionParameters: - host: {{ .Release.Name }}-cluster - user: standby - password: - name: standby.{{ .Release.Name }}-cluster.credentials - key: password - {{- else }}{{/* .db.bootstrap.mode != "transition" */}} bootstrap: {{- if eq .db.bootstrap.mode "normal" }} initdb: @@ -939,7 +916,7 @@ db: - name: bootstrap {{- $s3 := mergeOverwrite (deepCopy .db.backup.s3) ( .db.bootstrap.recovery.s3Override | default dict ) }} {{- /* server name is by default same as externalCluster's name */}} - {{- $s3 := merge $s3 ( dict "serverName" (print .Release.Name "-db") )}} + {{- $s3 := merge $s3 ( dict "serverName" (print .Release.Name "-db-pg16") )}} barmanObjectStore: {{ include "etrip.db.template.objectStore" $s3 | nindent 8 }} {{- else if eq .db.bootstrap.recovery.source "backup" }} backup: {{ .db.bootstrap.recovery.backupName }} @@ -984,13 +961,13 @@ db: enabled: true source: bootstrap {{- end }}{{/* bootstrap.standby */}} - {{- end }}{{/* bootstrap.mode != transition */}} {{- with .db.backup }} {{- if .s3.enabled }} backup: retentionPolicy: {{ .retentionPolicy | default "90d" }} target: {{ .target | default "prefer-standby" }} + {{- $s3 := merge .s3 ( dict "serverName" (print $.Release.Name "-db-pg16") ) }} barmanObjectStore: {{ include "etrip.db.template.objectStore" .s3 | nindent 4 }} {{- end }} {{- end }} @@ -1003,7 +980,6 @@ db: ## * normal: bootstrap a new cluster ## * recovery: recover a cluster from a backup ## * clone: clone a live cluster - ## * transition: transition a cluster from zalando to CNPG, observing transitionPhase recovery: source: objectstore ## * objectStore: recover from the object store as defined under backup.s3, @@ -1049,23 +1025,6 @@ db: # sslRootCert: # name: # key: - - transitionPhase: standby_fence - # one of: - # - "standby_fence" - CNPG initiates a clone from zalando, puts it in - # fenced mode and waits for operator intervention - # Operator needs to execute cleanup.sh script and then set - # .transitionPhase to "standby" - # - "standby" - Assumes that previous phase was "standby_fence" and that - # cleanup.sh was executed. Fence will be removed. - # - "full" - Assumes that previous phase was standby or standby_fence and - # that cleanup.sh was executed. Fence will be removed and standby will be - # promoted to master, new timeline will be created. - # While transitioning both databases will coexist. Provider will dictate which - # database is used by the application. You can switch provider while - # transitioning after unfencing, or you can switch it after fully - # transitioning. Once .transitionPhase is set back to null the other - # operator's database will be purged! standby: false ## creates a standby cluster using the same source as the clone or recovery @@ -1113,42 +1072,10 @@ db: # allowCloneFrom: - ## Zalando specific: - # clone: - # cluster: - # namespace: - # refresh: 10 2 * * * - standby: - enabled: false - bucket: k8s-pg-wal - uid: null - # don't override operator defaults resources: {} exporter: enabled: false - # zalando specific - image: quay.io/prometheuscommunity/postgres-exporter:v0.10.1 - extraQueries: - pg_replication: - query: |- - SELECT - CASE WHEN pg_is_in_recovery() - THEN extract(EPOCH FROM CURRENT_TIMESTAMP - - pg_last_xact_replay_timestamp()) - ELSE NULL END AS lag, - pg_wal_lsn_diff( - CASE WHEN pg_is_in_recovery() - THEN pg_last_wal_replay_lsn() - ELSE pg_current_wal_lsn() END, - '0/0') AS current_lsn; - metrics: - - lag: - usage: GAUGE - description: Replication lag behind master in seconds - - current_lsn: - usage: COUNTER - description: Current WAL/LSN stream location in bytes resources: migrate: