diff --git a/README.md b/README.md index 2b6dcdf..c54d2df 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ metabase: cdc: postgres: - hostname: pagilahost + connector: pagila-connector id: pagila dbname: pagila exclude: @@ -55,6 +56,10 @@ zalando: class: standard ``` +The following fields are immutable; modifying them will leave an orphan DB replication slot and _will_ result in space exhaustion: +* `cdc.postgres[*].id` +* `cdc.postgres[*].partitions[*].sink` + ## Roadmap In order of priority * Create python package for maintenance operations diff --git a/charts/dataplane/Chart.yaml b/charts/dataplane/Chart.yaml index 6a3a864..f2a142a 100644 --- a/charts/dataplane/Chart.yaml +++ b/charts/dataplane/Chart.yaml @@ -16,10 +16,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.15 +version: 0.1.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.1.15 +appVersion: 0.1.16 diff --git a/charts/dataplane/templates/strimzi/connector.yaml b/charts/dataplane/templates/strimzi/connector.yaml index bf12846..23c7833 100644 --- a/charts/dataplane/templates/strimzi/connector.yaml +++ b/charts/dataplane/templates/strimzi/connector.yaml @@ -4,11 +4,13 @@ {{- range $db := .Values.cdc.postgres }} {{- $dbId := required "database id is required" $db.id }} -{{- $dbConnectorName := lower (printf "%s-%s" $fullName $dbId) }} -{{- $dbSlotName := $dbConnectorName | replace "-" "_" }} +{{- $dbSlotName := lower $dbId | replace "." "-" }} {{- $exclude := list }} {{- $include := list }} +{{- $dbConnector := $db.connector | default $dbId }} +{{- $dbConnectorName := lower $dbConnector | replace "." "-" }} + {{- if and (hasKey $db "exclude") (hasKey $db "include") }} {{- fail "table excludes and includes are mutually exclusive" }} {{- end }} @@ -87,7 +89,8 @@ spec: {{- $partitionSource := required "partition source is required" $partition.source }} {{- $partitionSink := required "partition sink is required" $partition.sink }} {{- $partitionConnectorName := lower (printf "%s-%s" $dbConnectorName $partitionSink) | replace "." "-" }} -{{- $partitionSlotName := $partitionConnectorName | replace "-" "_" }} +{{- $partitionSlot := lower (printf "%s_%s" $dbSlotName $partitionSink) }} +{{- $partitionSlotName := $partitionSlot | replace "." "-" | replace "-" "_" }} --- apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaConnector @@ -131,7 +134,7 @@ spec: apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaConnector metadata: - name: {{ $fullName }}-warehouse-sink + name: {{ $fullName }}-warehouse-postgresql labels: strimzi.io/cluster: {{ $fullName }} spec: diff --git a/charts/dataplane/values.yaml b/charts/dataplane/values.yaml index b8db682..8bc03b8 100644 --- a/charts/dataplane/values.yaml +++ b/charts/dataplane/values.yaml @@ -7,7 +7,7 @@ util: image: repository: nephelaiio/dataplane-util pullPolicy: IfNotPresent - tag: dataplane-0.1.14 + tag: dataplane-0.1.16 resources: {} cdc: @@ -45,7 +45,7 @@ strimzi: connect: image: repository: nephelaiio/dataplane-connect - tag: dataplane-0.1.14 + tag: dataplane-0.1.16 replicas: 1 config: group.id: connect-cluster @@ -100,7 +100,7 @@ metabase: image: repository: nephelaiio/dataplane-util pullPolicy: IfNotPresent - tag: dataplane-0.1.14 + tag: dataplane-0.1.16 securityContext: {} image: diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 1775b0d..15ec3a0 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -91,6 +91,7 @@ provisioner: - hostname: "{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}" port: 5432 id: pagila + connector: pagila-connector dbname: "{{ dataplane_pagila_db }}" signaling: "{{ dataplane_pagila_signaling }}" exclude: diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index c718b68..6811493 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -156,6 +156,22 @@ - name: fail verification ansible.builtin.fail: + - name: validate kafka connector name + ansible.builtin.fail: + msg: "connector dataplane-connector was not found" + vars: + connector_matches: "{{ connector_query | selectattr('metadata.name', 'equalto', 'pagila-connector') | length }}" + connector_present: "{{ connector_matches | int > 0 }}" + connector_query: "{{ + query( + 'kubernetes.core.k8s', + api_version='', + kind='KafkaConnector', + kubeconfig=k8s_kubeconfig + ) }}" + until: connector_present + failed_when: not connector_present + - name: wait for cdc to complete pause: minutes: 5 diff --git a/pyproject.toml b/pyproject.toml index 5466a47..5e0965b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dataplane" -version = "0.1.15" +version = "0.1.16" description = "" authors = ["Teodoro Cook "] diff --git a/values.minimal.yml b/values.minimal.yml index 6d50906..77383fd 100644 --- a/values.minimal.yml +++ b/values.minimal.yml @@ -9,6 +9,7 @@ metabase: cdc: postgres: - hostname: pagilahost + connector: pagila-connector id: pagila dbname: pagila exclude: