Allow custom connector names (#44)

* Allow custom connector names

* Update README

* Add connector name verification task

README.md

@@ -34,6 +34,7 @@ metabase:
 cdc:
   postgres:
     - hostname: pagilahost
+      connector: pagila-connector
       id: pagila
       dbname: pagila
       exclude:
@@ -55,6 +56,10 @@ zalando:
     class: standard
 ```
 
+The following fields are immutable; modifying them will leave an orphan DB replication slot and _will_ result in space exhaustion:
+* `cdc.postgres[*].id`
+* `cdc.postgres[*].partitions[*].sink`
+
 ## Roadmap
 In order of priority
 * Create python package for maintenance operations

charts/dataplane/Chart.yaml

@@ -16,10 +16,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.15
+version: 0.1.16
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: 0.1.15
+appVersion: 0.1.16

charts/dataplane/templates/strimzi/connector.yaml

@@ -4,11 +4,13 @@
 
 {{- range $db := .Values.cdc.postgres }}
 {{-   $dbId := required "database id is required" $db.id }}
-{{-   $dbConnectorName := lower (printf "%s-%s" $fullName $dbId) }}
-{{-   $dbSlotName := $dbConnectorName | replace "-" "_" }}
+{{-   $dbSlotName := lower $dbId | replace "." "-" }}
 {{-   $exclude := list }}
 {{-   $include := list }}
 
+{{-   $dbConnector := $db.connector | default $dbId }}
+{{-   $dbConnectorName := lower $dbConnector | replace "." "-" }}
+
 {{-   if and (hasKey $db "exclude") (hasKey $db "include") }}
 {{-     fail "table excludes and includes are mutually exclusive"  }}
 {{-   end }}
@@ -87,7 +89,8 @@ spec:
 {{-         $partitionSource := required "partition source is required" $partition.source }}
 {{-         $partitionSink := required "partition sink is required" $partition.sink }}
 {{-         $partitionConnectorName := lower (printf "%s-%s" $dbConnectorName $partitionSink) | replace "." "-" }}
-{{-         $partitionSlotName := $partitionConnectorName | replace "-" "_" }}
+{{-         $partitionSlot :=  lower (printf "%s_%s" $dbSlotName $partitionSink) }}
+{{-         $partitionSlotName :=  $partitionSlot | replace "." "-" | replace "-" "_" }}
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaConnector
@@ -131,7 +134,7 @@ spec:
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaConnector
 metadata:
-  name: {{ $fullName }}-warehouse-sink
+  name: {{ $fullName }}-warehouse-postgresql
   labels:
     strimzi.io/cluster: {{ $fullName }}
 spec:

charts/dataplane/values.yaml

@@ -7,7 +7,7 @@ util:
   image:
     repository: nephelaiio/dataplane-util
     pullPolicy: IfNotPresent
-    tag: dataplane-0.1.14
+    tag: dataplane-0.1.16
   resources: {}
 
 cdc:
@@ -45,7 +45,7 @@ strimzi:
   connect:
     image:
       repository: nephelaiio/dataplane-connect
-      tag: dataplane-0.1.14
+      tag: dataplane-0.1.16
     replicas: 1
     config:
       group.id: connect-cluster
@@ -100,7 +100,7 @@ metabase:
     image:
       repository: nephelaiio/dataplane-util
       pullPolicy: IfNotPresent
-      tag: dataplane-0.1.14
+      tag: dataplane-0.1.16
     securityContext: {}
 
   image:

molecule/default/molecule.yml

@@ -91,6 +91,7 @@ provisioner:
                 - hostname: "{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}"
                   port: 5432
                   id: pagila
+                  connector: pagila-connector
                   dbname: "{{ dataplane_pagila_db }}"
                   signaling: "{{ dataplane_pagila_signaling }}"
                   exclude:

molecule/default/verify.yml

@@ -156,6 +156,22 @@
             - name: fail verification
               ansible.builtin.fail:
 
+        - name: validate kafka connector name
+          ansible.builtin.fail:
+            msg: "connector dataplane-connector was not found"
+          vars:
+            connector_matches: "{{ connector_query | selectattr('metadata.name', 'equalto', 'pagila-connector') | length }}"
+            connector_present: "{{ connector_matches | int > 0 }}"
+            connector_query: "{{
+              query(
+                'kubernetes.core.k8s',
+                api_version='',
+                kind='KafkaConnector',
+                kubeconfig=k8s_kubeconfig
+              ) }}"
+          until: connector_present
+          failed_when: not connector_present
+
         - name: wait for cdc to complete
           pause:
             minutes: 5

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dataplane"
-version = "0.1.15"
+version = "0.1.16"
 description = ""
 authors = ["Teodoro Cook <[email protected]>"]
 

values.minimal.yml

@@ -9,6 +9,7 @@ metabase:
 cdc:
   postgres:
     - hostname: pagilahost
+      connector: pagila-connector
       id: pagila
       dbname: pagila
       exclude: