Clean up molecule test play syntax (#35)

* Clean up yaml syntax for molecule test plays
nephelaiio · Jun 20, 2023 · 52e02bf · 52e02bf
1 parent 1bb38c3
commit 52e02bf
Show file tree

Hide file tree

Showing 9 changed files with 187 additions and 205 deletions.
diff --git a/README.md b/README.md
@@ -21,10 +21,10 @@ helm install dataplane/dataplane
 
 ## TODO
 In order of priority
+* Add support for partitioned PostgreSQL tables
 * Add support for MySQL sources
 * Add config options for warehouse backups
 * Add monitoring for Kafka topics
-* Add Apache Flink deployment
 * Add table exclude support for cdc connectors
 
 ## Dependencies

diff --git a/charts/dataplane/Chart.yaml b/charts/dataplane/Chart.yaml
@@ -16,10 +16,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.10
+version: 0.1.11
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: 0.1.10
+appVersion: 0.1.11
diff --git a/charts/dataplane/values.yaml b/charts/dataplane/values.yaml
@@ -7,11 +7,20 @@ util:
   image:
     repository: nephelaiio/dataplane-util
     pullPolicy: IfNotPresent
-    tag: dataplane-0.1.10
+    tag: dataplane-0.1.11
   resources: {}
 
 cdc:
   postgres: []
+  #  postgres:
+  #    - hostname: "network.hostname"
+  #      port: 5432
+  #      servername: "host.identifier"
+  #      dbname: "sourcedb"
+  #      signaling: "signaling_table"
+  #      partitions:
+  #        - source: "payments.*"
+  #          sink: "payments"
   mysql: []
 
 strimzi:
@@ -36,7 +45,7 @@ strimzi:
   connect:
     image:
       repository: nephelaiio/dataplane-connect
-      tag: dataplane-0.1.10
+      tag: dataplane-0.1.11
     replicas: 1
     config:
       group.id: connect-cluster
@@ -91,7 +100,7 @@ metabase:
     image:
       repository: nephelaiio/dataplane-util
       pullPolicy: IfNotPresent
-      tag: dataplane-0.1.10
+      tag: dataplane-0.1.11
     securityContext: {}
 
   image:

diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml
@@ -6,11 +6,9 @@
   gather_facts: false
 
   roles:
-
     - nephelaiio.plugins
 
   tasks:
-
     - name: wait for strimzi crd deployment
       kubernetes.core.k8s_info:
         api_version: v1
@@ -86,15 +84,14 @@
       ansible.builtin.fail:
         msg: "zalando clusters failed to come up on time"
       vars:
-        cluster_failed: "{{ cluster_status | rejectattr('status.PostgresClusterStatus', 'equalto', 'Running') | map(attribute='metadata.name') }}"
+        cluster_failed_data: "{{ cluster_status | rejectattr('status.PostgresClusterStatus', 'equalto', 'Running') }}"
+        cluster_failed: "{{ cluster_failed_data | map(attribute='metadata.name') }}"
         cluster_status: "{{ cluster_query | selectattr('status', 'defined') }}"
-        cluster_query: "{{
-          query(
-            'kubernetes.core.k8s',
-            api_version='acid.zalan.do/v1',
-            kind='postgresql',
-            kubeconfig=k8s_kubeconfig
-          )}}"
+        cluster_query: "{{ query(
+          'kubernetes.core.k8s',
+          api_version='acid.zalan.do/v1',
+          kind='postgresql',
+          kubeconfig=k8s_kubeconfig) }}"
       retries: 15
       delay: 30
       until: cluster_failed | length == 0
@@ -122,13 +119,11 @@
       vars:
         cluster_failed: "{{ cluster_status | rejectattr('status.PostgresClusterStatus', 'equalto', 'Running') | map(attribute='metadata.name') }}"
         cluster_status: "{{ cluster_query | selectattr('status', 'defined') }}"
-        cluster_query: "{{
-          query(
-            'kubernetes.core.k8s',
-            api_version='acid.zalan.do/v1',
-            kind='postgresql',
-            kubeconfig=k8s_kubeconfig
-          )}}"
+        cluster_query: "{{ query(
+          'kubernetes.core.k8s',
+          api_version='acid.zalan.do/v1',
+          kind='postgresql',
+          kubeconfig=k8s_kubeconfig) }}"
       retries: 15
       delay: 30
       until: cluster_failed | length == 0
@@ -138,32 +133,28 @@
 
     - name: query pagila connection data
       ansible.builtin.set_fact:
-        pagila_user: "{{ pagila_db_secret_data.data.username | b64decode }}"
-        pagila_pass: "{{ pagila_db_secret_data.data.password | b64decode }}"
+        pagila_user: "{{ _db_secret_data.data.username | b64decode }}"
+        pagila_pass: "{{ _db_secret_data.data.password | b64decode }}"
       vars:
-        pagila_db_secret_name: "{{ dataplane_pagila_user }}-{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}"
-        pagila_db_secret_data: "{{ secret_query | selectattr('metadata.name', 'equalto', pagila_db_secret_name) | first }}"
-        secret_query: "{{
-          query(
-            'kubernetes.core.k8s',
-            namespace=dataplane_pagila_namespace,
-            kind='Secret',
-            kubeconfig=k8s_kubeconfig
-          )}}"
+        _db_secret_name: "{{ dataplane_pagila_user }}-{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}"
+        _db_secret_data: "{{ secret_query | selectattr('metadata.name', 'equalto', _db_secret_name) | first }}"
+        secret_query: "{{ query(
+          'kubernetes.core.k8s',
+          namespace=dataplane_pagila_namespace,
+          kind='Secret',
+          kubeconfig=k8s_kubeconfig) }}"
 
     - name: query pagila service data
       ansible.builtin.set_fact:
-        pagila_host: "{{ pagila_db_svc_data.status.loadBalancer.ingress[0].ip }}"
+        pagila_host: "{{ _db_svc_data.status.loadBalancer.ingress[0].ip }}"
       vars:
-        pagila_db_svc_name: "{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}"
-        pagila_db_svc_data: "{{ service_query | selectattr('metadata.name', 'equalto', pagila_db_svc_name) | first }}"
-        service_query: "{{
-          query(
-            'kubernetes.core.k8s',
-            namespace=dataplane_pagila_namespace,
-            kind='Service',
-            kubeconfig=k8s_kubeconfig
-          )}}"
+        _db_svc_name: "{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}"
+        _db_svc_data: "{{ service_query | selectattr('metadata.name', 'equalto', _db_svc_name) | first }}"
+        service_query: "{{ query(
+          'kubernetes.core.k8s',
+          namespace=dataplane_pagila_namespace,
+          kind='Service',
+          kubeconfig=k8s_kubeconfig) }}"
 
     - name: wait for database port open
       ansible.builtin.wait_for:
@@ -185,8 +176,8 @@
       register: datadir
       changed_when: false
 
-    - block:
-
+    - name: load pagila data
+      block:
         - name: fetch pagila schema file
           ansible.builtin.get_url:
             url: https://github.com/devrimgunduz/pagila/raw/master/pagila-schema.sql
@@ -200,7 +191,8 @@
           changed_when: false
 
         - name: concatenate pagila sql files
-          ansible.builtin.shell: "cat {{ datadir.path }}/schema.sql {{ datadir.path }}/data.sql > {{ datadir.path }}/pagila.sql"
+          ansible.builtin.shell: >
+            cat {{ datadir.path }}/schema.sql {{ datadir.path }}/data.sql > {{ datadir.path }}/pagila.sql
 
         - name: load pagila db data
           community.postgresql.postgresql_db:
@@ -213,7 +205,6 @@
           changed_when: false
 
       always:
-
         - name: destroy data tempdir
           ansible.builtin.file:
             state: absent

diff --git a/molecule/default/create.yml b/molecule/default/create.yml
@@ -6,19 +6,16 @@
   gather_facts: false
 
   roles:
-
     - nephelaiio.plugins
 
   pre_tasks:
-
     - name: deploy kind cluster
       ansible.builtin.include_role:
         name: nephelaiio.kind
 
     - name: set helm release metadata
       when: helm_release is not defined
       block:
-
         - name: query helm releases
           ansible.builtin.uri:
             url: https://api.github.com/repos/helm/helm/releases/latest
@@ -35,13 +32,11 @@
             helm_release: "{{ helm_release_query.json.tag_name }}"
 
       rescue:
-
         - name: set kind release to failback
           ansible.builtin.set_fact:
             helm_release: v3.11.3
 
   tasks:
-
     - name: create temporary directory
       ansible.builtin.tempfile:
         state: directory
@@ -51,7 +46,6 @@
 
     - name: install helm binary
       block:
-
         - name: download helm release package
           ansible.builtin.unarchive:
             src: "https://get.helm.sh/helm-{{ helm_release }}-linux-amd64.tar.gz"
@@ -70,17 +64,16 @@
             recurse: true
             use_regex: true
             patterns:
-              - '^helm$'
+              - "^helm$"
           register: helm_bin
 
         - name: copy helm executable
           ansible.builtin.copy:
             src: "{{ helm_bin.files[0].path }}"
             dest: "{{ k8s_helm_bin }}"
-            mode: '0755'
+            mode: "0755"
 
       always:
-
         - name: cleanup temp files
           ansible.builtin.file:
             state: absent

diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
@@ -93,6 +93,9 @@ provisioner:
                   servername: pagila
                   dbname: "{{ dataplane_pagila_db }}"
                   signaling: "{{ dataplane_pagila_signaling }}"
+                  partitions:
+                    - source: "payments.*"
+                      sink: "payments"
             strimzi:
               connect:
                 image:

diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml
@@ -6,17 +6,14 @@
   gather_facts: false
 
   vars:
-
     k8s_deploy: true
     k8s_verify: false
 
   roles:
-
     - nephelaiio.plugins
     - nephelaiio.k8s
 
   tasks:
-
     - name: install postgresql binary
       ansible.builtin.package:
         name: postgresql-client