Merge pull request #97 from trifork/feature/topic-definitions

Feature/topic definitions

charts/flink-job/Chart.yaml

@@ -7,7 +7,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.6
+version: 0.4.7
 
 dependencies:
   - name: image-automation

charts/flink-job/README.md

@@ -1,6 +1,6 @@
 # flink-job
 
-![Version: 0.4.6](https://img.shields.io/badge/Version-0.4.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.4.7](https://img.shields.io/badge/Version-0.4.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 A Helm chart for handling Cheetah Data Platform Flink jobs
 
@@ -148,7 +148,7 @@ Read more about Flink and highly available job-managers [here](https://nightlies
 | job.jarURI | string | `""` | The path of the job jar |
 | job.entryClass | string | `""` | The name of the job class |
 | job.args | list | `[]` | Arguments for the job |
-| job.topics | list | `[]` | Define which topics this job will consume. Used for data-discovery in Cheetah Backstage. If the `arg` variable is set, adds `--<arg> <name>` to the arguments passed to the job See [values.yaml](values.yaml) for the format |
+| job.topics | list | `[]` | Define which topics this job will consume. Used for data-discovery in Cheetah Backstage. If the `arg` variable is set, adds `--<arg> <name>` to the arguments passed to the job If `arg` is not set, adds `--<type>-kafka-topic <name>` or `--<type>-kafka-topic-<postfix> <name>` See [values.yaml](values.yaml) for the format |
 | job.parallelism | int | `1` | How many jobs to run in parallel, see more here: <https://nightlies.apache.org/flink/flink-docs-master/docs/dev/datastream/execution/parallel/> |
 | job.state | string | `"running"` | Desired state of the job. Must be either: `running` or `suspended` |
 | job.upgradeMode | string | `"savepoint"` | Application upgrade mode. Must be either: stateless, last_state, savepoint `stateless` upgrades is done from an empty state `last-state` does a quick upgrade. Does not require the job to be in a healthy state, as it makes use of the HA metadata `savepoint` makes use of savepoints when upgrading and requires the job to be running. This provides maximal safety Read more here: <https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main/docs/custom-resource/job-management/#stateful-and-stateless-application-upgrades> |

charts/flink-job/ci/example-values.yaml

@@ -18,6 +18,10 @@ job:
   jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar
   entryClass: org.apache.flink.streaming.examples.statemachine.StateMachineExample
   parallelism: 2
+  topics:
+  - name: test
+    type: input
+    postfix: postfix
 taskManager:
   replicas: 2
   resource:

charts/flink-job/templates/flink-deployment.yaml

@@ -40,6 +40,13 @@ spec:
       {{- if .arg }}
       - {{ printf "--%s" (trimPrefix "--" .arg) | quote }}
       - {{ printf "%s" .name | quote }}
+      {{- else }}
+        {{- if .postfix }}
+      - {{ printf "--%s-kafka-topic-%s" .type .postfix | quote }}
+        {{- else }}
+      - {{ printf "--%s-kafka-topic" .type | quote }}
+        {{- end }}
+      - {{ printf "%s" .name | quote }}
       {{- end }}
       {{- end }}
       {{- range .Values.job.args }}

charts/flink-job/values.yaml

@@ -114,10 +114,12 @@ job:
   # -- Define which topics this job will consume.
   # Used for data-discovery in Cheetah Backstage.
   # If the `arg` variable is set, adds `--<arg> <name>` to the arguments passed to the job
+  # If `arg` is not set, adds `--<type>-kafka-topic <name>` or `--<type>-kafka-topic-<postfix> <name>`
   # See [values.yaml](values.yaml) for the format
   topics: []
   # must be defined as follows
   # - arg: <optional name of argument>
+  #   postfix: <optional postfix for topic argument>
   #   type: <type of topic (input/output)>
   #   name: <name of topic>
   # ie:
@@ -129,6 +131,9 @@ job:
   # - arg: output-kafka-topic
   #   type: output
   #   name: sinkTopic
+  # - postfix: special
+  #   type: output
+  #   name: specialSinkTopic
 
   # -- How many jobs to run in parallel,
   # see more here: <https://nightlies.apache.org/flink/flink-docs-master/docs/dev/datastream/execution/parallel/>