diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a67759f0..7c525962 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -39,15 +39,37 @@ jobs: with: k3s-channel: ${{ matrix.k3s-channel }} + - name: Setup cert-manager + run: | + helm repo add jetstack https://charts.jetstack.io --force-update + helm repo update + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.crds.yaml + helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.14.5 + - name: Update stackstorm-ha chart dependencies run: | set -x helm dependency update + - name: Helm template test + run: | + openssl req -x509 -nodes -days 2 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=StackStorm CA" + helm template --debug \ + --name-template stackstorm-ha . \ + --set "secret.ca.crt=$(cat tls.crt | base64 -w0)" + --set "secret.ca.key=$(cat tls.key | base64 -w0)" + - name: Helm install run: | + openssl req -x509 -nodes -days 2 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=StackStorm CA" helm install --timeout 15m0s --debug --wait \ - --name-template stackstorm-ha . + --name-template stackstorm-ha . \ + --set "secret.ca.crt=$(cat tls.crt | base64 -w0)" + --set "secret.ca.key=$(cat tls.key | base64 -w0)" - name: Helm test run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 919af036..88bf1c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## Development +* Enable the use of TLS between all internal components (aside from Redis), and enable it by default (#401) (by @jk464) ## v1.1.0 * Fix syntax with ensure-packs-volumes-are-writable job (#403, #411) (by @skiedude) diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 9cadeeab..138a05b8 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -422,3 +422,39 @@ Create the custom env list for each deployment value: {{ $value | quote }} {{- end }} {{- end -}} + +{{/* +Set up values for Internal TLS +*/}} +{{- define "stackstorm-ha.internal_tls.cert_volume.mount" -}} +{{- if or .Values.st2.tls.enabled .Values.mongodb.tls.enabled .Values.rabbitmq.tls.enabled }} +- name: {{ .Values.st2.tls.secretName }} + mountPath: {{ .Values.st2.tls.mountPath }}/ + readOnly: true +{{- end }} +{{- end -}} +{{- define "stackstorm-ha.internal_tls.cert_volume.volume" -}} +{{- if or .Values.st2.tls.enabled .Values.mongodb.tls.enabled .Values.rabbitmq.tls.enabled }} +- name: {{ .Values.st2.tls.secretName }} + secret: + secretName: {{ .Values.st2.tls.secretName }} +{{- end }} +{{- end -}} + +{{/* +Define st2web ports +*/}} +{{- define "stackstorm-ha.st2web.http_port" -}} +{{- if ne (default 0 ((($.Values.st2web.securityContext).runAsUser) | int)) 0 -}} +8080 +{{- else -}} +80 +{{- end -}} +{{- end -}} +{{- define "stackstorm-ha.st2web.https_port" -}} +{{- if ne (default 0 ((($.Values.st2web.securityContext).runAsUser) | int)) 0 -}} +8443 +{{- else -}} +443 +{{- end -}} +{{- end -}} diff --git a/templates/ca.yaml b/templates/ca.yaml new file mode 100644 index 00000000..efdccc18 --- /dev/null +++ b/templates/ca.yaml @@ -0,0 +1,21 @@ +{{- if not ( .Values.st2.tls.certificate_issuer.existing ) -}} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Values.st2.tls.certificate_issuer.name }} + namespace: "{{ $.Release.Namespace }}" +spec: + ca: + secretName: {{ .Values.st2.tls.certificate_issuer.name }}-tls +--- +apiVersion: v1 +data: + tls.crt: "{{ .Values.secret.ca.crt }}" + tls.key: "{{ .Values.secret.ca.key }}" +kind: Secret +metadata: + name: {{ .Values.st2.tls.certificate_issuer.name }}-tls + namespace: "{{ $.Release.Namespace }}" +type: kubernetes.io/tls +{{- end -}} diff --git a/templates/certificate.yaml b/templates/certificate.yaml new file mode 100644 index 00000000..f58ffd10 --- /dev/null +++ b/templates/certificate.yaml @@ -0,0 +1,28 @@ +{{- if or .Values.st2.tls.enabled .Values.mongodb.tls.enabled .Values.rabbitmq.tls.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.st2.tls.secretName }} + labels: + app: stackstorm + heritage: {{.Release.Service | quote}} + release: {{.Release.Name | quote}} + chart: {{ replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name }} +spec: + secretName: {{ .Values.st2.tls.secretName }} + dnsNames: + - "*.{{ $.Release.Namespace }}.svc.{{ $.Values.clusterDomain }}" +{{ include "stackstorm-ha.mongodb-nodes" $ | splitList "," | toYaml | indent 4 }} + ipAddresses: + - "127.0.0.1" + renewBefore: 360h # 15d + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 3072 + issuerRef: + name: {{ .Values.st2.tls.certificate_issuer.name }} + kind: Issuer + group: cert-manager.io +{{- end -}} diff --git a/templates/configmaps_st2-conf.yaml b/templates/configmaps_st2-conf.yaml index c2473b3a..6193167b 100644 --- a/templates/configmaps_st2-conf.yaml +++ b/templates/configmaps_st2-conf.yaml @@ -11,19 +11,49 @@ data: # The order of merging: st2.conf < st2.docker.conf < st2.user.conf st2.docker.conf: | [auth] + {{- if .Values.rabbitmq.tls.enabled }} + api_url = https://{{ .Release.Name }}-st2api:9111/ + {{- else }} api_url = http://{{ .Release.Name }}-st2api:9101/ + {{- end -}} + {{- if and .Values.st2.tls.enabled .Values.st2auth.tls.enabled }} + use_ssl = True + key = {{ .Values.st2.tls.mountPath }}/tls.key + cert = {{ .Values.st2.tls.mountPath }}/tls.crt + debug = False + enable = True + {{- else }} + use_ssl = False + {{- end }} + [system_user] user = {{ .Values.st2.system_user.user }} ssh_key_file = {{ tpl .Values.st2.system_user.ssh_key_file . }} {{- if index .Values "redis" "enabled" }} + [coordination] url = redis://{{ template "stackstorm-ha.redis-password" $ }}{{ template "stackstorm-ha.redis-nodes" $ }} {{- end }} {{- if index .Values "rabbitmq" "enabled" }} + [messaging] + {{- if .Values.rabbitmq.tls.enabled }} + url = amqp://{{ required "rabbitmq.auth.username is required!" (index .Values "rabbitmq" "auth" "username") }}:{{ required "rabbitmq.auth.password is required!" (index .Values "rabbitmq" "auth" "password") }}@{{ .Release.Name }}-rabbitmq:5671{{ required "rabbitmq.ingress.path is required!" (index .Values "rabbitmq" "ingress" "path") }} + {{- else }} url = amqp://{{ required "rabbitmq.auth.username is required!" (index .Values "rabbitmq" "auth" "username") }}:{{ required "rabbitmq.auth.password is required!" (index .Values "rabbitmq" "auth" "password") }}@{{ .Release.Name }}-rabbitmq:5672{{ required "rabbitmq.ingress.path is required!" (index .Values "rabbitmq" "ingress" "path") }} + {{- end -}} {{- end }} {{- if index .Values "mongodb" "enabled" }} + {{- if .Values.rabbitmq.tls.enabled }} + ssl = True + ssl_ca_certs = {{ .Values.st2.tls.mountPath }}/ca.crt + ssl_cert_reqs = optional + ssl_certfile = {{ .Values.st2.tls.mountPath }}/tls.crt + ssl_keyfile = {{ .Values.st2.tls.mountPath }}/tls.key + {{- else }} + ssl = False + {{- end }} + [database] {{- if index .Values "mongodb" "auth" "enabled" }} host = mongodb://{{ template "stackstorm-ha.mongodb-nodes" $ }}/{{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}?authSource={{ required "mongodb.auth.database is required!" (index .Values "mongodb" "auth" "database") }}&replicaSet={{ index .Values "mongodb" "replicaSetName" }} @@ -36,10 +66,21 @@ data: port = {{ index .Values "mongodb" "service" "port" }} {{- end }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} + {{- if .Values.mongodb.tls.enabled }} + ssl = True + ssl_ca_certs = {{ .Values.st2.tls.mountPath }}/ca.crt + ssl_cert_reqs = optional + ssl_certfile = {{ .Values.st2.tls.mountPath }}/tls.crt + ssl_keyfile = {{ .Values.st2.tls.mountPath }}/tls.key + {{- else }} + ssl = False + {{- end }} + [keyvalue] encryption_key_path = /etc/st2/keys/datastore_key.json {{- end }} {{- if .Values.st2.rbac.enabled }} + [rbac] enable = True backend = default diff --git a/templates/configmaps_st2-urls.yaml b/templates/configmaps_st2-urls.yaml index e02dccc1..53158c18 100644 --- a/templates/configmaps_st2-urls.yaml +++ b/templates/configmaps_st2-urls.yaml @@ -7,6 +7,18 @@ metadata: description: StackStorm service URLs, used across entire st2 cluster labels: {{- include "stackstorm-ha.labels" (list $ "st2") | nindent 4 }} data: + {{- if and .Values.st2.tls.enabled .Values.st2auth.tls.enabled }} + ST2_AUTH_URL: https://{{ .Release.Name }}-st2auth:9100/ + {{- else }} ST2_AUTH_URL: http://{{ .Release.Name }}-st2auth:9100/ + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + ST2_API_URL: https://{{ .Release.Name }}-st2api:9111/ + {{- else }} ST2_API_URL: http://{{ .Release.Name }}-st2api:9101/ + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + ST2_STREAM_URL: https://{{ .Release.Name }}-st2stream:9112/ + {{- else }} ST2_STREAM_URL: http://{{ .Release.Name }}-st2stream:9102/ + {{- end }} diff --git a/templates/deployments.yaml b/templates/deployments.yaml index 7cbceded..23390856 100644 --- a/templates/deployments.yaml +++ b/templates/deployments.yaml @@ -86,6 +86,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: htpasswd-vol mountPath: /etc/st2/htpasswd @@ -115,6 +116,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} - name: htpasswd-vol emptyDir: @@ -198,8 +200,11 @@ spec: {{- with .Values.securityContext }} securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{/* When using internal TLS don't expose API Port outside pod */}} + {{- if not (and .Values.st2.tls.enabled .Values.st2api.tls.enabled) }} ports: - containerPort: 9101 + {{- end }} # TODO: Add liveness/readiness probes (#3) #livenessProbe: #readinessProbe: @@ -214,6 +219,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -244,10 +250,45 @@ spec: {{- end }} resources: {{- toYaml .Values.st2api.resources | nindent 10 }} + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + # StackStorm doesn't provide TLS for the API endpoint. + # Therefore when using Internal TLS, we need use ghostunnel (https://github.com/ghostunnel/ghostunnel) to provide a TLS proxy layer + - name: tls-proxy + image: {{ .Values.ghostunnel.image.repository }}/{{ .Values.ghostunnel.image.name }}:{{ .Values.ghostunnel.image.tag }} + imagePullPolicy: {{ .Values.ghostunnel.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} + {{- end }} + args: + - server + - --listen=:9111 + - --target=127.0.0.1:9101 + - --key={{ .Values.st2.tls.mountPath }}/tls.key + - --cert={{ .Values.st2.tls.mountPath }}/tls.crt + - --disable-authentication + resources: + requests: + memory: 25Mi + cpu: 100m + limits: + memory: 50Mi + cpu: 200m + ports: + - containerPort: 9111 + name: https + protocol: TCP + volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 10 }} + {{- range .Values.st2api.extra_volumes }} + - name: {{ required "Each volume must have a 'name' in st2api.extra_volumes" .name }} + {{- tpl (required "Each volume must have a 'mount' definition in st2api.extra_volumes" .mount | toYaml) $ | nindent 12 }} + {{- end }} + {{- end }} {{- if .Values.st2api.serviceAccount.attach }} serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -331,8 +372,11 @@ spec: {{- with .Values.securityContext }} securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{/* When using internal TLS don't expose Stream Port outside pod */}} + {{- if not (and .Values.st2.tls.enabled .Values.st2stream.tls.enabled) }} ports: - containerPort: 9102 + {{- end }} # TODO: Add liveness/readiness probes (#3) #livenessProbe: #readinessProbe: @@ -347,6 +391,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2stream.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2stream.extra_volumes" .name }} @@ -368,10 +413,45 @@ spec: {{- end }} resources: {{- toYaml .Values.st2stream.resources | nindent 10 }} + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + # StackStorm doesn't provide TLS for the Stream endpoint. + # Therefore when using Internal TLS, we need use ghostunnel (https://github.com/ghostunnel/ghostunnel) to provide a TLS proxy layer + - name: tls-proxy + image: {{ .Values.ghostunnel.image.repository }}/{{ .Values.ghostunnel.image.name }}:{{ .Values.ghostunnel.image.tag }} + imagePullPolicy: {{ .Values.ghostunnel.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} + {{- end }} + args: + - server + - --listen=:9112 + - --target=127.0.0.1:9102 + - --key={{ .Values.st2.tls.mountPath }}/tls.key + - --cert={{ .Values.st2.tls.mountPath }}/tls.crt + - --disable-authentication + resources: + requests: + memory: 25Mi + cpu: 100m + limits: + memory: 50Mi + cpu: 200m + ports: + - containerPort: 9112 + name: https + protocol: TCP + volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} + {{- range .Values.st2stream.extra_volumes }} + - name: {{ required "Each volume must have a 'name' in st2stream.extra_volumes" .name }} + {{- tpl (required "Each volume must have a 'mount' definition in st2stream.extra_volumes" .mount | toYaml) $ | nindent 12 }} + {{- end }} + {{- end }} {{- if .Values.st2stream.serviceAccount.attach }} serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2stream.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2stream.extra_volumes" .name }} @@ -439,13 +519,15 @@ spec: securityContext: {{- toYaml . | nindent 10 }} {{- end }} ports: - - containerPort: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary 443 80 }} + {{- $https_port := include "stackstorm-ha.st2web.https_port" . }} + {{- $http_port := include "stackstorm-ha.st2web.http_port" . }} + - containerPort: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} # Probe to check if app is running. Failure will lead to a pod restart. livenessProbe: httpGet: - scheme: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary "HTTPS" "HTTP" }} + scheme: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}HTTPS{{- else -}}HTTP{{- end }} path: / - port: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary 443 80 }} + port: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} initialDelaySeconds: 1 # Probe to check if app is ready to serve traffic. Failure will lead to temp stop serving traffic. # TODO: Failing to add readinessProbe, since st2 requires authorization (401) and we don't have `/healthz` endpoints yet (https://github.com/StackStorm/st2/issues/4020) @@ -458,9 +540,18 @@ spec: # path: /api/ # port: 443 # initialDelaySeconds: 3 + env: {{- if .Values.st2web.env }} - env: {{- include "stackstorm-ha.customEnv" .Values.st2web | nindent 8 }} + {{- include "stackstorm-ha.customEnv" .Values.st2web | nindent 8 }} + {{- end }} + {{- if and .Values.st2.tls.enabled .Values.st2web.tls.enabled }} + - name: "ST2WEB_HTTPS" + value: "1" {{- end }} + - name: "ST2_PORT_HTTP" + value: "{{ $http_port }}" + - name: "ST2_PORT_HTTPS" + value: "{{ $https_port }}" envFrom: - configMapRef: name: {{ .Release.Name }}-st2-urls @@ -469,6 +560,7 @@ spec: {{- else }} volumeMounts: [] {{- end }} + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 10 }} {{- range .Values.st2web.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2web.extra_volumes" .name }} {{- tpl (required "Each volume must have a 'mount' definition in st2web.extra_volumes" .mount | toYaml) $ | nindent 12 }} @@ -502,6 +594,7 @@ spec: {{- else }} volumes: [] {{- end }} + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- range .Values.st2web.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2web.extra_volumes" .name }} {{- tpl (required "Each volume must have a 'volume' definition in st2web.extra_volumes" .volume | toYaml) $ | nindent 10 }} @@ -593,6 +686,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -618,6 +712,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -711,6 +806,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2timersengine.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2timersengine.extra_volumes" .name }} @@ -731,6 +827,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2timersengine.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2timersengine.extra_volumes" .name }} @@ -819,6 +916,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -844,6 +942,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -938,6 +1037,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol @@ -963,6 +1063,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -1055,6 +1156,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2notifier.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2notifier.extra_volumes" .name }} @@ -1075,6 +1177,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2notifier.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2notifier.extra_volumes" .name }} @@ -1242,6 +1345,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" $ | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" $ | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }} {{- if $some_sensors_per_pod }} @@ -1274,6 +1378,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" $ }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" $ | nindent 8 }} {{- if ne "disable" (default "" $.Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -1398,6 +1503,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: st2-ssh-key-vol mountPath: {{ tpl .Values.st2.system_user.ssh_key_file . | dir | dir }}/.ssh-key-vol/ @@ -1427,6 +1533,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: @@ -1529,6 +1636,7 @@ spec: - configMapRef: name: {{ .Release.Name }}-st2-urls volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- range .Values.st2garbagecollector.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2garbagecollector.extra_volumes" .name }} @@ -1549,6 +1657,7 @@ spec: serviceAccountName: {{ template "stackstorm-ha.serviceAccountName" . }} {{- end }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- range .Values.st2garbagecollector.extra_volumes }} - name: {{ required "Each volume must have a 'name' in st2garbagecollector.extra_volumes" .name }} @@ -1672,6 +1781,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- if .Values.st2.rbac.enabled }} - name: st2-rbac-roles-vol @@ -1712,6 +1822,7 @@ spec: resources: {{- toYaml .Values.st2client.resources | nindent 10 }} volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- if ne "disable" (default "" .Values.st2.datastore_crypto_key) }} - name: st2-encryption-key-vol secret: diff --git a/templates/jobs.yaml b/templates/jobs.yaml index 592936fb..3d7a89cb 100644 --- a/templates/jobs.yaml +++ b/templates/jobs.yaml @@ -51,6 +51,7 @@ spec: {{- end }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: st2-rbac-roles-vol mountPath: /opt/stackstorm/rbac/roles/ @@ -65,6 +66,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} - name: st2-rbac-roles-vol configMap: @@ -143,7 +145,11 @@ spec: - 'sh' - '-c' - > + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + until nc -z -w 2 {{ .Release.Name }}-st2api 9111 && echo st2api ready; + {{- else }} until nc -z -w 2 {{ .Release.Name }}-st2api 9101 && echo st2api ready; + {{- end }} do sleep 2; done # Sidecar container for generating st2client config with st2 username & password pair and sharing produced file with the main container @@ -163,6 +169,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ # `st2 login` doesn't exit on failure correctly, use old methods instead. See bug: https://github.com/StackStorm/st2/issues/4338 @@ -196,6 +203,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ - name: st2-apikeys-vol @@ -208,6 +216,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} - name: st2client-config-vol emptyDir: medium: Memory @@ -289,6 +298,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ # `st2 login` doesn't exit on failure correctly, use old methods instead. See bug: https://github.com/StackStorm/st2/issues/4338 @@ -323,6 +333,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ @@ -336,6 +347,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} - name: st2client-config-vol emptyDir: @@ -418,6 +430,7 @@ spec: {{- end }} command: {{- toYaml $.Values.jobs.preRegisterContentCommand | nindent 8 }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.overrides-config-mounts" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- include "stackstorm-ha.pack-configs-volume-mount" . | nindent 8 }} @@ -447,6 +460,7 @@ spec: {{- end }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.overrides-config-mounts" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" . | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts-for-register-job" . | nindent 8 }} @@ -458,6 +472,7 @@ spec: # TODO: Find out default resource limits for this specific service (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.overrides-configs" . | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" . | nindent 8 }} {{- include "stackstorm-ha.packs-volumes" . | nindent 8 }} @@ -567,6 +582,7 @@ spec: echo DONE volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts" $ | nindent 8 }} {{/* do not include the pack-configs-volume-mount helper here */}} - name: st2-pack-configs-vol @@ -575,6 +591,7 @@ spec: # TODO: Find out default resource limits for this specific job (#5) #resources: volumes: + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }} {{- if $.Values.st2.packs.volumes.configs }} {{/* do not include the pack-configs-volume helper here */}} @@ -659,6 +676,7 @@ spec: name: {{ . }} {{- end }} volumeMounts: + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} - name: st2client-config-vol mountPath: /root/.st2/ # `st2 login` doesn't exit on failure correctly, use old methods instead. See bug: https://github.com/StackStorm/st2/issues/4338 @@ -693,6 +711,7 @@ spec: volumeMounts: - name: st2client-config-vol mountPath: /root/.st2/ + {{- include "stackstorm-ha.internal_tls.cert_volume.mount" . | nindent 8 }} {{- include "stackstorm-ha.overrides-config-mounts" $ | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume-mounts" $ | nindent 8 }} {{- include "stackstorm-ha.packs-volume-mounts-for-register-job" $ | nindent 8 }} @@ -708,6 +727,7 @@ spec: - name: st2client-config-vol emptyDir: medium: Memory + {{- include "stackstorm-ha.internal_tls.cert_volume.volume" . | nindent 8 }} {{- include "stackstorm-ha.overrides-configs" $ | nindent 8 }} {{- include "stackstorm-ha.st2-config-volume" $ | nindent 8 }} {{- include "stackstorm-ha.packs-volumes" $ | nindent 8 }} diff --git a/templates/services.yaml b/templates/services.yaml index 6b0e385d..57ca35aa 100644 --- a/templates/services.yaml +++ b/templates/services.yaml @@ -48,7 +48,11 @@ spec: {{- end }} ports: - protocol: TCP + {{- if and .Values.st2.tls.enabled .Values.st2api.tls.enabled }} + port: 9111 + {{- else }} port: 9101 + {{- end }} --- kind: Service @@ -74,7 +78,11 @@ spec: {{- end }} ports: - protocol: TCP + {{- if and .Values.st2.tls.enabled .Values.st2stream.tls.enabled }} + port: 9112 + {{- else }} port: 9102 + {{- end }} --- kind: Service @@ -99,8 +107,12 @@ spec: {{- end }} {{- end }} ports: - - protocol: TCP - port: {{ eq (get .Values.st2web.env "ST2WEB_HTTPS" | toString) "1" | ternary 443 80 }} + {{- $https_port := include "stackstorm-ha.st2web.https_port" . }} + {{- $http_port := include "stackstorm-ha.st2web.http_port" . }} + - name: st2web + protocol: TCP + port: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} + targetPort: {{ if and .Values.st2.tls.enabled .Values.st2web.tls.enabled -}}{{ $https_port }}{{- else -}}{{ $http_port }}{{- end }} {{ if .Values.st2chatops.enabled -}} --- diff --git a/values.yaml b/values.yaml index 8f4d90fc..d0767e52 100644 --- a/values.yaml +++ b/values.yaml @@ -96,11 +96,11 @@ st2: [api] allow_origin = '*' # fixes no replicaset found bug; + [database] # Connection and server selection timeout (in ms). connection_timeout = 5000 - #Override Definitions can be added here. #https://docs.stackstorm.com/latest/packs.html#overriding-pack-defaults overrides: {} @@ -296,6 +296,15 @@ st2: # roles: # - "admin" + # Controls configuring TLS between internal inter-pod communications + tls: + enabled: true + secretName: "internal-tls" + mountPath: "/etc/ssl/internal" + certificate_issuer: + existing: false + name: stackstorm-issuer + ## ## Default SecurityContext for pods and containers. ## Overrides available for st2web, st2actionrunner, st2sensorcontainer, st2client pods, and custom packs images. @@ -380,7 +389,19 @@ st2web: securityContext: {} # NB: nginx requires some capabilities, drop ALL will cause issues. # mount extra volumes on the st2web pod(s) (primarily useful for k8s-provisioned secrets) ## Note that Helm templating is supported in 'mount' and 'volume' - extra_volumes: [] + extra_volumes: + - name: st2-cert + mount: + mountPath: "/etc/ssl/st2/" + readOnly: true + volume: + secret: + secretName: "{{ .Release.Name }}-stackstorm-tls" + items: + - key: tls.crt + path: st2.crt + - key: tls.key + path: st2.key # see examples under st2actionrunner.extra_volumes # User-defined st2web config with custom settings to replace default config.js # See https://github.com/StackStorm/st2web#connecting-to-st2-server for more info @@ -392,6 +413,9 @@ st2web: # see: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ postStartScript: "" preStopSleep: "10" + # Use TLS on HTTP connections to st2web (i.e. between pod and ingress) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2auth # Multiple st2auth processes can be behind a load balancer in an active-active configuration. @@ -432,6 +456,9 @@ st2auth: ## Note that Helm templating is supported in 'mount' and 'volume' extra_volumes: [] # see examples under st2actionrunner.extra_volumes + # Use TLS on HTTP connections to st2auth (i.e. between pod and ingress, and inter-pod) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2api # Multiple st2api process can be behind a load balancer in an active-active configuration. @@ -472,6 +499,9 @@ st2api: ## Note that Helm templating is supported in 'mount' and 'volume' extra_volumes: [] # see examples under st2actionrunner.extra_volumes + # Use TLS on HTTP connections to st2api (i.e. between pod and ingress, and inter-pod) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2stream # Multiple st2stream process can be behind a load balancer in an active-active configuration. @@ -512,6 +542,9 @@ st2stream: ## Note that Helm templating is supported in 'mount' and 'volume' extra_volumes: [] # see examples under st2actionrunner.extra_volumes + # Use TLS on HTTP connections to st2stream (i.e. between pod and ingress, and inter-pod) + tls: + enabled: true # https://docs.stackstorm.com/reference/ha.html#st2rulesengine # Multiple st2rulesengine processes can run in active-active with only connections to MongoDB and RabbitMQ. All these will share the TriggerInstance load and naturally pick up more work if one or more of the processes becomes unavailable. @@ -1050,6 +1083,13 @@ mongodb: arbiter: enabled: false resources: {} + tls: + enabled: true + replicaset: + existingSecrets: + - internal-tls + - internal-tls + - internal-tls ## ## RabbitMQ configuration (3rd party chart dependency) @@ -1067,6 +1107,10 @@ rabbitmq: forceBoot: true # Authentication Details auth: + tls: + enabled: true + existingSecret: "internal-tls" + # Fail over to username admin if LDAP is down: username: admin # TODO: Use default random 10 character password, but need to fetch this string for use by downstream services password: 9jS+w1u07NbHtZke1m+jW4Cj @@ -1097,7 +1141,9 @@ rabbitmq: # As RabbitMQ enabled prometheus operator monitoring by default, disable it for non-prometheus users metrics: enabled: false - + tls: + enabled: true + existingSecret: "internal-tls" ## ## Redis HA configuration (3rd party chart dependency) ## @@ -1133,6 +1179,13 @@ redis: usePassword: false metrics: enabled: false + tls: + enabled: true + authClients: false + certificatesSecret: internal-tls + certFilename: tls.crt + certKeyFilename: tls.key + certCAFilename: ca.crt ## ## Settings to be applied to all stackstorm-ha pods @@ -1168,3 +1221,13 @@ external-dns: aws: zoneType: "public" domainFilters: [] + +## +## Image details for ghostunnel +## +ghostunnel: + image: + name: ghostunnel + repository: ghostunnel + tag: v1.6.0 + pullPolicy: IfNotPresent