[helm] Initial Helm chart (#950)

* [helm-chart] WIP

* [helm-chart] WIP

* [helm-chart] DSS operational on Google and AWS

* Remove unused statements

* Add README and schema validation

* Add missing READMEs

* typo

* Clean up certificate for helm

* Update README

* Fix deprecated warning

* Keep backward compatibility

* Update documentation

* Update documentation

build/apply-certs.sh

@@ -48,11 +48,13 @@ kubectl create secret generic cockroachdb.client.root --namespace default --from
 if [[ $NAMESPACE != "default" ]]; then
   kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR"  --context "$CONTEXT"
 fi
-kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR"  --context "$CONTEXT"
+kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT"
 # The ca key is not needed for any typical operations, but might be required to sign new certificates.
 $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR"  --context "$CONTEXT"
 # The ca.crt is kept in it's own secret to more easily manage cert rotation and
-# adding other operators' certificates.
+# adding other operators' certificates. Note that, for the purpose of the migration to helm and
+# to comply with cockroach db standard configuration, ca.crt has been kept inside cockroach.* secrets.
+# This secret is kept for backward compatibility.
 kubectl create secret generic cockroachdb.ca.crt --namespace "$NAMESPACE" --from-file "$CA_CRT_DIR"  --context "$CONTEXT"
 kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR"  --context "$CONTEXT"
 

build/make-certs.py

@@ -138,7 +138,16 @@ def main():
         '*.cockroachdb',
         '*.cockroachdb.%s' % cr.namespace,
         'cockroachdb.%s' % cr.namespace,
-        '*.cockroachdb.%s.svc.cluster.local' % cr.namespace
+        '*.cockroachdb.%s.svc.cluster.local' % cr.namespace,
+        # New helm generated address
+        # Individual nodes
+        '*.dss-cockroachdb',
+        '*.dss-cockroachdb.%s' % cr.namespace,
+        '*.dss-cockroachdb.%s.svc.cluster.local' % cr.namespace,
+        # Internal load balancer
+        'dss-cockroachdb-public',
+        'dss-cockroachdb-public.%s' % cr.namespace,
+        'dss-cockroachdb-public.%s.svc.cluster.local' % cr.namespace,
     ])
 
     subprocess.check_call([
@@ -147,9 +156,6 @@ def main():
         '--ca-key', cr.ca_key_file]
         + node_addresses)
 
-    os.remove(os.path.join(cr.node_certs_dir, 'ca.crt'))
-    os.remove(os.path.join(cr.client_certs_dir, 'ca.crt'))
-
     print('Created new node certificate in {}'.format(cr.node_certs_dir))
 
 

deploy/README.md

@@ -1,11 +1,13 @@
 # DSS Deployment
 
-**Work in progress**
-
 This folder contains the increments toward the new deployment approach as described in [#874](https://github.com/interuss/dss/issues/874).
 
 The infrastructure folder contains the terraform modules to deploy the DSS to kubernetes clusters of various cloud providers:
 
 - Amazon Web Services: [terraform-aws-dss](./infrastructure/modules/terraform-aws-dss/README.md)
 - Google Cloud Engine: [terraform-google-dss](./infrastructure/modules/terraform-google-dss/README.md)
 
+The service folder contains the scripts required to deploy the DSS to a Kubernetes cluster:
+
+- Helm Charts: [services/helm-charts](./services/helm-charts)
+- Tanka: [../build/deploy/](../build/deploy)

deploy/infrastructure/README.md

@@ -0,0 +1,6 @@
+# DSS Deployment
+
+The `modules` directory contains the terraform modules required to deploy the DSS using cloud providers.
+
+- [terraform-aws-dss](./modules/terraform-aws-dss/README.md): Amazon Web Services deployment
+- [terraform-google-dss](./modules/terraform-google-dss/README.md): Google Cloud Engine deployment

deploy/services/README.md

@@ -0,0 +1,5 @@
+# Services
+
+This folder contains the tools to deploy a DSS to a Kubernetes cluster.
+
+- [helm-charts/dss](./helm-charts/dss/README.md): Helm chart to deploy the DSS.

deploy/services/helm-charts/dss/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deploy/services/helm-charts/dss/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+name: dss
+description: A Helm chart to deploy the InterUSS DSS to Kubernetes
+type: application
+version: 0.1.0
+appVersion: "snapshot"
+dependencies:
+  - name: cockroachdb
+    repository: https://charts.cockroachdb.com/
+    version: 10.0.7

deploy/services/helm-charts/dss/README.md

@@ -0,0 +1,23 @@
+# DSS Helm Chart
+This [Helm Chart](https://helm.sh/) deploys the DSS and cockroachdb kubernetes resources.
+
+## Requirements
+1. A Kubernetes cluster should be running and you should be properly authenticated.
+Requirements and instructions to create a new Kubernetes cluster can be found [here](../../../infrastructure/README.md).
+2. Create the certificates and apply them to the cluster using the instructions of [section 6 and 7](../../../../build/README.md)
+3. Install [Helm](https://helm.sh/) version 3.11.3 or higher
+
+## Usage
+1. Copy `values.example.yaml` to `values.dev.yaml` and edit it. See `values.schema.json` for schema definition. (Note that the key `cockroachdb` supports all values supported by the [`cockroachdb` Chart](https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb#configuration)).
+Note that values.yaml contains the default values and are always passed to helm.
+2. Validate the configuration: `helm lint -f values.dev.yaml .`
+3. Set a RELEASE_NAME to `dss`: `export RELEASE_NAME=dss`
+It is temporarily the only release name possible.
+4. Set the kube client context of your , example: `export KUBE_CONTEXT=gke_interuss-deploy-example_europe-west6-a_dss-dev-w6`
+5. Run `helm dep update --kube-context=$KUBE_CONTEXT`
+6. Install the chart: `helm install --kube-context=$KUBE_CONTEXT -f values.dev.yaml $RELEASE_NAME .`
+
+### Update the chart
+When changing the values in values.dev.yaml, values.yaml, the templates or upgrading the helm chart dependencies, changes can be applied to the cluster using the following command:
+
+1. Run `helm upgrade --kube-context=$KUBE_CONTEXT -f values.dev.yaml $RELEASE_NAME .`

deploy/services/helm-charts/dss/templates/_helpers.tpl

@@ -0,0 +1,26 @@
+{{- define "cockroachImage" -}}
+{{ (printf "%s:%s" $.Values.cockroachdb.image.repository $.Values.cockroachdb.image.tag) }}
+{{- end -}}
+
+{{- define "cockroachHost" -}}
+{{- printf "%s-public.default" $.Values.cockroachdb.fullnameOverride -}}
+{{- end -}}
+
+{{- define "init-container-wait-for-http" -}}
+- name: wait-for-{{.serviceName}}
+  image: alpine:3.17.3
+  command: [ 'sh', '-c', "until wget -nv {{.url}}; do echo waiting for {{.serviceName}}; sleep 2; done" ]
+{{- end -}}
+
+{{- define "init-container-wait-for-schema" -}}
+{{/*For some reason, calling the template cockroachImage fails here.*/}}
+- name: wait-for-schema-{{.schemaName}}
+  image: {{.cockroachImage}}
+  volumeMounts:
+    {{- include "ca-certs:volumeMount" . | nindent 4 }}
+    {{- include "client-certs:volumeMount" . | nindent 4 }}
+  command:
+    - sh
+    - -c
+    - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.cockroachHost}} --port \"26257\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}"
+{{- end -}}

deploy/services/helm-charts/dss/templates/_networking-aws.tpl

@@ -0,0 +1,26 @@
+{{- define "aws-lb-default-annotations" -}}
+service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+service.beta.kubernetes.io/aws-load-balancer-name: {{.name}}
+service.beta.kubernetes.io/aws-load-balancer-eip-allocations: {{.ip}}
+service.beta.kubernetes.io/aws-load-balancer-subnets: {{.subnet}}
+service.beta.kubernetes.io/aws-load-balancer-type: external
+{{- end -}}
+
+{{- define "aws-lb-crdb-annotations" -}}
+{{- include "aws-lb-default-annotations" . }}
+{{- end -}}
+
+{{- define "aws-lb-spec" -}}
+loadBalancerClass: service.k8s.aws/nlb
+{{- end -}}
+
+{{- define "aws-ingress-dss-gateway-annotations" -}}
+{{- include "aws-lb-default-annotations" . }}
+service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{.certName}}
+service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443"
+{{- end -}}
+
+{{- define "aws-ingress-spec" -}}
+loadBalancerClass: service.k8s.aws/nlb
+{{- end -}}

deploy/services/helm-charts/dss/templates/_networking-google.tpl

@@ -0,0 +1,15 @@
+{{- define "google-lb-crdb-annotations" -}}
+{{- end -}}
+
+{{- define "google-lb-spec" -}}
+loadBalancerIP: {{.ip}}
+{{- end -}}
+
+{{- define "google-ingress-dss-gateway-annotations" -}}
+kubernetes.io/ingress.allow-http: "false"
+kubernetes.io/ingress.global-static-ip-name: {{.ip}}
+networking.gke.io/managed-certificates: {{.certName}}
+{{- end -}}
+
+{{- define "google-ingress-spec" -}}
+{{- end -}}

deploy/services/helm-charts/dss/templates/_volumes.tpl

@@ -0,0 +1,38 @@
+{{- define "ca-certs:volume" -}}
+- name: ca-certs
+  secret:
+    defaultMode: 256
+    secretName: cockroachdb.ca.crt
+{{- end -}}
+{{- define "ca-certs:volumeMount" -}}
+- mountPath: /cockroach/cockroach-certs/ca.crt
+  name: ca-certs
+  subPath: ca.crt
+{{- end -}}
+
+{{- define "client-certs:volume" -}}
+- name: client-certs
+  secret:
+    defaultMode: 256
+    secretName: cockroachdb.client.root
+{{- end -}}
+{{- define "client-certs:volumeMount" -}}
+- mountPath: /cockroach/cockroach-certs/client.root.crt
+  name: client-certs
+  subPath: client.root.crt
+- mountPath: /cockroach/cockroach-certs/client.root.key
+  name: client-certs
+  subPath: client.root.key
+{{- end -}}
+
+
+{{- define "public-certs:volume" -}}
+- name: public-certs
+  secret:
+    defaultMode: 256
+    secretName: dss.public.certs
+{{- end -}}
+{{- define "public-certs:volumeMount" -}}
+- mountPath: /public-certs
+  name: public-certs
+{{- end -}}

deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml

@@ -0,0 +1,34 @@
+{{- $cloudProvider := $.Values.global.cloudProvider}}
+
+# Node Gateways
+{{- range $i, $lb := .Values.loadBalancers.cockroachdbNodes }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+    {{- include (printf "%s-lb-crdb-annotations" $cloudProvider)
+      (dict
+        "name" (printf "%s-%s" "cockroach-db-external-node" ( $i | toString) )
+        "ip" $lb.ip
+        "subnet" $lb.subnet
+        "cloudProvider" $cloudProvider
+      ) | nindent 4
+    }}
+  labels:
+    app: cockroachdb
+    name: cockroach-db-external-node-{{$i}}
+  name: cockroach-db-external-node-{{$i}}
+  namespace: default
+spec:
+  {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}}
+  ports:
+    - name: cockroach-db-external-node-{{$i}}
+      port: 26257
+      targetPort: 26257
+  publishNotReadyAddresses: true
+  selector:
+    statefulset.kubernetes.io/pod-name: {{$.Release.Name}}-cockroachdb-{{$i}}
+  type: LoadBalancer
+{{- end }}

deploy/services/helm-charts/dss/templates/dss-core-service.yaml

@@ -0,0 +1,81 @@
+{{- $dss := .Values.dss -}}
+{{- $cockroachImage := (include "cockroachImage" .) -}}
+{{- $cockroachHost :=  (include "cockroachHost" .) -}}
+{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}}
+{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}}
+{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}}
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    name: {{.Release.Name}}-core-service
+  name: {{.Release.Name}}-core-service
+spec:
+  minReadySeconds: 30
+  replicas: 3
+  selector:
+    matchLabels:
+      app: {{.Release.Name}}-core-service
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{.Release.Name}}-core-service
+    spec:
+      initContainers:
+        {{- $waitForCockroachDB | nindent 8 }}
+        {{- $waitForRIDSchema | nindent 8 }}
+        {{- $waitForSCDSchema | nindent 8 }}
+      containers:
+        - args:
+            - --accepted_jwt_audiences={{$dss.conf.hostname}}
+            - --addr=:8080
+            - --cockroach_host={{$cockroachHost}}
+            - --cockroach_port=26257
+            - --cockroach_ssl_dir=/cockroach/cockroach-certs
+            - --cockroach_ssl_mode=verify-full
+            - --cockroach_user=root
+            - --dump_requests=true
+            - --enable_scd={{$dss.enableScd | default true}}
+            - --garbage_collector_spec=@every 30m
+            - --gcp_prof_service_name=
+            {{- if $dss.conf.jwksEndpoint }}
+            - --jwks_endpoint={{ $dss.jwksEndpoint }}
+            {{- end }}
+            {{- if $dss.conf.jwksKeyIds }}
+            - --jwks_key_ids={{ $dss.jwksKeyIds | join "," }}
+            {{- end }}
+            - --locality={{ .Values.cockroachdb.conf.locality }}
+            {{- if $dss.conf.pubKeys}}
+            - --public_key_files={{ $dss.conf.pubKeys | join "," }}
+            {{- end }}
+          command:
+            - core-service
+          image: {{ $dss.image }}
+          imagePullPolicy: Always
+          name: core-service
+          ports:
+            - containerPort: 8080
+              name: http
+          readinessProbe:
+            httpGet:
+              path: /healthy
+              port: 8080
+          stdin: false
+          tty: false
+          volumeMounts:
+            {{- include "ca-certs:volumeMount" . | nindent 12 }}
+            {{- include "client-certs:volumeMount" . | nindent 12 }}
+            {{- include "public-certs:volumeMount" . | nindent 12 }}
+      imagePullSecrets: []
+      terminationGracePeriodSeconds: 30
+      volumes:
+        {{- include "ca-certs:volume" . | nindent 8 }}
+        {{- include "client-certs:volume" . | nindent 8 }}
+        {{- include "public-certs:volume" . | nindent 8 }}

deploy/services/helm-charts/dss/templates/dss-ingress-aws.yaml

@@ -0,0 +1,37 @@
+{{- $cloudProvider := $.Values.global.cloudProvider}}
+{{- if eq $cloudProvider "aws" }}
+{{/*
+AWS application load balancer Ingress do not support elastic ip assignment yet. Therefore, the
+ingress is replaced by a network load balancer (Kubernetes Service of type Load Balancer)
+*/}}
+{{- with $.Values.loadBalancers.dssGateway }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    {{- include (printf "%s-ingress-dss-gateway-annotations" $cloudProvider)
+      (merge .
+        (dict
+          "name" "dss-gateway-external"
+          "cloudProvider" $cloudProvider
+        )
+      ) | nindent 4
+    }}
+  labels:
+    app: {{$.Release.Name}}-core-service
+    name: {{$.Release.Name}}-dss-gateway
+  name: {{$.Release.Name}}-dss-gateway
+  namespace: default
+spec:
+  {{- include (printf "%s-ingress-spec" $cloudProvider) . | nindent 2 }}
+  ports:
+    - name: http
+      port: 443
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: {{$.Release.Name}}-core-service
+  type: LoadBalancer
+{{- end }}
+{{- end }}