From 26d1c8f1bc027e85cdc6e5759ef8c6017ee908a2 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 14 Apr 2023 14:16:48 +0200 Subject: [PATCH 01/13] [helm-chart] WIP --- deploy/services/helm-charts/dss/.helmignore | 23 ++++ deploy/services/helm-charts/dss/Chart.yaml | 11 ++ .../dss/templates/_aws_networking.tpl | 33 ++++++ .../helm-charts/dss/templates/_helpers.tpl | 26 ++++ .../helm-charts/dss/templates/_volumes.tpl | 38 ++++++ .../dss/templates/dss-core-service.yaml | 82 +++++++++++++ .../dss/templates/loadbalancers.yaml | 111 ++++++++++++++++++ .../dss/templates/schema-manager.yaml | 53 +++++++++ 8 files changed, 377 insertions(+) create mode 100644 deploy/services/helm-charts/dss/.helmignore create mode 100644 deploy/services/helm-charts/dss/Chart.yaml create mode 100644 deploy/services/helm-charts/dss/templates/_aws_networking.tpl create mode 100644 deploy/services/helm-charts/dss/templates/_helpers.tpl create mode 100644 deploy/services/helm-charts/dss/templates/_volumes.tpl create mode 100644 deploy/services/helm-charts/dss/templates/dss-core-service.yaml create mode 100644 deploy/services/helm-charts/dss/templates/loadbalancers.yaml create mode 100644 deploy/services/helm-charts/dss/templates/schema-manager.yaml diff --git a/deploy/services/helm-charts/dss/.helmignore b/deploy/services/helm-charts/dss/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/services/helm-charts/dss/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/services/helm-charts/dss/Chart.yaml b/deploy/services/helm-charts/dss/Chart.yaml new file mode 100644 index 000000000..f9c061f76 --- /dev/null +++ b/deploy/services/helm-charts/dss/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: dss +description: A Helm chart to deploy the InterUSS DSS to Kubernetes +type: application +version: 0.1.0 +appVersion: "snapshot" +dependencies: + - name: cockroachdb + + repository: file://../../../../../helm-charts/build/artifacts/cockroachdb-10.0.7.tgz #https://charts.cockroachdb.com/ + version: 10.0.7 diff --git a/deploy/services/helm-charts/dss/templates/_aws_networking.tpl b/deploy/services/helm-charts/dss/templates/_aws_networking.tpl new file mode 100644 index 000000000..418d2740c --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/_aws_networking.tpl @@ -0,0 +1,33 @@ + +{{/* Annotations */}} + +{{/* AWS */}} +{{- define "aws-lb-default-annotations" -}} +service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing +service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip +service.beta.kubernetes.io/aws-load-balancer-name: {{.name}} +service.beta.kubernetes.io/aws-load-balancer-eip-allocations: {{.ip}} +service.beta.kubernetes.io/aws-load-balancer-subnets: {{.subnet}} +service.beta.kubernetes.io/aws-load-balancer-type: external +{{- end -}} + +{{- define "aws-lb-crdb-annotations" -}} +{{- include "aws-lb-default-annotations" . }} +{{- end -}} + +{{- define "aws-lb-dss-gateway-annotations" -}} +{{- include "aws-lb-default-annotations" . }} +service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{.certName}} +service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" +{{- end -}} + +{{/* Google */}} + +{{/* Spec */}} +{{- define "aws-lb-spec" -}} +loadBalancerClass: service.k8s.aws/nlb +{{- end -}} + +{{- define "google-lb-spec" -}} +loadBalancerIP: {{.ip}} +{{- end -}} \ No newline at end of file diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl new file mode 100644 index 000000000..f7005f370 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{- define "cockroachImage" -}} +{{ (printf "%s:%s" $.Values.cockroachdb.image.repository $.Values.cockroachdb.image.tag) }} +{{- end -}} + +{{- define "cockroachHost" -}} +{{- printf "%s-public.default" $.Values.cockroachdb.fullnameOverride -}} +{{- end -}} + +{{- define "init-container-wait-for-http" -}} +- name: wait-for-{{.serviceName}} + image: alpine:3.17.3 + command: [ 'sh', '-c', "until wget -nv {{.url}}; do echo waiting for {{.serviceName}}; sleep 2; done" ] +{{- end -}} + +{{- define "init-container-wait-for-schema" -}} +{{/*For some reason, calling the template cockroachImage fails here.*/}} +- name: wait-for-schema-{{.schemaName}} + image: {{.cockroachImage}} + volumeMounts: + {{- include "ca-certs:volumeMount" . | nindent 4 }} + {{- include "client-certs:volumeMount" . | nindent 4 }} + command: + - sh + - -c + - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.cockroachHost}} --port \"26257\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" +{{- end -}} \ No newline at end of file diff --git a/deploy/services/helm-charts/dss/templates/_volumes.tpl b/deploy/services/helm-charts/dss/templates/_volumes.tpl new file mode 100644 index 000000000..e21dc30e4 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/_volumes.tpl @@ -0,0 +1,38 @@ +{{- define "ca-certs:volume" -}} +- name: ca-certs + secret: + defaultMode: 256 + secretName: cockroachdb.ca.crt +{{- end -}} +{{- define "ca-certs:volumeMount" -}} +- mountPath: /cockroach/cockroach-certs/ca.crt + name: ca-certs + subPath: ca.crt +{{- end -}} + +{{- define "client-certs:volume" -}} +- name: client-certs + secret: + defaultMode: 256 + secretName: cockroachdb.client.root +{{- end -}} +{{- define "client-certs:volumeMount" -}} +- mountPath: /cockroach/cockroach-certs/client.root.crt + name: client-certs + subPath: client.root.crt +- mountPath: /cockroach/cockroach-certs/client.root.key + name: client-certs + subPath: client.root.key +{{- end -}} + + +{{- define "public-certs:volume" -}} +- name: public-certs + secret: + defaultMode: 256 + secretName: dss.public.certs +{{- end -}} +{{- define "public-certs:volumeMount" -}} +- mountPath: /public-certs + name: public-certs +{{- end -}} \ No newline at end of file diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml new file mode 100644 index 000000000..f1238aacf --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -0,0 +1,82 @@ +{{- $dss := .Values.dss -}} +{{- $cockroachImage := (include "cockroachImage" .) -}} +{{- $cockroachHost := (include "cockroachHost" .) -}} +{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}} +{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}} +{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + name: {{.Release.Name}}-core-service + name: core-service + namespace: default +spec: + minReadySeconds: 30 + replicas: 3 + selector: + matchLabels: + app: {{.Release.Name}}-core-service + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: {{.Release.Name}}-core-service + spec: + initContainers: + {{- $waitForCockroachDB | nindent 8 }} + {{- $waitForRIDSchema | nindent 8 }} + {{- $waitForSCDSchema | nindent 8 }} + containers: + - args: + - --accepted_jwt_audiences={{$dss.conf.hostname}} + - --addr=:8080 + - --cockroach_host={{$cockroachHost}} + - --cockroach_port=26257 + - --cockroach_ssl_dir=/cockroach/cockroach-certs + - --cockroach_ssl_mode=verify-full + - --cockroach_user=root + - --dump_requests=true + - --enable_scd=true + - --garbage_collector_spec=@every 30m + - --gcp_prof_service_name= + {{- if $dss.conf.jwksEndpoint }} + - --jwks_endpoint={{ $dss.jwksEndpoint }} + {{- end }} + {{- if $dss.conf.jwksKeyIds }} + - --jwks_key_ids={{ $dss.jwksKeyIds | join "," }} + {{- end }} + - --locality={{ .Values.cockroachdb.conf.locality }} + {{- if $dss.conf.pubKeys}} + - --public_key_files={{ $dss.conf.pubKeys | join "," }} + {{- end }} + command: + - core-service + image: {{ $dss.image }} + imagePullPolicy: Always + name: core-service + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthy + port: 8080 + stdin: false + tty: false + volumeMounts: + {{- include "ca-certs:volumeMount" . | nindent 12 }} + {{- include "client-certs:volumeMount" . | nindent 12 }} + {{- include "public-certs:volumeMount" . | nindent 12 }} + imagePullSecrets: [] + terminationGracePeriodSeconds: 30 + volumes: + {{- include "ca-certs:volume" . | nindent 8 }} + {{- include "client-certs:volume" . | nindent 8 }} + {{- include "public-certs:volume" . | nindent 8 }} diff --git a/deploy/services/helm-charts/dss/templates/loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/loadbalancers.yaml new file mode 100644 index 000000000..e5927e59b --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/loadbalancers.yaml @@ -0,0 +1,111 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} + +# Node Gateways +{{- range $i, $lb := .Values.loadBalancers.cockroachdbNodes }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) + (dict + "name" (printf "%s-%s" "cockroach-db-external-node" ( $i | toString) ) + "ip" $lb.ip + "subnet" $lb.subnet + "cloudProvider" $cloudProvider + ) | nindent 4 + }} + labels: + app: cockroachdb + name: cockroach-db-external-node-{{$i}} + name: cockroach-db-external-node-{{$i}} + namespace: default +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} + ports: + - name: cockroach-db-external-node-{{$i}} + port: 26257 + targetPort: 26257 + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: {{$.Release.Name}}-cockroachdb-{{$i}} + type: LoadBalancer +{{- end }} + +# DSS HTTPS Gateway +{{- with $.Values.loadBalancers.dssGateway }} +{{ if eq $cloudProvider "aws" }} +{{/* +AWS application load balancer Ingress do not support elastic ip assignment yet. Therefore, the +ingress is replaced by a network load balancer (Kubernetes Service of type Load Balancer) +*/}} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- include (printf "%s-lb-dss-gateway-annotations" $cloudProvider) + (merge . + (dict + "name" "dss-gateway-external" + "cloudProvider" $cloudProvider + ) + ) | nindent 4 + }} + labels: + app: core-service + name: {{$.Release.Name}}-dss-gateway + name: {{$.Release.Name}}-dss-gateway + namespace: default +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) . | nindent 2 }} + ports: + - name: http + port: 443 + protocol: TCP + targetPort: 8080 + selector: + app: core-service + type: LoadBalancer +{{- else -}} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{$.Release.Name}}-core-service + name: {{$.Release.Name}}-core-service + name: {{$.Release.Name}}-core-service +spec: + ports: + - name: core-service + port: 8080 + targetPort: 8080 + selector: + app: {{$.Release.Name}}-core-service + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.allow-http: "false" + kubernetes.io/ingress.global-static-ip-name: dss-dev-w1a-ip-gateway + networking.gke.io/managed-certificates: https-certificate + labels: + name: {{$.Release.Name}}-core-service-https-ingress + name: {{$.Release.Name}}-core-service-https-ingress +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{$.Release.Name}}-core-service + port: + number: 8080 +{{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml new file mode 100644 index 000000000..d50290c82 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -0,0 +1,53 @@ +{{- $image := .Values.dss.image }} +{{- $cockroachHost := (include "cockroachHost" .) -}} +{{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}} +{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}} + +{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.1.0" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + name: {{$service}}-schema-manager-{{$jobVersion}} + name: {{$service}}-schema-manager-{{$jobVersion}} + namespace: default +spec: + completions: 1 + parallelism: 1 +{{/* TODO: use selector instead of incrementing jobVersion in name to mitigate immutable template*/}} + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + name: {{$service}}-schema-manager-{{$jobVersion}} + spec: + initContainers: + {{- $waitForCockroachDB | nindent 8 }} + containers: + - args: + - --cockroach_host={{$cockroachHost}} + - --cockroach_port=26257 + - --cockroach_ssl_dir=/cockroach/cockroach-certs + - --cockroach_ssl_mode=verify-full + - --cockroach_user=root + - --db_version={{$schemaVersion}} + - --schemas_dir=/db-schemas/{{$service}} + command: + - db-manager + image: {{$image}} + imagePullPolicy: IfNotPresent + name: {{$service}}-schema-manager-{{$jobVersion}} + stdin: false + tty: false + volumeMounts: + {{- include "ca-certs:volumeMount" . | nindent 12 }} + {{- include "client-certs:volumeMount" . | nindent 12 }} + imagePullSecrets: [] + restartPolicy: OnFailure + terminationGracePeriodSeconds: 30 + volumes: + {{- include "ca-certs:volume" . | nindent 8 }} + {{- include "client-certs:volume" . | nindent 8 }} +{{- end -}} \ No newline at end of file From b09a163e09c2126bcacb23ae2082db0a3dbced15 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 14 Apr 2023 15:09:12 +0200 Subject: [PATCH 02/13] [helm-chart] WIP --- build/apply-certs.sh | 6 +- build/make-certs.py | 11 +- ...aws_networking.tpl => _networking-aws.tpl} | 19 +-- .../dss/templates/_networking-google.tpl | 15 +++ .../templates/cockroachdb-loadbalancers.yaml | 34 ++++++ .../dss/templates/dss-core-service.yaml | 3 +- .../dss/templates/dss-ingress-aws.yaml | 37 ++++++ .../dss/templates/dss-ingress-google.yaml | 60 ++++++++++ .../dss/templates/loadbalancers.yaml | 111 ------------------ 9 files changed, 168 insertions(+), 128 deletions(-) rename deploy/services/helm-charts/dss/templates/{_aws_networking.tpl => _networking-aws.tpl} (82%) create mode 100644 deploy/services/helm-charts/dss/templates/_networking-google.tpl create mode 100644 deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml create mode 100644 deploy/services/helm-charts/dss/templates/dss-ingress-aws.yaml create mode 100644 deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml delete mode 100644 deploy/services/helm-charts/dss/templates/loadbalancers.yaml diff --git a/build/apply-certs.sh b/build/apply-certs.sh index b2074ab51..e4fd0427b 100755 --- a/build/apply-certs.sh +++ b/build/apply-certs.sh @@ -37,25 +37,29 @@ UPLOAD_CA_KEY=true # Delete previous secrets in case they have changed. kubectl create namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret cockroachdb.client.root --namespace default --context "$CONTEXT" || true kubectl delete secret cockroachdb.client.root --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret cockroachdb.node --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret cockroachdb.ca.crt --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret cockroachdb.ca.key --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret cockroachdb.tls --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl create secret generic cockroachdb.client.root --namespace default --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" if [[ $NAMESPACE != "default" ]]; then kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" fi kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" + # The ca key is not needed for any typical operations, but might be required to sign new certificates. $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR" --context "$CONTEXT" # The ca.crt is kept in it's own secret to more easily manage cert rotation and # adding other operators' certificates. kubectl create secret generic cockroachdb.ca.crt --namespace "$NAMESPACE" --from-file "$CA_CRT_DIR" --context "$CONTEXT" +kubectl create secret tls cockroachdb.tls --namespace "$NAMESPACE" --cert="$CA_CRT_DIR/ca.crt" --key="$CA_KEY_DIR/ca.key" --context "$CONTEXT" kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" +kubectl create secret tls cockroachdb.tls --namespace "$NAMESPACE" --cert="$CA_CRT_DIR/ca.crt" --key="$CA_KEY_DIR/ca.key" --context "$CONTEXT" + echo '=========================================================================' echo '= Secrets uploaded successfully. =' echo '=========================================================================' diff --git a/build/make-certs.py b/build/make-certs.py index 0d25f4d46..7aa58d555 100755 --- a/build/make-certs.py +++ b/build/make-certs.py @@ -138,7 +138,16 @@ def main(): '*.cockroachdb', '*.cockroachdb.%s' % cr.namespace, 'cockroachdb.%s' % cr.namespace, - '*.cockroachdb.%s.svc.cluster.local' % cr.namespace + '*.cockroachdb.%s.svc.cluster.local' % cr.namespace, + # New helm generated address + # Individual nodes + '*.dss-cockroachdb', + '*.dss-cockroachdb.%s' % cr.namespace, + '*.dss-cockroachdb.%s.svc.cluster.local' % cr.namespace, + # Internal load balancer + 'dss-cockroachdb-public', + 'dss-cockroachdb-public.%s' % cr.namespace, + 'dss-cockroachdb-public.%s.svc.cluster.local' % cr.namespace, ]) subprocess.check_call([ diff --git a/deploy/services/helm-charts/dss/templates/_aws_networking.tpl b/deploy/services/helm-charts/dss/templates/_networking-aws.tpl similarity index 82% rename from deploy/services/helm-charts/dss/templates/_aws_networking.tpl rename to deploy/services/helm-charts/dss/templates/_networking-aws.tpl index 418d2740c..ed7b3bdbb 100644 --- a/deploy/services/helm-charts/dss/templates/_aws_networking.tpl +++ b/deploy/services/helm-charts/dss/templates/_networking-aws.tpl @@ -1,7 +1,3 @@ - -{{/* Annotations */}} - -{{/* AWS */}} {{- define "aws-lb-default-annotations" -}} service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip @@ -15,19 +11,16 @@ service.beta.kubernetes.io/aws-load-balancer-type: external {{- include "aws-lb-default-annotations" . }} {{- end -}} -{{- define "aws-lb-dss-gateway-annotations" -}} +{{- define "aws-lb-spec" -}} +loadBalancerClass: service.k8s.aws/nlb +{{- end -}} + +{{- define "aws-ingress-dss-gateway-annotations" -}} {{- include "aws-lb-default-annotations" . }} service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{.certName}} service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" {{- end -}} -{{/* Google */}} - -{{/* Spec */}} -{{- define "aws-lb-spec" -}} +{{- define "aws-ingress-spec" -}} loadBalancerClass: service.k8s.aws/nlb -{{- end -}} - -{{- define "google-lb-spec" -}} -loadBalancerIP: {{.ip}} {{- end -}} \ No newline at end of file diff --git a/deploy/services/helm-charts/dss/templates/_networking-google.tpl b/deploy/services/helm-charts/dss/templates/_networking-google.tpl new file mode 100644 index 000000000..4b476efd5 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/_networking-google.tpl @@ -0,0 +1,15 @@ +{{- define "google-lb-crdb-annotations" -}} +{{- end -}} + +{{- define "google-lb-spec" -}} +loadBalancerIP: {{.ip}} +{{- end -}} + +{{- define "google-ingress-dss-gateway-annotations" -}} +kubernetes.io/ingress.allow-http: "false" +kubernetes.io/ingress.global-static-ip-name: {{.ip}} +networking.gke.io/managed-certificates: {{.certName}} +{{- end -}} + +{{- define "google-ingress-spec" -}} +{{- end -}} \ No newline at end of file diff --git a/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml new file mode 100644 index 000000000..19e002bca --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml @@ -0,0 +1,34 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} + +# Node Gateways +{{- range $i, $lb := .Values.loadBalancers.cockroachdbNodes }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) + (dict + "name" (printf "%s-%s" "cockroach-db-external-node" ( $i | toString) ) + "ip" $lb.ip + "subnet" $lb.subnet + "cloudProvider" $cloudProvider + ) | nindent 4 + }} + labels: + app: cockroachdb + name: cockroach-db-external-node-{{$i}} + name: cockroach-db-external-node-{{$i}} + namespace: default +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} + ports: + - name: cockroach-db-external-node-{{$i}} + port: 26257 + targetPort: 26257 + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: {{$.Release.Name}}-cockroachdb-{{$i}} + type: LoadBalancer +{{- end }} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index f1238aacf..194858c24 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -11,8 +11,7 @@ kind: Deployment metadata: labels: name: {{.Release.Name}}-core-service - name: core-service - namespace: default + name: {{.Release.Name}}-core-service spec: minReadySeconds: 30 replicas: 3 diff --git a/deploy/services/helm-charts/dss/templates/dss-ingress-aws.yaml b/deploy/services/helm-charts/dss/templates/dss-ingress-aws.yaml new file mode 100644 index 000000000..efac60b46 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/dss-ingress-aws.yaml @@ -0,0 +1,37 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} +{{- if eq $cloudProvider "aws" }} +{{/* +AWS application load balancer Ingress do not support elastic ip assignment yet. Therefore, the +ingress is replaced by a network load balancer (Kubernetes Service of type Load Balancer) +*/}} +{{- with $.Values.loadBalancers.dssGateway }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- include (printf "%s-ingress-dss-gateway-annotations" $cloudProvider) + (merge . + (dict + "name" "dss-gateway-external" + "cloudProvider" $cloudProvider + ) + ) | nindent 4 + }} + labels: + app: {{$.Release.Name}}-core-service + name: {{$.Release.Name}}-dss-gateway + name: {{$.Release.Name}}-dss-gateway + namespace: default +spec: + {{- include (printf "%s-ingress-spec" $cloudProvider) . | nindent 2 }} + ports: + - name: http + port: 443 + protocol: TCP + targetPort: 8080 + selector: + app: {{$.Release.Name}}-core-service + type: LoadBalancer +{{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml b/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml new file mode 100644 index 000000000..0f2cab4f4 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml @@ -0,0 +1,60 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} +{{- $dssHostname := $.Values.dss.conf.hostname }} +{{- if eq $cloudProvider "google" }} +# DSS HTTPS Gateway +{{- with $.Values.loadBalancers.dssGateway }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{$.Release.Name}}-core-service + name: {{$.Release.Name}}-core-service + name: {{$.Release.Name}}-core-service +spec: + ports: + - name: core-service + port: 8080 + targetPort: 8080 + selector: + app: {{$.Release.Name}}-core-service + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + {{- include (printf "%s-ingress-dss-gateway-annotations" $cloudProvider) + (dict + "certName" (printf "%s-core-service-https-certificate" $.Release.Name) + "ip" .ip + ) | nindent 4 + }} + labels: + name: {{$.Release.Name}}-core-service-https-ingress + name: {{$.Release.Name}}-core-service-https-ingress +spec: + {{- include (printf "%s-ingress-spec" $cloudProvider) . | nindent 2 }} + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{$.Release.Name}}-core-service + port: + number: 8080 +--- +apiVersion: networking.gke.io/v1beta1 +kind: ManagedCertificate +metadata: + labels: + name: {{$.Release.Name}}-core-service-https-certificate + name: {{$.Release.Name}}-core-service-https-certificate +spec: + domains: + - {{$dssHostname}} + +{{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/templates/loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/loadbalancers.yaml deleted file mode 100644 index e5927e59b..000000000 --- a/deploy/services/helm-charts/dss/templates/loadbalancers.yaml +++ /dev/null @@ -1,111 +0,0 @@ -{{- $cloudProvider := $.Values.global.cloudProvider}} - -# Node Gateways -{{- range $i, $lb := .Values.loadBalancers.cockroachdbNodes }} ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" - {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) - (dict - "name" (printf "%s-%s" "cockroach-db-external-node" ( $i | toString) ) - "ip" $lb.ip - "subnet" $lb.subnet - "cloudProvider" $cloudProvider - ) | nindent 4 - }} - labels: - app: cockroachdb - name: cockroach-db-external-node-{{$i}} - name: cockroach-db-external-node-{{$i}} - namespace: default -spec: - {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} - ports: - - name: cockroach-db-external-node-{{$i}} - port: 26257 - targetPort: 26257 - publishNotReadyAddresses: true - selector: - statefulset.kubernetes.io/pod-name: {{$.Release.Name}}-cockroachdb-{{$i}} - type: LoadBalancer -{{- end }} - -# DSS HTTPS Gateway -{{- with $.Values.loadBalancers.dssGateway }} -{{ if eq $cloudProvider "aws" }} -{{/* -AWS application load balancer Ingress do not support elastic ip assignment yet. Therefore, the -ingress is replaced by a network load balancer (Kubernetes Service of type Load Balancer) -*/}} ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - {{- include (printf "%s-lb-dss-gateway-annotations" $cloudProvider) - (merge . - (dict - "name" "dss-gateway-external" - "cloudProvider" $cloudProvider - ) - ) | nindent 4 - }} - labels: - app: core-service - name: {{$.Release.Name}}-dss-gateway - name: {{$.Release.Name}}-dss-gateway - namespace: default -spec: - {{- include (printf "%s-lb-spec" $cloudProvider) . | nindent 2 }} - ports: - - name: http - port: 443 - protocol: TCP - targetPort: 8080 - selector: - app: core-service - type: LoadBalancer -{{- else -}} ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: {{$.Release.Name}}-core-service - name: {{$.Release.Name}}-core-service - name: {{$.Release.Name}}-core-service -spec: - ports: - - name: core-service - port: 8080 - targetPort: 8080 - selector: - app: {{$.Release.Name}}-core-service - type: ClusterIP ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - kubernetes.io/ingress.allow-http: "false" - kubernetes.io/ingress.global-static-ip-name: dss-dev-w1a-ip-gateway - networking.gke.io/managed-certificates: https-certificate - labels: - name: {{$.Release.Name}}-core-service-https-ingress - name: {{$.Release.Name}}-core-service-https-ingress -spec: - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{$.Release.Name}}-core-service - port: - number: 8080 -{{- end }} -{{- end }} From 4c3d44542f551feb63a2ca19107f03d9cc7b0d74 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 14 Apr 2023 16:19:42 +0200 Subject: [PATCH 03/13] [helm-chart] DSS operational on Google and AWS --- deploy/services/helm-charts/dss/Chart.yaml | 3 +-- .../services/helm-charts/dss/templates/dss-ingress-google.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/deploy/services/helm-charts/dss/Chart.yaml b/deploy/services/helm-charts/dss/Chart.yaml index f9c061f76..e3043c1d2 100644 --- a/deploy/services/helm-charts/dss/Chart.yaml +++ b/deploy/services/helm-charts/dss/Chart.yaml @@ -6,6 +6,5 @@ version: 0.1.0 appVersion: "snapshot" dependencies: - name: cockroachdb - - repository: file://../../../../../helm-charts/build/artifacts/cockroachdb-10.0.7.tgz #https://charts.cockroachdb.com/ + repository: https://charts.cockroachdb.com/ version: 10.0.7 diff --git a/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml b/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml index 0f2cab4f4..9f64165bc 100644 --- a/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml @@ -13,7 +13,7 @@ metadata: name: {{$.Release.Name}}-core-service spec: ports: - - name: core-service + - name: {{$.Release.Name}}-core-service port: 8080 targetPort: 8080 selector: From 8f9f2ef44dcb432f59eacca685c1863e62e25a4e Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Tue, 25 Apr 2023 08:00:24 +0200 Subject: [PATCH 04/13] Remove unused statements --- build/apply-certs.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build/apply-certs.sh b/build/apply-certs.sh index e4fd0427b..b2074ab51 100755 --- a/build/apply-certs.sh +++ b/build/apply-certs.sh @@ -37,29 +37,25 @@ UPLOAD_CA_KEY=true # Delete previous secrets in case they have changed. kubectl create namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret cockroachdb.client.root --namespace default --context "$CONTEXT" || true kubectl delete secret cockroachdb.client.root --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret cockroachdb.node --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret cockroachdb.ca.crt --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret cockroachdb.ca.key --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret cockroachdb.tls --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl create secret generic cockroachdb.client.root --namespace default --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" if [[ $NAMESPACE != "default" ]]; then kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" fi kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" - # The ca key is not needed for any typical operations, but might be required to sign new certificates. $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR" --context "$CONTEXT" # The ca.crt is kept in it's own secret to more easily manage cert rotation and # adding other operators' certificates. kubectl create secret generic cockroachdb.ca.crt --namespace "$NAMESPACE" --from-file "$CA_CRT_DIR" --context "$CONTEXT" -kubectl create secret tls cockroachdb.tls --namespace "$NAMESPACE" --cert="$CA_CRT_DIR/ca.crt" --key="$CA_KEY_DIR/ca.key" --context "$CONTEXT" kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" -kubectl create secret tls cockroachdb.tls --namespace "$NAMESPACE" --cert="$CA_CRT_DIR/ca.crt" --key="$CA_KEY_DIR/ca.key" --context "$CONTEXT" - echo '=========================================================================' echo '= Secrets uploaded successfully. =' echo '=========================================================================' From 9a26aa86b0486ac37ab15b4587602a91b1b4dc01 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 17:24:50 +0200 Subject: [PATCH 05/13] Add README and schema validation --- deploy/services/helm-charts/dss/README.md | 21 ++ .../dss/templates/dss-core-service.yaml | 2 +- .../helm-charts/dss/values.example.yaml | 44 +++++ .../helm-charts/dss/values.schema.json | 184 ++++++++++++++++++ deploy/services/helm-charts/dss/values.yaml | 19 ++ 5 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 deploy/services/helm-charts/dss/README.md create mode 100644 deploy/services/helm-charts/dss/values.example.yaml create mode 100644 deploy/services/helm-charts/dss/values.schema.json create mode 100644 deploy/services/helm-charts/dss/values.yaml diff --git a/deploy/services/helm-charts/dss/README.md b/deploy/services/helm-charts/dss/README.md new file mode 100644 index 000000000..584cd2528 --- /dev/null +++ b/deploy/services/helm-charts/dss/README.md @@ -0,0 +1,21 @@ +# DSS Helm Chart +This [Helm Chart](https://helm.sh/) deploys the DSS and cockroachdb kubernetes resources. + +## Requirements +1. A Kubernetes cluster should be running and you should be properly authenticated. +Requirements and instructions to create a new Kubernetes cluster can be found [here](../../../infrastructure/README.md). +2. Create the certificates and apply them to the cluster using the instructions of [section 6 and 7](../../../../build/README.md) +3. Install [Helm](https://helm.sh/) version 3.11.3 or higher + +## Usage +1. Edit `values.example.yaml`. See `values.schema.json` for schema definition. (Note that the key `cockroachdb` supports all values supported by the [`cockroachdb` Chart](https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb#configuration)) +2. Validate the configuration: `helm lint -f values.example.yaml .` +3. Set a RELEASE_NAME, example: `export RELEASE_NAME=dss` +4. Set the kube client context of your , example: `export KUBE_CONTEXT=gke_interuss-deploy-example_europe-west6-a_dss-dev-w6` +5. Run `helm dep update --kube-context=$KUBE_CONTEXT` +6. Install the chart: `helm install --kube-context=$KUBE_CONTEXT -f values.example.yaml $RELEASE_NAME .` + +### Update the chart +When changing the values in values.example.yaml, values.yaml, the templates or upgrading the helm chart dependencies, changes can be applied to the cluster using the following command: + +1. Run `helm upgrade --kube-context=$KUBE_CONTEXT -f values.example.yaml $RELEASE_NAME .` diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 194858c24..77cccafa3 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -42,7 +42,7 @@ spec: - --cockroach_ssl_mode=verify-full - --cockroach_user=root - --dump_requests=true - - --enable_scd=true + - --enable_scd={{$dss.enableScd | default false}} - --garbage_collector_spec=@every 30m - --gcp_prof_service_name= {{- if $dss.conf.jwksEndpoint }} diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml new file mode 100644 index 000000000..efd9a996c --- /dev/null +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -0,0 +1,44 @@ +# See values.schema.json for schema description + +cockroachdb: + # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml + fullnameOverride: dss-cockroachdb + conf: + join: [] + cluster-name: interuss-example + single-node: false + locality: zone=interuss-example-google-ew1 + + statefulset: + args: + - --locality-advertise-addr=zone=interuss-example-google-ew1@$(hostname -f) + - --advertise-addr=${HOSTNAME##*-}.db.example.com + + storage: + persistentVolume: + storageClass: standard + +loadBalancers: + cockroachdbNodes: + - ip: 192.168.0.21 +# subnet: subnet-xxx # for aws only + - ip: 192.168.0.22 +# subnet: subnet-xxx # for aws only + - ip: 192.168.0.23 +# subnet: subnet-xxx # for aws only + dssGateway: + ip: 192.168.0.10 +# subnet: subnet-xxx # for aws only +# certName: arn:xxxx # for aws only + +dss: + conf: + pubKeys: + - /test-certs/auth2.pem + jwksEndpoint: '' + jwksKeyIds: [] + hostname: dss.example.com + enableScd: true + +global: + cloudProvider: aws diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json new file mode 100644 index 000000000..58dcc3d3a --- /dev/null +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -0,0 +1,184 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + "cockroachdb": { + "description": "Cockroach DB related configuration", + "type": "object", + "properties": { + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "fullnameOverride": { + "description": "Name of the internal statefulset", + "type": "string" + }, + "conf": { + "type": "object", + "properties": { + "cluster-name": { + "description": "Name of CockroachDB cluster", + "type": "string" + }, + "join": { + "description": "List of already-existing CockroachDB instances. If empty, the chart will automatically discover the internal nodes. If set to join a DSS Region and other DSS instance, please add the addresses of the statefulset's pod as well.", + "type": "array", + "items": { + "type": "string" + } + }, + "locality": { + "description": "Locality attribute for this deployment. Example: zone=example", + "type": "string" + } + }, + "required": [ + "cluster-name", + "locality" + ] + }, + "statefulset": { + "type": "object", + "properties": { + "args": { + "type": "array", + "description": "Additional argument passed to the statefulset pod command. In the context of the dss deployment it requires the following arguments:\n `--locality-advertise-addr=zone=example@$(hostname -f)`\n`--advertise-addr=${HOSTNAME##*-}.db.example.com`", + "items": { + "type": "string" + }, + "additionalItems": true + } + }, + "required": [ + "args" + ] + }, + "storage": { + "type": "object", + "properties": { + "persistentVolume": { + "properties": { + "storageClass": { + "description": "Storage class depending on your cloud provider.", + "type": "string" + } + } + } + } + } + }, + "required": [ + "image", + "fullnameOverride", + "conf", + "statefulset" + ] + }, + "loadBalancers": { + "description": "Load balancers configuration", + "type": "object", + "properties": { + "cockroachdbNodes": { + "type": "array", + "items": { + "properties": { + "ip": { + "description": "Public ip or cloud resource name.", + "type": "string" + }, + "subnet": { + "description": "Optional: Subnet of the public ip for AWS only", + "type": "string" + } + }, + "required": [ + "ip" + ] + } + }, + "dssGateway": { + "type": "object", + "properties": { + "ip": { + "description": "Public ip or cloud resource name.", + "type": "string" + }, + "subnet": { + "description": "Optional: Subnet of the public ip for AWS only", + "type": "string" + }, + "certName": { + "description": "Optional: ARN of the certificate for AWS only", + "type": "string" + } + }, + "required": [ + "ip" + ] + } + } + }, + "dss": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "conf": { + "type": "object", + "properties": { + "pubKeys": { + "type": "array", + "items": { + "type": "string" + } + }, + "jwksEndpoint": { + "type": "string" + }, + "jwksKeyIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "enableScd": { + "type": "boolean" + }, + "hostname": { + "type": "string", + "description": "Public hostname of the dss. Example: dss.example.com" + } + }, + "required": ["hostname"] + } + }, + "required": ["image", "conf"] + }, + "global": { + "type": "object", + "properties": { + "cloudProvider": { + "description": "Cloud provider identifier. `aws` or `google`", + "type": "string" + } + }, + "required": ["cloudProvider"] + } + }, + "required": [ + "cockroachdb", + "loadBalancers", + "dss", + "global" + ], + "title": "Values", + "type": "object" +} \ No newline at end of file diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml new file mode 100644 index 000000000..ea1a53e67 --- /dev/null +++ b/deploy/services/helm-charts/dss/values.yaml @@ -0,0 +1,19 @@ +# Default configuration +cockroachdb: + # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml + image: + repository: cockroachdb/cockroach + tag: v21.2.7 + tls: + certs: + provided: true + nodeSecret: cockroachdb.node + clientRootSecret: cockroachdb.client.root + tlsSecret: cockroachdb.tls + selfSigner: + enabled: false + ingress: + enabled: false + +dss: + image: docker.io/interuss/dss:v0.7.0 \ No newline at end of file From 46f8eefaf7a2314f8c705273e7760e970aaab206 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 17:32:47 +0200 Subject: [PATCH 06/13] Add missing READMEs --- deploy/infrastructure/README.md | 6 ++++++ deploy/services/README.md | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 deploy/infrastructure/README.md create mode 100644 deploy/services/README.md diff --git a/deploy/infrastructure/README.md b/deploy/infrastructure/README.md new file mode 100644 index 000000000..0625d4409 --- /dev/null +++ b/deploy/infrastructure/README.md @@ -0,0 +1,6 @@ +# DSS Deployment + +The `modules` directory contains the terraform modules required to deploy the DSS using cloud providers. + +- [terraform-aws-dss](./terraform-aws-dss/README.md): Amazon Web Services deployment +- [terraform-google-dss](./terraform-google-dss/README.md): Google Cloud Engine deployment diff --git a/deploy/services/README.md b/deploy/services/README.md new file mode 100644 index 000000000..fde48f581 --- /dev/null +++ b/deploy/services/README.md @@ -0,0 +1,6 @@ +# Services + +This folder contains the tools to deploy a DSS to a Kubernetes cluster. + +- [helm-charts/dss](./helm-charts/dss/README.md): Helm chart to deploy the DSS. + From 9185e1b9022cc1077f1a0b09bac72211a1940648 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 17:33:21 +0200 Subject: [PATCH 07/13] typo --- deploy/services/README.md | 1 - deploy/services/helm-charts/dss/values.example.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/services/README.md b/deploy/services/README.md index fde48f581..1e90fb6f1 100644 --- a/deploy/services/README.md +++ b/deploy/services/README.md @@ -3,4 +3,3 @@ This folder contains the tools to deploy a DSS to a Kubernetes cluster. - [helm-charts/dss](./helm-charts/dss/README.md): Helm chart to deploy the DSS. - diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index efd9a996c..deff2328f 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -41,4 +41,4 @@ dss: enableScd: true global: - cloudProvider: aws + cloudProvider: google From 7ad40ed09e48894f2cce953b6f273ae1656f7267 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 18:57:16 +0200 Subject: [PATCH 08/13] Clean up certificate for helm --- build/apply-certs.sh | 11 +++++------ build/make-certs.py | 3 --- deploy/services/helm-charts/dss/values.yaml | 1 - 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/build/apply-certs.sh b/build/apply-certs.sh index b2074ab51..141247c5a 100755 --- a/build/apply-certs.sh +++ b/build/apply-certs.sh @@ -44,15 +44,14 @@ kubectl delete secret cockroachdb.ca.crt --namespace "$NAMESPACE" --context "$C kubectl delete secret cockroachdb.ca.key --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl create secret generic cockroachdb.client.root --namespace default --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" -if [[ $NAMESPACE != "default" ]]; then - kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" -fi -kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" # The ca key is not needed for any typical operations, but might be required to sign new certificates. $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR" --context "$CONTEXT" # The ca.crt is kept in it's own secret to more easily manage cert rotation and -# adding other operators' certificates. +# adding other operators' certificates. Note that, for the purpose of the migration to helm and +# to comply with cockroach db standard configuration, ca.crt has been kept inside cockroach.* secrets. +# This secret is kept for backward compatibility. kubectl create secret generic cockroachdb.ca.crt --namespace "$NAMESPACE" --from-file "$CA_CRT_DIR" --context "$CONTEXT" kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" diff --git a/build/make-certs.py b/build/make-certs.py index 7aa58d555..2ea8f1b0e 100755 --- a/build/make-certs.py +++ b/build/make-certs.py @@ -156,9 +156,6 @@ def main(): '--ca-key', cr.ca_key_file] + node_addresses) - os.remove(os.path.join(cr.node_certs_dir, 'ca.crt')) - os.remove(os.path.join(cr.client_certs_dir, 'ca.crt')) - print('Created new node certificate in {}'.format(cr.node_certs_dir)) diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index ea1a53e67..cf3fab895 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -9,7 +9,6 @@ cockroachdb: provided: true nodeSecret: cockroachdb.node clientRootSecret: cockroachdb.client.root - tlsSecret: cockroachdb.tls selfSigner: enabled: false ingress: From a1f965c9dc1dac4b6ad963058d48738e72bff07a Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 19:17:22 +0200 Subject: [PATCH 09/13] Update README --- deploy/infrastructure/README.md | 4 ++-- deploy/services/helm-charts/dss/README.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/infrastructure/README.md b/deploy/infrastructure/README.md index 0625d4409..6b84c0e51 100644 --- a/deploy/infrastructure/README.md +++ b/deploy/infrastructure/README.md @@ -2,5 +2,5 @@ The `modules` directory contains the terraform modules required to deploy the DSS using cloud providers. -- [terraform-aws-dss](./terraform-aws-dss/README.md): Amazon Web Services deployment -- [terraform-google-dss](./terraform-google-dss/README.md): Google Cloud Engine deployment +- [terraform-aws-dss](./modules/terraform-aws-dss/README.md): Amazon Web Services deployment +- [terraform-google-dss](./modules/terraform-google-dss/README.md): Google Cloud Engine deployment diff --git a/deploy/services/helm-charts/dss/README.md b/deploy/services/helm-charts/dss/README.md index 584cd2528..a6386ddf9 100644 --- a/deploy/services/helm-charts/dss/README.md +++ b/deploy/services/helm-charts/dss/README.md @@ -10,7 +10,8 @@ Requirements and instructions to create a new Kubernetes cluster can be found [h ## Usage 1. Edit `values.example.yaml`. See `values.schema.json` for schema definition. (Note that the key `cockroachdb` supports all values supported by the [`cockroachdb` Chart](https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb#configuration)) 2. Validate the configuration: `helm lint -f values.example.yaml .` -3. Set a RELEASE_NAME, example: `export RELEASE_NAME=dss` +3. Set a RELEASE_NAME to `dss`: `export RELEASE_NAME=dss` +It is temporarily the only release name possible. 4. Set the kube client context of your , example: `export KUBE_CONTEXT=gke_interuss-deploy-example_europe-west6-a_dss-dev-w6` 5. Run `helm dep update --kube-context=$KUBE_CONTEXT` 6. Install the chart: `helm install --kube-context=$KUBE_CONTEXT -f values.example.yaml $RELEASE_NAME .` From c9dc6edfce83c6eb234f56dede3e72400e8fa852 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 22:09:16 +0200 Subject: [PATCH 10/13] Fix deprecated warning --- .../services/helm-charts/dss/templates/dss-ingress-google.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml b/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml index 9f64165bc..988ec007f 100644 --- a/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-ingress-google.yaml @@ -46,7 +46,7 @@ spec: port: number: 8080 --- -apiVersion: networking.gke.io/v1beta1 +apiVersion: networking.gke.io/v1 kind: ManagedCertificate metadata: labels: From bb2d38033ba3a5f93f8def9190100fb5e924bd5e Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 26 Apr 2023 22:11:52 +0200 Subject: [PATCH 11/13] Keep backward compatibility --- build/apply-certs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/apply-certs.sh b/build/apply-certs.sh index 141247c5a..32403eecc 100755 --- a/build/apply-certs.sh +++ b/build/apply-certs.sh @@ -44,7 +44,10 @@ kubectl delete secret cockroachdb.ca.crt --namespace "$NAMESPACE" --context "$C kubectl delete secret cockroachdb.ca.key --namespace "$NAMESPACE" --context "$CONTEXT" || true kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic cockroachdb.client.root --namespace default --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" +if [[ $NAMESPACE != "default" ]]; then + kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" +fi kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" # The ca key is not needed for any typical operations, but might be required to sign new certificates. $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR" --context "$CONTEXT" From a6ea2a3489012eea33148ac9fbea8891191b2344 Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Fri, 17 Nov 2023 13:56:53 +0100 Subject: [PATCH 12/13] Update documentation --- deploy/services/helm-charts/dss/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/deploy/services/helm-charts/dss/README.md b/deploy/services/helm-charts/dss/README.md index a6386ddf9..7e4a15d5e 100644 --- a/deploy/services/helm-charts/dss/README.md +++ b/deploy/services/helm-charts/dss/README.md @@ -8,15 +8,16 @@ Requirements and instructions to create a new Kubernetes cluster can be found [h 3. Install [Helm](https://helm.sh/) version 3.11.3 or higher ## Usage -1. Edit `values.example.yaml`. See `values.schema.json` for schema definition. (Note that the key `cockroachdb` supports all values supported by the [`cockroachdb` Chart](https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb#configuration)) -2. Validate the configuration: `helm lint -f values.example.yaml .` +1. Copy `values.example.yaml` to `values.dev.yaml` and edit it. See `values.schema.json` for schema definition. (Note that the key `cockroachdb` supports all values supported by the [`cockroachdb` Chart](https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb#configuration)). +Note that values.yaml contains the default values and are always passed to helm. +2. Validate the configuration: `helm lint -f values.dev.yaml .` 3. Set a RELEASE_NAME to `dss`: `export RELEASE_NAME=dss` It is temporarily the only release name possible. 4. Set the kube client context of your , example: `export KUBE_CONTEXT=gke_interuss-deploy-example_europe-west6-a_dss-dev-w6` 5. Run `helm dep update --kube-context=$KUBE_CONTEXT` -6. Install the chart: `helm install --kube-context=$KUBE_CONTEXT -f values.example.yaml $RELEASE_NAME .` +6. Install the chart: `helm install --kube-context=$KUBE_CONTEXT -f values.dev.yaml $RELEASE_NAME .` ### Update the chart -When changing the values in values.example.yaml, values.yaml, the templates or upgrading the helm chart dependencies, changes can be applied to the cluster using the following command: +When changing the values in values.dev.yaml, values.yaml, the templates or upgrading the helm chart dependencies, changes can be applied to the cluster using the following command: -1. Run `helm upgrade --kube-context=$KUBE_CONTEXT -f values.example.yaml $RELEASE_NAME .` +1. Run `helm upgrade --kube-context=$KUBE_CONTEXT -f values.dev.yaml $RELEASE_NAME .` From 469a2138283a7b297ddf8d68af8386abd476bbba Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Wed, 3 Jan 2024 16:14:40 +0100 Subject: [PATCH 13/13] Update documentation --- deploy/README.md | 6 ++++-- .../helm-charts/dss/templates/dss-core-service.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 5cdb759ec..fdbcdc1b2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,7 +1,5 @@ # DSS Deployment -**Work in progress** - This folder contains the increments toward the new deployment approach as described in [#874](https://github.com/interuss/dss/issues/874). The infrastructure folder contains the terraform modules to deploy the DSS to kubernetes clusters of various cloud providers: @@ -9,3 +7,7 @@ The infrastructure folder contains the terraform modules to deploy the DSS to ku - Amazon Web Services: [terraform-aws-dss](./infrastructure/modules/terraform-aws-dss/README.md) - Google Cloud Engine: [terraform-google-dss](./infrastructure/modules/terraform-google-dss/README.md) +The service folder contains the scripts required to deploy the DSS to a Kubernetes cluster: + +- Helm Charts: [services/helm-charts](./services/helm-charts) +- Tanka: [../build/deploy/](../build/deploy) diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 77cccafa3..da3a2dc33 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -42,7 +42,7 @@ spec: - --cockroach_ssl_mode=verify-full - --cockroach_user=root - --dump_requests=true - - --enable_scd={{$dss.enableScd | default false}} + - --enable_scd={{$dss.enableScd | default true}} - --garbage_collector_spec=@every 30m - --gcp_prof_service_name= {{- if $dss.conf.jwksEndpoint }}