feat: cloud sql support (#1531)

Implementation of interaction of cd-service with remote postgressql database instance Story: SRX-QJWBAC --------- Co-authored-by: Ahmed Nour <[email protected]> Co-authored-by: Sven Urbanski <[email protected]>
freiheit-com · Apr 25, 2024 · 87a546b · 87a546b
1 parent 777dc1d
commit 87a546b
Show file tree

Hide file tree

Showing 10 changed files with 423 additions and 69 deletions.
diff --git a/charts/kuberpult/run-kind.sh b/charts/kuberpult/run-kind.sh
@@ -51,7 +51,7 @@ function waitForDeployment() {
   sleep 10
   until kubectl wait --for=condition=ready pod -n "$ns" -l "$label" --timeout=30s
   do
-    sleep 4s
+    sleep 4
     print "logs:"
     kubectl -n "$ns" logs -l "$label" || echo "could not get logs for $label"
     print "describe pod:"
@@ -74,7 +74,7 @@ function portForwardAndWait() {
   sleep 10
   until nc -vz localhost "$portHere"
   do
-    sleep 3s
+    sleep 3
     print "logs:"
     kubectl -n "$ns" logs "$deployment"
     print "describe deployment:"
@@ -172,7 +172,10 @@ print "$frontend_imagename"
 kind load docker-image "$cd_imagename"
 kind load docker-image "$frontend_imagename"
 kind load docker-image "$rollout_imagename"
-
+kind load docker-image quay.io/argoproj/argocd:v2.7.4
+kind load docker-image ghcr.io/dexidp/dex:v2.36.0
+kind load docker-image gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.11.0
+kind load docker-image public.ecr.aws/docker/library/redis:7.0.11-alpine
 
 ## argoCd
 
@@ -270,6 +273,9 @@ cd:
     requests:
       memory: 200Mi
       cpu: 0.05
+  db:
+    dbOption: sqlite
+    location: /sqlite
 frontend:
   resources:
     limits:

diff --git a/charts/kuberpult/templates/cd-service.yaml b/charts/kuberpult/templates/cd-service.yaml
@@ -32,7 +32,9 @@
 {{- if .Values.cd.tag }}
 {{ fail "Values.cd.tag cannot be used anymore. We only support the same appVersion for all services at this point."}}
 {{ end -}}
-
+{{- if not (or (eq .Values.cd.db.dbOption "cloudsql") (eq .Values.cd.db.dbOption "sqlite") (eq .Values.cd.db.dbOption "NO_DB")) }}
+{{ fail ".Values.cd.db.dbOption does not contain a valid value (NO_DB, sqlite, cloudsql)."}}
+{{ end -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -73,7 +75,48 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- if eq .Values.cd.db.dbOption "cloudsql" }}
+      serviceAccountName: {{ .Values.cd.db.k8sName }}  
+{{- end }} 
       containers:
+{{- if eq .Values.cd.db.dbOption "cloudsql" }}
+      - name: cloud-sql-proxy
+        # It is recommended to use the latest version of the Cloud SQL Auth Proxy
+        # Make sure to update on a regular schedule!
+        image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.11.0
+        args:
+          # If connecting from a VPC-native GKE cluster, you can use the
+          # following flag to have the proxy connect over private IP
+          - "--private-ip"
+
+          # Enable structured logging with LogEntry format:
+          - "--structured-logs"
+
+          # Replace DB_PORT with the port the proxy should listen on
+
+          - "--port={{ .Values.cd.db.authProxyPort }}"
+
+          - {{ .Values.cd.db.dbConnectionName | quote }}
+        securityContext:
+          # The default Cloud SQL Auth Proxy image runs as the
+          # "nonroot" user and group (uid: 65532) by default.
+          runAsNonRoot: true
+        # You should use resource requests/limits as a best practice to prevent
+        # pods from consuming too many resources and affecting the execution of
+        # other pods. You should adjust the following values based on what your
+        # application needs. For details, see
+        # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+        resources:
+          requests:
+            # The proxy's memory use scales linearly with the number of active
+            # connections. Fewer open connections will use less memory. Adjust
+            # this value based on your application's requirements.
+            memory: {{ .Values.cd.db.requests.memory }}
+            # The proxy's CPU use scales linearly with the amount of IO between
+            # the database and the application. Adjust this value based on your
+            # application's requirements.
+            cpu:  {{ .Values.cd.db.requests.cpu }}
+{{- end }}
       - name: service
         image: "{{ .Values.hub }}/{{ .Values.cd.image }}:{{ $.Chart.AppVersion }}"
         ports:
@@ -183,8 +226,6 @@ spec:
         - name: KUBERPULT_BOOTSTRAP_MODE
           value: "{{ .Values.environment_configs.bootstrap_mode }}"
 {{- end }}
-        - name: KUBERPULT_ENABLE_SQLITE
-          value: "{{ .Values.cd.enableSqlite }}"
         - name: KUBERPULT_GIT_NETWORK_TIMEOUT
           value: "{{ .Values.git.networkTimeout }}"
         - name: KUBERPULT_GIT_WRITE_COMMIT_DATA
@@ -195,9 +236,35 @@ spec:
           value: "{{ .Values.datadogProfiling.enabled }}"
         - name: KUBERPULT_MAXIMUM_QUEUE_SIZE
           value: "{{ .Values.cd.backendConfig.queueSize }}"
+        - name: KUBERPULT_DB_OPTION # { NO_DB, cloudsql, sqlite }
+          value: {{ .Values.cd.db.dbOption }}
+{{- if or (eq .Values.cd.db.dbOption "cloudsql") (eq .Values.cd.db.dbOption "sqlite") }}
+        - name: KUBERPULT_DB_LOCATION
+          value: {{ .Values.cd.db.location }}
+        - name: KUBERPULT_DB_MIGRATIONS_LOCATION
+          value: {{ .Values.cd.db.migrations }}
+{{- end }}
+{{- if eq .Values.cd.db.dbOption "cloudsql" }}
+        - name: KUBERPULT_DB_NAME
+          value: "{{ .Values.cd.db.dbName }}"
+        - name: KUBERPULT_DB_USER_NAME
+          value: "{{ .Values.cd.db.dbUser }}"
+        - name: KUBERPULT_DB_USER_PASSWORD
+          value: "{{ .Values.cd.db.dbPassword }}"
+{{- end }}
         - name: KUBERPULT_ALLOW_LONG_APP_NAMES
           value: "{{ .Values.cd.allowLongAppNames }}"
         volumeMounts:
+{{- if or (eq .Values.cd.db.dbOption "cloudsql") (eq .Values.cd.db.dbOption "sqlite") }}
+        - name: migrations
+          mountPath: {{ .Values.cd.db.migrations }}
+          readOnly: false
+{{- end }}
+{{- if eq .Values.cd.db.dbOption "sqlite" }}
+        - name: sqlite
+          mountPath: /sqlite
+          readOnly: false
+{{- end }}
         - name: repository
           mountPath: /repository
         - name: ssh
@@ -244,6 +311,16 @@ spec:
         configMap:
           name: kuberpult-keyring
 {{- end }}
+{{- if or (eq .Values.cd.db.dbOption "cloudsql") (eq .Values.cd.db.dbOption "sqlite") }}
+      - name: migrations
+        configMap:
+          name: kuberpult-migrations
+{{- end }} 
+{{- if eq .Values.cd.db.dbOption "sqlite" }}
+      - name: sqlite
+        emptyDir:
+          sizeLimit: 1Gi
+{{- end }}
 {{- if .Values.environment_configs.bootstrap_mode }}
       - name: environment-configs
         configMap:
@@ -343,3 +420,11 @@ metadata:
 data:
   policy.csv: {{ .Values.auth.dexAuth.policy_csv | quote}}
 {{- end }}
+{{- if or (eq .Values.cd.db.dbOption "cloudsql") (eq .Values.cd.db.dbOption "sqlite") }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kuberpult-migrations
+data:
+  20210813132049_test.up.sql: "CREATE TABLE IF NOT EXISTS dummy_table(id BIGINT, created int, data int, PRIMARY KEY(id));"
+{{- end }}
diff --git a/charts/kuberpult/tests/charts_test.go b/charts/kuberpult/tests/charts_test.go
@@ -18,15 +18,16 @@ package main_test
 
 import (
 	"fmt"
-	apps "k8s.io/api/apps/v1"
-	core "k8s.io/api/core/v1"
 	"math/rand"
 	"os"
 	"os/exec"
-	k8sYaml "sigs.k8s.io/yaml" //Needed to properly parse the yaml generated by helm, "gopkg.in/yaml.v3" cannot do it properly
 	"strconv"
 	"strings"
 	"testing"
+
+	apps "k8s.io/api/apps/v1"
+	core "k8s.io/api/core/v1"
+	k8sYaml "sigs.k8s.io/yaml" //Needed to properly parse the yaml generated by helm, "gopkg.in/yaml.v3" cannot do it properly
 )
 
 func runHelm(t *testing.T, valuesData []byte, dirName string) string {
@@ -327,6 +328,122 @@ argocd:
 			},
 			ExpectedMissing: []core.EnvVar{},
 		},
+		{
+			Name: "Database disabled",
+			Values: `
+git:
+  url: "testURL"
+ingress:
+  domainName: "kuberpult-example.com"
+cd:
+  db:
+    dbOption: NO_DB
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_DB_OPTION",
+					Value: "NO_DB",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{
+
+				{
+					Name:  "KUBERPULT_DB_LOCATION",
+					Value: "",
+				},
+				{
+					Name:  "KUBERPULT_DB_NAME",
+					Value: "",
+				},
+				{
+					Name:  "KUBERPULT_DB_USER_NAME",
+					Value: "",
+				},
+				{
+					Name:  "KUBERPULT_DB_USER_PASSWORD",
+					Value: "",
+				},
+			},
+		},
+		{
+			Name: "Database cloudsql enabled",
+			Values: `
+git:
+  url: "testURL"
+ingress:
+  domainName: "kuberpult-example.com"
+cd:
+  db:
+    dbOption: cloudsql
+    location: "127.0.0.1"
+    dbName: dbName
+    dbUser: dbUser
+    dbPassword: dbPassword
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_DB_OPTION",
+					Value: "cloudsql",
+				},
+				{
+					Name:  "KUBERPULT_DB_LOCATION",
+					Value: "127.0.0.1",
+				},
+				{
+					Name:  "KUBERPULT_DB_NAME",
+					Value: "dbName",
+				},
+				{
+					Name:  "KUBERPULT_DB_USER_NAME",
+					Value: "dbUser",
+				},
+				{
+					Name:  "KUBERPULT_DB_USER_PASSWORD",
+					Value: "dbPassword",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{},
+		},
+		{
+			Name: "Database cloudsql enabled",
+			Values: `
+git:
+  url: "testURL"
+ingress:
+  domainName: "kuberpult-example.com"
+cd:
+  db:
+    dbOption: sqlite
+    location: /kp/database
+    dbName: does
+    dbUser: not
+    dbPassword: matter
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_DB_OPTION",
+					Value: "sqlite",
+				},
+				{
+					Name:  "KUBERPULT_DB_LOCATION",
+					Value: "/kp/database",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_DB_NAME",
+					Value: "",
+				},
+				{
+					Name:  "KUBERPULT_DB_USER_NAME",
+					Value: "",
+				},
+				{
+					Name:  "KUBERPULT_DB_USER_PASSWORD",
+					Value: "",
+				},
+			},
+		},
 	}
 
 	for _, tc := range tcs {

diff --git a/charts/kuberpult/values.yaml b/charts/kuberpult/values.yaml
@@ -84,7 +84,22 @@ cd:
     requests:
       cpu: 2
       memory: 3Gi
-  enableSqlite: true
+# The Database is not ready for use on production and the helm options might change in the near future.
+  db:
+    location: "127.0.0.1"
+    authProxyPort: 5432
+    dbOption: "cloudsql"
+# k8sName is required if `dbOption = "cloudsql"`, otherwise it's ignored.
+# k8sName is the name of the kubernetes service account.
+    k8sName: "k8sname"
+    dbConnectionName: "connectioname"
+    dbName: "databaseName"
+    dbUser: "username"
+    dbPassword: "password"
+    migrations: /migrations
+    requests:
+      cpu: "100m"
+      memory: "200Mi"
   probes:
     liveness:
       periodSeconds: 10

diff --git a/docker-compose-earthly.yml b/docker-compose-earthly.yml
@@ -6,6 +6,8 @@ services:
       - KUBERPULT_GIT_URL=/kp/kuberpult/repository_remote
       - KUBERPULT_DB_ENABLED=true
       - KUBERPULT_DB_LOCATION=/kp/database
+      - KUBERPULT_DB_MIGRATIONS_LOCATION=/kp/database/migrations
+      - KUBERPULT_DB_OPTION=sqlite
       - KUBERPULT_GIT_BRANCH=master
       - KUBERPULT_GIT_NETWORK_TIMEOUT=3s
       - KUBERPULT_DEX_MOCK=false

diff --git a/go.mod b/go.mod
@@ -173,6 +173,7 @@ require (
 	github.com/lestrrat-go/httprc v1.0.5 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect