feat: require database

Ref: SRX-NMC7OD

BREAKING CHANGE: The Database is now required.

charts/kuberpult/templates/cd-service.yaml

@@ -18,11 +18,8 @@
 {{- if .Values.cd.tag }}
 {{ fail "Values.cd.tag cannot be used anymore. We only support the same appVersion for all services at this point."}}
 {{ end -}}
-{{- if not (or (eq .Values.db.dbOption "postgreSQL") (eq .Values.db.dbOption "NO_DB")) }}
-{{ fail ".Values.db.dbOption does not contain a valid value (NO_DB, postgreSQL)."}}
-{{ end -}}
-{{- if (and (.Values.db.cloudSqlProxyEnabled) (eq .Values.db.dbOption "NO_DB")) }}
-{{ fail "Cloudsql proxy cannot be used with NO_DB option"}}
+{{- if not (eq .Values.db.dbOption "postgreSQL") }}
+{{ fail ".Values.db.dbOption does not contain a valid value (Only postgreSQL is allowed from now on)."}}
 {{ end -}}
 ---
 apiVersion: apps/v1
@@ -241,10 +238,8 @@ spec:
           value: "{{ .Values.datadogProfiling.enabled }}"
         - name: KUBERPULT_MAXIMUM_QUEUE_SIZE
           value: "{{ .Values.cd.backendConfig.queueSize }}"
-        - name: KUBERPULT_DB_OPTION # { NO_DB, postgreSQL}
+        - name: KUBERPULT_DB_OPTION # { postgreSQL}
           value: {{ .Values.db.dbOption }}
-        - name: KUBERPULT_DB_WRITE_ESL_TABLE_ONLY
-          value: "{{ .Values.db.writeEslTableOnly }}"
 {{- if (eq .Values.db.dbOption "postgreSQL") }}
         - name: KUBERPULT_DB_LOCATION
           value: {{ .Values.db.location }}

charts/kuberpult/values.yaml

@@ -94,9 +94,9 @@ cd:
     create: false   # Add backend config for health checks on GKE only
     timeoutSec: 300  # 30sec is the default on gcp loadbalancers, however kuberpult needs more with parallel requests. It is the time how long the loadbalancer waits for kuberpult to finish calls to the rest endpoint "release"
     queueSize: 5
-# Disabling the queue is as of now an experimental feature. It's only possible to use with db.writeEslTableOnly=false.
+# Disabling the queue is as of now an experimental feature.
 # With the queue, the cd-service processes only one request at a time, which is very much required when using git.
-# With the database enabled, this is not required anymore.
+# With the database, this is not required anymore.
     disableQueue: false
   resources:
     limits:
@@ -118,29 +118,21 @@ cd:
       timeoutSeconds: 5
       failureThreshold: 10
       initialDelaySeconds: 5
-# The Database is not ready for use on production and the helm options might change in the near future.
 db:
   location: "127.0.0.1"
   authProxyPort: 5432
-# As the Database feature is still work in progress, for now we recommend to set:
-# dbOption: "NO_DB"
-# Other valid values are "postgreSQL".
-# If you do want to try out the database, for now use
-# writeEslTableOnly: true
-  dbOption: "NO_DB"
-# k8sServiceAccountName is required if `dbOption = "postgreSQL"`, otherwise it's ignored.
+# The Database feature is now required:
+  dbOption: "postgreSQL"
+# k8sServiceAccountName is required.
 # k8sServiceAccountName is the name of the kubernetes service account.
   k8sServiceAccountName: "k8sServiceAccountName"
-# cloudSqlProxyEnabled enables the cloudsql proxy. Not possible with `dbOption = "NO_DB"`.
+# cloudSqlProxyEnabled enables the cloudsql proxy.
   cloudSqlProxyEnabled: false
   dbConnectionName: "connectioname"
   dbName: "databaseName"
   dbUser: "username"
   dbPassword: "password"
   migrations: /migrations
-  # If set to true, kuberpult will write only the ESL table.
-  # This is useful to already collect historical data in the database, while waiting for the full database implementation.
-  writeEslTableOnly: false
   # SSL mode to be used by the database
   sslMode: verify-full
   requests:

docs/database.md

@@ -1,15 +1,11 @@
 # Database
 
-## The Database feature is not ready for production yet
-
-However, you can already prepare for the database feature,
-see [Preparation](#preparation)
-
+## The Database feature is now required
 
 ## Background
 
-Kuberpult is switching over to use a database. The rough timeline to have the database production-ready is summer 2024.
-As of now Kuberpult is using the manifest repository to store data.
+Kuberpult switched over to use a database.
+In the past, Kuberpult was using the manifest repository to store data.
 This worked fine for a while, and it had the added bonus
 that ArgoCD is reading from the same manifest git repository.
 
@@ -21,17 +17,14 @@ Therefore, we will use a database and not rely on git anymore in the future.
 
 Git will still be used as an *output* of kuberpult, but not as the source of truth.
 
-As of now, kuberpult still supports the option to not have the database,
-but this option will be removed in a few weeks with another breaking change in kuberpult.
-
-
 ## Preparation
 
 Our recommendation is to enable the database mode in 2 steps:
 
 ### Step 1: writeEslTableOnly=true
 
 Enable the Database with `db.dbOption: "postgreSQL"` and `db.writeEslTableOnly: true`.
+This requires kuberpult version <= 10.3.10.
 This means that kuberpult will connect to the DB, but only write one table.
 Kuberpult will not read from the database in this state,
 so the manifest-repository is still considered the source of truth.

pkg/db/db.go

@@ -54,7 +54,6 @@ type DBConfig struct {
 	DriverName     string
 	DbPassword     string
 	MigrationsPath string
-	WriteEslOnly   bool
 	SSLMode        string
 
 	MaxIdleConnections uint
@@ -70,14 +69,6 @@ type DBHandler struct {
 	DB             *sql.DB
 	DBDriver       *database.Driver
 
-	/*
-		There are 3 modes:
-		1) DBHandler==nil: do not write anything to the DB
-		2) DBHandler!=nil && WriteEslOnly==true: write only the ESL table to the database. Stores all incoming data in the DB, but does not read the DB.
-		3) DBHandler!=nil && WriteEslOnly==false: write everything to the database.
-	*/
-	WriteEslOnly bool
-
 	// InsertAppFun is intended to be used to add more to inserting an app: specifically to update the overview cache
 	InsertAppFun InsertAppFun
 }
@@ -102,13 +93,11 @@ const (
 )
 
 func (h *DBHandler) ShouldUseEslTable() bool {
-	return h != nil
+	return true
 }
 
-// ShouldUseOtherTables returns true if the db is enabled and WriteEslOnly=false
-// ShouldUseOtherTables should never be used in the manifest-repo-export-service.
 func (h *DBHandler) ShouldUseOtherTables() bool {
-	return h != nil && !h.WriteEslOnly
+	return true
 }
 
 func Connect(ctx context.Context, cfg DBConfig) (*DBHandler, error) {
@@ -123,7 +112,6 @@ func Connect(ctx context.Context, cfg DBConfig) (*DBHandler, error) {
 		MigrationsPath: cfg.MigrationsPath,
 		DB:             db,
 		DBDriver:       &driver,
-		WriteEslOnly:   cfg.WriteEslOnly,
 		InsertAppFun:   nil,
 	}
 	handler.InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error {

pkg/db/db_test.go

@@ -763,22 +763,18 @@ func TestHelperFunctions(t *testing.T) {
 		{
 			Name:                "nil handler",
 			inputHandler:        nil,
-			expectedEslTable:    false,
-			expectedOtherTables: false,
+			expectedEslTable:    true,
+			expectedOtherTables: true,
 		},
 		{
-			Name: "esl only",
-			inputHandler: &DBHandler{
-				WriteEslOnly: true,
-			},
+			Name:                "esl only",
+			inputHandler:        &DBHandler{},
 			expectedEslTable:    true,
-			expectedOtherTables: false,
+			expectedOtherTables: true,
 		},
 		{
-			Name: "other tables",
-			inputHandler: &DBHandler{
-				WriteEslOnly: false,
-			},
+			Name:                "other tables",
+			inputHandler:        &DBHandler{},
 			expectedEslTable:    true,
 			expectedOtherTables: true,
 		},

services/cd-service/pkg/argocd/reposerver/reposerver_test.go

@@ -587,7 +587,7 @@ func setupRepository(t *testing.T) (repository.Repository, repository.Repository
 	return repo, cfg
 }
 
-func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) (repository.Repository, *repository.RepositoryConfig) {
+func SetupRepositoryTestWithDBOptions(t *testing.T) (repository.Repository, *repository.RepositoryConfig) {
 	ctx := context.Background()
 	migrationsPath, err := testutil.CreateMigrationsPath(5)
 	if err != nil {
@@ -596,7 +596,6 @@ func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) (reposito
 	dbConfig := &db.DBConfig{
 		DriverName:     "sqlite3",
 		MigrationsPath: migrationsPath,
-		WriteEslOnly:   writeEslOnly,
 	}
 
 	dir := t.TempDir()

services/cd-service/pkg/cmd/server.go

@@ -97,7 +97,6 @@ type Config struct {
 	DbAuthProxyPort       string `default:"5432" split_words:"true"`
 	DbMigrationsLocation  string `default:"" split_words:"true"`
 	DexDefaultRoleEnabled bool   `default:"false" split_words:"true"`
-	DbWriteEslTableOnly   bool   `default:"false" split_words:"true"`
 	DbMaxIdleConnections  uint   `required:"true" split_words:"true"`
 	DbMaxOpenConnections  uint   `required:"true" split_words:"true"`
 
@@ -265,7 +264,6 @@ func RunServer() {
 					DbPassword:     c.DbUserPassword,
 					DbUser:         c.DbUserName,
 					MigrationsPath: c.DbMigrationsLocation,
-					WriteEslOnly:   c.DbWriteEslTableOnly,
 					SSLMode:        c.DbSslMode,
 
 					MaxIdleConnections: c.DbMaxIdleConnections,
@@ -280,7 +278,6 @@ func RunServer() {
 					DbPassword:     c.DbUserPassword,
 					DbUser:         c.DbUserName,
 					MigrationsPath: c.DbMigrationsLocation,
-					WriteEslOnly:   c.DbWriteEslTableOnly,
 					SSLMode:        c.DbSslMode,
 
 					MaxIdleConnections: c.DbMaxIdleConnections,

services/cd-service/pkg/repository/repository.go

@@ -866,7 +866,7 @@ func (r *repository) ApplyTransformersInternal(ctx context.Context, transaction
 				}
 				t.SetEslVersion(db.TransformerID(internal.EslVersion))
 
-				if r.DB != nil && r.DB.WriteEslOnly {
+				if r.DB != nil {
 					// if we were previously running with `db.writeEslTableOnly=true`, but now are running with
 					// `db.writeEslTableOnly=false` (which is the recommended way to enable the database),
 					// then we would have many events in the event_sourcing_light table that have not been processed.

services/cd-service/pkg/repository/transformer_test.go

@@ -7038,10 +7038,10 @@ func makeTransformersForDelete(numVersions uint64) []Transformer {
 }
 
 func SetupRepositoryTestWithDB(t *testing.T) Repository {
-	return SetupRepositoryTestWithDBOptions(t, false)
+	return SetupRepositoryTestWithDBOptions(t)
 }
 
-func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) Repository {
+func SetupRepositoryTestWithDBOptions(t *testing.T) Repository {
 	ctx := context.Background()
 	migrationsPath, err := testutil.CreateMigrationsPath(4)
 	if err != nil {
@@ -7050,7 +7050,6 @@ func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) Repositor
 	dbConfig := &db.DBConfig{
 		DriverName:     "sqlite3",
 		MigrationsPath: migrationsPath,
-		WriteEslOnly:   writeEslOnly,
 	}
 
 	dir := t.TempDir()

services/cd-service/pkg/service/esl_test.go

@@ -108,7 +108,6 @@ func TestGetFailedEslsService(t *testing.T) {
 			dbConfig := &db.DBConfig{
 				DriverName:     "sqlite3",
 				MigrationsPath: migrationsPath,
-				WriteEslOnly:   false,
 			}
 			repo, err := setupRepositoryTestWithDB(t, dbConfig)
 			if err != nil {

services/cd-service/pkg/service/overview_test.go

@@ -769,7 +769,6 @@ func TestOverviewService(t *testing.T) {
 				dbConfig := &db.DBConfig{
 					DriverName:     "sqlite3",
 					MigrationsPath: migrationsPath,
-					WriteEslOnly:   false,
 				}
 				repo, err = setupRepositoryTestWithDB(t, dbConfig)
 				if err != nil {
@@ -934,7 +933,6 @@ func TestGetApplicationDetails(t *testing.T) {
 			dbConfig := &db.DBConfig{
 				DriverName:     "sqlite3",
 				MigrationsPath: migrationsPath,
-				WriteEslOnly:   false,
 			}
 			repo, err = setupRepositoryTestWithDB(t, dbConfig)
 			if err != nil {

services/cloudrun-service/pkg/cmd/server.go

@@ -87,7 +87,6 @@ func runServer(ctx context.Context) error {
 			DbPassword:     dbPassword,
 			DbUser:         dbUserName,
 			MigrationsPath: "",
-			WriteEslOnly:   false,
 			SSLMode:        "verify-full",
 
 			MaxIdleConnections: dbMaxIdle,

services/manifest-repo-export-service/pkg/cmd/server.go

@@ -190,7 +190,6 @@ func Run(ctx context.Context) error {
 			DbPassword:     dbPassword,
 			DbUser:         dbUserName,
 			MigrationsPath: "",
-			WriteEslOnly:   false,
 			SSLMode:        sslMode,
 
 			MaxIdleConnections: dbMaxIdle,
@@ -205,7 +204,6 @@ func Run(ctx context.Context) error {
 			DbPassword:     dbPassword,
 			DbUser:         dbUserName,
 			MigrationsPath: "",
-			WriteEslOnly:   false,
 			SSLMode:        sslMode,
 
 			MaxIdleConnections: dbMaxIdle,

tests/integration-tests/release_test.go

@@ -337,14 +337,13 @@ func TestEnvironmentLock(t *testing.T) {
 			appName:                   "test-app",
 			expectedBodyCreateRelease: "{\"Success\":{}}\n",
 			dbConfig: db.DBConfig{
-				DbName:       "kuberpult",
-				DbUser:       "postgres",
-				DbHost:       "localhost",
-				DbPort:       "5432",
-				DbPassword:   "mypassword",
-				WriteEslOnly: false,
-				DriverName:   "postgres",
-				SSLMode:      "disable",
+				DbName:     "kuberpult",
+				DbUser:     "postgres",
+				DbHost:     "localhost",
+				DbPort:     "5432",
+				DbPassword: "mypassword",
+				DriverName: "postgres",
+				SSLMode:    "disable",
 			},
 		},
 	}