diff --git a/charts/kuberpult/templates/cd-service.yaml b/charts/kuberpult/templates/cd-service.yaml index 8eb29e17da..c3338abfc6 100644 --- a/charts/kuberpult/templates/cd-service.yaml +++ b/charts/kuberpult/templates/cd-service.yaml @@ -18,11 +18,8 @@ {{- if .Values.cd.tag }} {{ fail "Values.cd.tag cannot be used anymore. We only support the same appVersion for all services at this point."}} {{ end -}} -{{- if not (or (eq .Values.db.dbOption "postgreSQL") (eq .Values.db.dbOption "NO_DB")) }} -{{ fail ".Values.db.dbOption does not contain a valid value (NO_DB, postgreSQL)."}} -{{ end -}} -{{- if (and (.Values.db.cloudSqlProxyEnabled) (eq .Values.db.dbOption "NO_DB")) }} -{{ fail "Cloudsql proxy cannot be used with NO_DB option"}} +{{- if not (eq .Values.db.dbOption "postgreSQL") }} +{{ fail ".Values.db.dbOption does not contain a valid value (Only postgreSQL is allowed from now on)."}} {{ end -}} --- apiVersion: apps/v1 @@ -241,10 +238,8 @@ spec: value: "{{ .Values.datadogProfiling.enabled }}" - name: KUBERPULT_MAXIMUM_QUEUE_SIZE value: "{{ .Values.cd.backendConfig.queueSize }}" - - name: KUBERPULT_DB_OPTION # { NO_DB, postgreSQL} + - name: KUBERPULT_DB_OPTION # { postgreSQL} value: {{ .Values.db.dbOption }} - - name: KUBERPULT_DB_WRITE_ESL_TABLE_ONLY - value: "{{ .Values.db.writeEslTableOnly }}" {{- if (eq .Values.db.dbOption "postgreSQL") }} - name: KUBERPULT_DB_LOCATION value: {{ .Values.db.location }} diff --git a/charts/kuberpult/values.yaml b/charts/kuberpult/values.yaml index 82dbe65636..e0c2638a61 100644 --- a/charts/kuberpult/values.yaml +++ b/charts/kuberpult/values.yaml @@ -94,9 +94,9 @@ cd: create: false # Add backend config for health checks on GKE only timeoutSec: 300 # 30sec is the default on gcp loadbalancers, however kuberpult needs more with parallel requests. It is the time how long the loadbalancer waits for kuberpult to finish calls to the rest endpoint "release" queueSize: 5 -# Disabling the queue is as of now an experimental feature. It's only possible to use with db.writeEslTableOnly=false. +# Disabling the queue is as of now an experimental feature. # With the queue, the cd-service processes only one request at a time, which is very much required when using git. -# With the database enabled, this is not required anymore. +# With the database, this is not required anymore. disableQueue: false resources: limits: @@ -118,29 +118,21 @@ cd: timeoutSeconds: 5 failureThreshold: 10 initialDelaySeconds: 5 -# The Database is not ready for use on production and the helm options might change in the near future. db: location: "127.0.0.1" authProxyPort: 5432 -# As the Database feature is still work in progress, for now we recommend to set: -# dbOption: "NO_DB" -# Other valid values are "postgreSQL". -# If you do want to try out the database, for now use -# writeEslTableOnly: true - dbOption: "NO_DB" -# k8sServiceAccountName is required if `dbOption = "postgreSQL"`, otherwise it's ignored. +# The Database feature is now required: + dbOption: "postgreSQL" +# k8sServiceAccountName is required. # k8sServiceAccountName is the name of the kubernetes service account. k8sServiceAccountName: "k8sServiceAccountName" -# cloudSqlProxyEnabled enables the cloudsql proxy. Not possible with `dbOption = "NO_DB"`. +# cloudSqlProxyEnabled enables the cloudsql proxy. cloudSqlProxyEnabled: false dbConnectionName: "connectioname" dbName: "databaseName" dbUser: "username" dbPassword: "password" migrations: /migrations - # If set to true, kuberpult will write only the ESL table. - # This is useful to already collect historical data in the database, while waiting for the full database implementation. - writeEslTableOnly: false # SSL mode to be used by the database sslMode: verify-full requests: diff --git a/docs/database.md b/docs/database.md index 69cf3578ba..1d8fc048b3 100644 --- a/docs/database.md +++ b/docs/database.md @@ -1,15 +1,11 @@ # Database -## The Database feature is not ready for production yet - -However, you can already prepare for the database feature, -see [Preparation](#preparation) - +## The Database feature is now required ## Background -Kuberpult is switching over to use a database. The rough timeline to have the database production-ready is summer 2024. -As of now Kuberpult is using the manifest repository to store data. +Kuberpult switched over to use a database. +In the past, Kuberpult was using the manifest repository to store data. This worked fine for a while, and it had the added bonus that ArgoCD is reading from the same manifest git repository. @@ -21,10 +17,6 @@ Therefore, we will use a database and not rely on git anymore in the future. Git will still be used as an *output* of kuberpult, but not as the source of truth. -As of now, kuberpult still supports the option to not have the database, -but this option will be removed in a few weeks with another breaking change in kuberpult. - - ## Preparation Our recommendation is to enable the database mode in 2 steps: @@ -32,6 +24,7 @@ Our recommendation is to enable the database mode in 2 steps: ### Step 1: writeEslTableOnly=true Enable the Database with `db.dbOption: "postgreSQL"` and `db.writeEslTableOnly: true`. +This requires kuberpult version <= 10.3.10. This means that kuberpult will connect to the DB, but only write one table. Kuberpult will not read from the database in this state, so the manifest-repository is still considered the source of truth. diff --git a/pkg/db/db.go b/pkg/db/db.go index 1d810800a9..7cc26af0d2 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -54,7 +54,6 @@ type DBConfig struct { DriverName string DbPassword string MigrationsPath string - WriteEslOnly bool SSLMode string MaxIdleConnections uint @@ -70,14 +69,6 @@ type DBHandler struct { DB *sql.DB DBDriver *database.Driver - /* - There are 3 modes: - 1) DBHandler==nil: do not write anything to the DB - 2) DBHandler!=nil && WriteEslOnly==true: write only the ESL table to the database. Stores all incoming data in the DB, but does not read the DB. - 3) DBHandler!=nil && WriteEslOnly==false: write everything to the database. - */ - WriteEslOnly bool - // InsertAppFun is intended to be used to add more to inserting an app: specifically to update the overview cache InsertAppFun InsertAppFun } @@ -102,13 +93,11 @@ const ( ) func (h *DBHandler) ShouldUseEslTable() bool { - return h != nil + return true } -// ShouldUseOtherTables returns true if the db is enabled and WriteEslOnly=false -// ShouldUseOtherTables should never be used in the manifest-repo-export-service. func (h *DBHandler) ShouldUseOtherTables() bool { - return h != nil && !h.WriteEslOnly + return true } func Connect(ctx context.Context, cfg DBConfig) (*DBHandler, error) { @@ -123,7 +112,6 @@ func Connect(ctx context.Context, cfg DBConfig) (*DBHandler, error) { MigrationsPath: cfg.MigrationsPath, DB: db, DBDriver: &driver, - WriteEslOnly: cfg.WriteEslOnly, InsertAppFun: nil, } handler.InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error { diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index 63dc1813fd..0ddb9f10d3 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -763,22 +763,18 @@ func TestHelperFunctions(t *testing.T) { { Name: "nil handler", inputHandler: nil, - expectedEslTable: false, - expectedOtherTables: false, + expectedEslTable: true, + expectedOtherTables: true, }, { - Name: "esl only", - inputHandler: &DBHandler{ - WriteEslOnly: true, - }, + Name: "esl only", + inputHandler: &DBHandler{}, expectedEslTable: true, - expectedOtherTables: false, + expectedOtherTables: true, }, { - Name: "other tables", - inputHandler: &DBHandler{ - WriteEslOnly: false, - }, + Name: "other tables", + inputHandler: &DBHandler{}, expectedEslTable: true, expectedOtherTables: true, }, diff --git a/services/cd-service/pkg/argocd/reposerver/reposerver_test.go b/services/cd-service/pkg/argocd/reposerver/reposerver_test.go index 9cabd7793c..b7617c2711 100644 --- a/services/cd-service/pkg/argocd/reposerver/reposerver_test.go +++ b/services/cd-service/pkg/argocd/reposerver/reposerver_test.go @@ -587,7 +587,7 @@ func setupRepository(t *testing.T) (repository.Repository, repository.Repository return repo, cfg } -func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) (repository.Repository, *repository.RepositoryConfig) { +func SetupRepositoryTestWithDBOptions(t *testing.T) (repository.Repository, *repository.RepositoryConfig) { ctx := context.Background() migrationsPath, err := testutil.CreateMigrationsPath(5) if err != nil { @@ -596,7 +596,6 @@ func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) (reposito dbConfig := &db.DBConfig{ DriverName: "sqlite3", MigrationsPath: migrationsPath, - WriteEslOnly: writeEslOnly, } dir := t.TempDir() diff --git a/services/cd-service/pkg/cmd/server.go b/services/cd-service/pkg/cmd/server.go index 2ccf322115..4cef919b4f 100755 --- a/services/cd-service/pkg/cmd/server.go +++ b/services/cd-service/pkg/cmd/server.go @@ -97,7 +97,6 @@ type Config struct { DbAuthProxyPort string `default:"5432" split_words:"true"` DbMigrationsLocation string `default:"" split_words:"true"` DexDefaultRoleEnabled bool `default:"false" split_words:"true"` - DbWriteEslTableOnly bool `default:"false" split_words:"true"` DbMaxIdleConnections uint `required:"true" split_words:"true"` DbMaxOpenConnections uint `required:"true" split_words:"true"` @@ -265,7 +264,6 @@ func RunServer() { DbPassword: c.DbUserPassword, DbUser: c.DbUserName, MigrationsPath: c.DbMigrationsLocation, - WriteEslOnly: c.DbWriteEslTableOnly, SSLMode: c.DbSslMode, MaxIdleConnections: c.DbMaxIdleConnections, @@ -280,7 +278,6 @@ func RunServer() { DbPassword: c.DbUserPassword, DbUser: c.DbUserName, MigrationsPath: c.DbMigrationsLocation, - WriteEslOnly: c.DbWriteEslTableOnly, SSLMode: c.DbSslMode, MaxIdleConnections: c.DbMaxIdleConnections, diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index 6e5bd359c5..7b7768aeb4 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -866,7 +866,7 @@ func (r *repository) ApplyTransformersInternal(ctx context.Context, transaction } t.SetEslVersion(db.TransformerID(internal.EslVersion)) - if r.DB != nil && r.DB.WriteEslOnly { + if r.DB != nil { // if we were previously running with `db.writeEslTableOnly=true`, but now are running with // `db.writeEslTableOnly=false` (which is the recommended way to enable the database), // then we would have many events in the event_sourcing_light table that have not been processed. diff --git a/services/cd-service/pkg/repository/transformer_test.go b/services/cd-service/pkg/repository/transformer_test.go index 385142846a..813b14da67 100644 --- a/services/cd-service/pkg/repository/transformer_test.go +++ b/services/cd-service/pkg/repository/transformer_test.go @@ -7038,10 +7038,10 @@ func makeTransformersForDelete(numVersions uint64) []Transformer { } func SetupRepositoryTestWithDB(t *testing.T) Repository { - return SetupRepositoryTestWithDBOptions(t, false) + return SetupRepositoryTestWithDBOptions(t) } -func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) Repository { +func SetupRepositoryTestWithDBOptions(t *testing.T) Repository { ctx := context.Background() migrationsPath, err := testutil.CreateMigrationsPath(4) if err != nil { @@ -7050,7 +7050,6 @@ func SetupRepositoryTestWithDBOptions(t *testing.T, writeEslOnly bool) Repositor dbConfig := &db.DBConfig{ DriverName: "sqlite3", MigrationsPath: migrationsPath, - WriteEslOnly: writeEslOnly, } dir := t.TempDir() diff --git a/services/cd-service/pkg/service/esl_test.go b/services/cd-service/pkg/service/esl_test.go index eb1e8dc20e..a66e46eb26 100644 --- a/services/cd-service/pkg/service/esl_test.go +++ b/services/cd-service/pkg/service/esl_test.go @@ -108,7 +108,6 @@ func TestGetFailedEslsService(t *testing.T) { dbConfig := &db.DBConfig{ DriverName: "sqlite3", MigrationsPath: migrationsPath, - WriteEslOnly: false, } repo, err := setupRepositoryTestWithDB(t, dbConfig) if err != nil { diff --git a/services/cd-service/pkg/service/overview_test.go b/services/cd-service/pkg/service/overview_test.go index 0b653cab2a..8dc77fcd57 100644 --- a/services/cd-service/pkg/service/overview_test.go +++ b/services/cd-service/pkg/service/overview_test.go @@ -769,7 +769,6 @@ func TestOverviewService(t *testing.T) { dbConfig := &db.DBConfig{ DriverName: "sqlite3", MigrationsPath: migrationsPath, - WriteEslOnly: false, } repo, err = setupRepositoryTestWithDB(t, dbConfig) if err != nil { @@ -934,7 +933,6 @@ func TestGetApplicationDetails(t *testing.T) { dbConfig := &db.DBConfig{ DriverName: "sqlite3", MigrationsPath: migrationsPath, - WriteEslOnly: false, } repo, err = setupRepositoryTestWithDB(t, dbConfig) if err != nil { diff --git a/services/cloudrun-service/pkg/cmd/server.go b/services/cloudrun-service/pkg/cmd/server.go index 9e7aee7009..4869dd3315 100644 --- a/services/cloudrun-service/pkg/cmd/server.go +++ b/services/cloudrun-service/pkg/cmd/server.go @@ -87,7 +87,6 @@ func runServer(ctx context.Context) error { DbPassword: dbPassword, DbUser: dbUserName, MigrationsPath: "", - WriteEslOnly: false, SSLMode: "verify-full", MaxIdleConnections: dbMaxIdle, diff --git a/services/manifest-repo-export-service/pkg/cmd/server.go b/services/manifest-repo-export-service/pkg/cmd/server.go index ac10c4b97a..a91eefb4e5 100755 --- a/services/manifest-repo-export-service/pkg/cmd/server.go +++ b/services/manifest-repo-export-service/pkg/cmd/server.go @@ -190,7 +190,6 @@ func Run(ctx context.Context) error { DbPassword: dbPassword, DbUser: dbUserName, MigrationsPath: "", - WriteEslOnly: false, SSLMode: sslMode, MaxIdleConnections: dbMaxIdle, @@ -205,7 +204,6 @@ func Run(ctx context.Context) error { DbPassword: dbPassword, DbUser: dbUserName, MigrationsPath: "", - WriteEslOnly: false, SSLMode: sslMode, MaxIdleConnections: dbMaxIdle, diff --git a/tests/integration-tests/release_test.go b/tests/integration-tests/release_test.go index 513dcb3a6b..7872abaebd 100644 --- a/tests/integration-tests/release_test.go +++ b/tests/integration-tests/release_test.go @@ -337,14 +337,13 @@ func TestEnvironmentLock(t *testing.T) { appName: "test-app", expectedBodyCreateRelease: "{\"Success\":{}}\n", dbConfig: db.DBConfig{ - DbName: "kuberpult", - DbUser: "postgres", - DbHost: "localhost", - DbPort: "5432", - DbPassword: "mypassword", - WriteEslOnly: false, - DriverName: "postgres", - SSLMode: "disable", + DbName: "kuberpult", + DbUser: "postgres", + DbHost: "localhost", + DbPort: "5432", + DbPassword: "mypassword", + DriverName: "postgres", + SSLMode: "disable", }, }, }