feat(git): Allow exporting only strictly required data for Argo CD

Ref: SRX-2CSAEK

charts/kuberpult/templates/manifest-repo-export-service.yaml

@@ -231,6 +231,8 @@ spec:
           value: "{{ .Values.datadogProfiling.enabled }}"
         - name: KUBERPULT_RELEASE_VERSIONS_LIMIT
           value: "{{ .Values.git.releaseVersionsLimit }}"
+        - name: KUBERPULT_MINIMIZE_EXPORTED_DATA
+          value: "{{ .Values.git.minimizeExportedData }}"
         volumeMounts:
         - name: repository
           # The repository volume, an emptyDir, is mounted to the kp directory.

charts/kuberpult/tests/charts_test.go

@@ -1051,6 +1051,47 @@ manifestRepoExport:
 			},
 			ExpectedMissing: []core.EnvVar{},
 		},
+		{
+			Name: "Git MinimizeExportData mode explicitely set",
+			Values: `
+git:
+  url:  "testURL"
+  minimizeExportedData: true
+ingress:
+  domainName: "kuberpult-example.com"
+db:
+  dbOption: "postgreSQL"
+  writeEslTableOnly: false
+  sslMode: disable
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_MINIMIZE_EXPORTED_DATA",
+					Value: "true",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{},
+		},
+		{
+			Name: "Git MinimizeExportData mode default value",
+			Values: `
+git:
+  url:  "testURL"
+ingress:
+  domainName: "kuberpult-example.com"
+db:
+  dbOption: "postgreSQL"
+  writeEslTableOnly: false
+  sslMode: disable
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_MINIMIZE_EXPORTED_DATA",
+					Value: "false",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{},
+		},
 		{
 			Name: "DB ssl mode",
 			Values: `

charts/kuberpult/values.yaml

@@ -67,6 +67,12 @@ git:
   # Values outside of the range will fail the service during startup.
   releaseVersionsLimit: 20
 
+  # When minimizeExportedData==false, all data will be written to git, including meta information that is not relevant for argoCd, like locks.
+  # When true, only the files needed for Argo CD to work are written.
+  # Note that when this option is true, in case of data loss in the database, kuberpult can *not* recover all information from the git repo.
+  # Recommendation: Only activate this option, if frequent database backups are made and if the git repo takes up too many resources.
+  minimizeExportedData: false
+
 hub: europe-west3-docker.pkg.dev/fdc-public-docker-registry/kuberpult
 kubernetesEngine: "GKE"
 

docker-compose.yml

@@ -1,4 +1,20 @@
 services:
+  datadog-agent:
+    image: datadog/agent:latest
+    env_file:
+      - docker.env
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+    ports:
+      - 127.0.0.1:8126:8126/tcp
+    environment:
+      - DD_APM_ENABLED=true
+      - DD_APM_NON_LOCAL_TRAFFIC=true
+      - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
+      - DD_AC_EXCLUDE=name:datadog-agent
+    stop_grace_period: 2.0s
   postgres:
     image: postgres:13.15
     restart: unless-stopped
@@ -32,14 +48,20 @@ services:
       - KUBERPULT_DEX_RBAC_POLICY_PATH=/etc/policy.csv
       - KUBERPULT_DEX_RBAC_TEAM_PATH=/etc/team.csv
       - KUBERPULT_DEX_MOCK_ROLE=developer
-      - KUBERPULT_GIT_WRITE_COMMIT_DATA=true
+      - KUBERPULT_GIT_WRITE_COMMIT_DATA=false
       - KUBERPULT_MAXIMUM_QUEUE_SIZE=2
       - KUBERPULT_ALLOW_LONG_APP_NAMES=true
       - KUBERPULT_ARGO_CD_GENERATE_FILES=true
       - KUBERPULT_DB_SSL_MODE=disable
       - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=10
       - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=5
       - KUBERPULT_ALLOWED_DOMAINS=localhost
+      - KUBERPULT_ENABLE_METRICS=true
+      - DD_AGENT_HOST=datadog-agent
+      - KUBERPULT_ENABLE_TRACING=true
+      - DD_SERVICE=kuberpult-cd-service
+      - DD_VERSION=0.0.0
+      - DD_ENV=example-su-oct-30-b
     ports:
       - "8080:8080"
       - "8443:8443"
@@ -48,6 +70,8 @@ services:
       - ./database:/kp/database
     stop_grace_period: 0.5s
     depends_on:
+      datadog-agent:
+        condition: service_healthy
       postgres:
         condition: service_healthy
   manifest-repo-export-service:
@@ -74,6 +98,7 @@ services:
       - KUBERPULT_DB_SSL_MODE=disable
       - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=5
       - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=1
+      - KUBERPULT_MINIMIZE_EXPORTED_DATA=false
     volumes:
       - ./services/cd-service:/kp/kuberpult
       - ./database:/kp/database
@@ -119,8 +144,8 @@ services:
     container_name: kuberpult-ui
     ports:
       - "3000:3000"
-    depends_on:
-      - frontend-service
+#    depends_on:
+#      - frontend-service
     volumes:
       - ./services/frontend-service/src:/kp/src
       - ./services/frontend-service/public:/kp/public

pkg/db/db.go

@@ -883,6 +883,33 @@ func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context
 	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false)
 }
 
+func (h *DBHandler) DBSelectLastReleasesByApp(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) {
+	span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByApp")
+	defer span.Finish()
+	selectQuery := h.AdaptQuery(fmt.Sprintf(
+		"SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " +
+			" FROM releases " +
+			" WHERE appName=? AND deleted=?" +
+			" ORDER BY eslVersion DESC, releaseVersion DESC" +
+			" LIMIT 1;"))
+	span.SetTag("query", selectQuery)
+	rows, err := tx.QueryContext(
+		ctx,
+		selectQuery,
+		app,
+		deleted,
+	)
+
+	releases, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false)
+	if err != nil {
+		return nil, err
+	}
+	if len(releases) == 0 {
+		return nil, nil
+	}
+	return releases[0], nil
+}
+
 func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectAllReleasesOfApp")
 	defer span.Finish()
@@ -2286,6 +2313,9 @@ type DBAppWithMetaData struct {
 func (h *DBHandler) DBInsertApplication(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error {
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBInsertApplication")
 	defer span.Finish()
+	log := logger.FromContext(ctx).Sugar()
+	log.Warnf("plain dbinsert app: %s/%v", appName, stateChange)
+
 	jsonToInsert, err := json.Marshal(metaData)
 	if err != nil {
 		return fmt.Errorf("could not marshal json data: %w", err)
@@ -2418,14 +2448,32 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro
 	if err != nil {
 		return nil, fmt.Errorf("could not query apps table from DB. Error: %w\n", err)
 	}
+	return processAppRow(ctx, rows)
+}
+
+func (h *DBHandler) DBSelectExistingApp(ctx context.Context, tx *sql.Tx, appName string) (*DBAppWithMetaData, error) {
+	app, err := h.DBSelectApp(ctx, tx, appName)
+	if err != nil {
+		return nil, err
+	}
+	if app == nil {
+		return nil, nil
+	}
+	if app.StateChange == AppStateChangeDelete {
+		return nil, nil
+	}
+	return app, nil
+}
+
+func processAppRow(ctx context.Context, rows *sql.Rows) (*DBAppWithMetaData, error) {
+	//exhaustruct:ignore
 	defer func(rows *sql.Rows) {
 		err := rows.Close()
 		if err != nil {
 			logger.FromContext(ctx).Sugar().Warnf("row could not be closed: %v", err)
 		}
 	}(rows)
 
-	//exhaustruct:ignore
 	var row = &DBAppWithMetaData{}
 	if rows.Next() {
 		var metadataStr string
@@ -2445,7 +2493,7 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro
 	} else {
 		row = nil
 	}
-	err = closeRows(rows)
+	err := closeRows(rows)
 	if err != nil {
 		return nil, err
 	}

pkg/db/overview.go

@@ -114,6 +114,63 @@ func (h *DBHandler) UpdateOverviewEnvironmentLock(ctx context.Context, transacti
 	return nil
 }
 
+func (h *DBHandler) UpdateOverviewDeployment(ctx context.Context, transaction *sql.Tx, deployment Deployment, createdTime time.Time) error {
+	latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction)
+	if err != nil {
+		return err
+	}
+	if h.IsOverviewEmpty(latestOverview) {
+		return nil
+	}
+	env := getEnvironmentByName(latestOverview.EnvironmentGroups, deployment.Env)
+	if env == nil {
+		return fmt.Errorf("could not find environment %s in overview", deployment.Env)
+	}
+
+	selectApp, err := h.DBSelectExistingApp(ctx, transaction, deployment.App)
+	if err != nil {
+		return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", deployment.App, err)
+	}
+	if selectApp == nil {
+		return fmt.Errorf("could not find application '%s' in apps table: got no result", deployment.App)
+	}
+
+	err = h.WriteOverviewCache(ctx, transaction, latestOverview)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (h *DBHandler) UpdateOverviewDeploymentAttempt(ctx context.Context, transaction *sql.Tx, queuedDeployment *QueuedDeployment) error {
+	latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction)
+	if err != nil {
+		return err
+	}
+	if h.IsOverviewEmpty(latestOverview) {
+		return nil
+	}
+	if queuedDeployment == nil {
+		return nil
+	}
+	env := getEnvironmentByName(latestOverview.EnvironmentGroups, queuedDeployment.Env)
+	if env == nil {
+		return fmt.Errorf("could not find environment %s in overview", queuedDeployment.Env)
+	}
+	selectApp, err := h.DBSelectExistingApp(ctx, transaction, queuedDeployment.App)
+	if err != nil {
+		return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", queuedDeployment.App, err)
+	}
+	if selectApp == nil {
+		return fmt.Errorf("could not find application '%s' in apps table: got no result", queuedDeployment.App)
+	}
+	err = h.WriteOverviewCache(ctx, transaction, latestOverview)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transaction *sql.Tx, applicationLock ApplicationLock, createdTime time.Time) error {
 	latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction)
 	if err != nil {
@@ -126,7 +183,7 @@ func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transacti
 	if env == nil {
 		return fmt.Errorf("could not find environment %s in overview", applicationLock.Env)
 	}
-	selectApp, err := h.DBSelectApp(ctx, transaction, applicationLock.App)
+	selectApp, err := h.DBSelectExistingApp(ctx, transaction, applicationLock.App)
 	if err != nil {
 		return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", applicationLock.App, err)
 	}

pkg/valid/validations.go

@@ -105,3 +105,11 @@ func ReadEnvVarUInt(envName string) (uint, error) {
 	}
 	return uint(i), nil
 }
+
+func ReadEnvVarBool(envName string) (bool, error) {
+	envValue, err := ReadEnvVar(envName)
+	if err != nil {
+		return false, err
+	}
+	return envValue == "true", nil
+}

services/cd-service/pkg/repository/repository.go

@@ -2462,6 +2462,9 @@ func (s *State) WriteAllCommitEvents(ctx context.Context, transaction *sql.Tx, d
 }
 
 func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion db.EslVersion, stateChange db.AppStateChange, metaData db.DBAppMetaData) error {
+	log := logger.FromContext(ctx).Sugar()
+	log.Warnf("dbinsert app with overview: %s/%v", appName, stateChange)
+
 	h := s.DBHandler
 	err := h.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData)
 	if err != nil {
@@ -2478,12 +2481,11 @@ func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction
 	}
 
 	shouldDelete := stateChange == db.AppStateChangeDelete
-
 	if shouldDelete {
 		lApps := make([]*api.OverviewApplication, 0)
 
 		for _, curr := range cache.LightweightApps {
-			if curr.Name != appName {
+			if curr != nil && curr.Name != appName {
 				lApps = append(lApps, curr)
 			}
 		}
@@ -2533,6 +2535,30 @@ func (s *State) DBInsertEnvironmentWithOverview(ctx context.Context, tx *sql.Tx,
 	return nil
 }
 
+func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sql.Tx, appName string, result *api.GetOverviewResponse, deleteApp bool, allReleasesOfAllApps map[string][]int64) error {
+	if deleteApp {
+		removeOverviewAppFromLightweightApps(result, appName)
+		return nil
+	}
+	team, err := s.GetApplicationTeamOwner(ctx, transaction, appName)
+	if err != nil {
+		return fmt.Errorf("could not obtain application team owner to update top level app in overview: %w", err)
+	}
+	result.LightweightApps = append(result.LightweightApps, &api.OverviewApplication{Name: appName, Team: team})
+	return nil
+}
+
+func removeOverviewAppFromLightweightApps(result *api.GetOverviewResponse, appName string) {
+	lApps := make([]*api.OverviewApplication, max(len(result.LightweightApps)-1, 0))
+
+	for _, curr := range result.LightweightApps {
+		if curr != nil && curr.Name != appName {
+			lApps = append(lApps, curr)
+		}
+	}
+	result.LightweightApps = lApps
+}
+
 func getEnvironmentInGroup(groups []*api.EnvironmentGroup, groupNameToReturn string, envNameToReturn string) *api.Environment {
 	for _, currentGroup := range groups {
 		if currentGroup.EnvironmentGroupName == groupNameToReturn {

services/cd-service/pkg/repository/transformer.go

@@ -1493,8 +1493,8 @@ func (u *UndeployApplication) GetDBEventType() db.EventType {
 	return db.EvtUndeployApplication
 }
 
-func (c *UndeployApplication) SetEslVersion(id db.TransformerID) {
-	c.TransformerEslVersion = id
+func (u *UndeployApplication) SetEslVersion(id db.TransformerID) {
+	u.TransformerEslVersion = id
 }
 
 func (u *UndeployApplication) Transform(
@@ -1616,10 +1616,11 @@ func (u *UndeployApplication) Transform(
 		if err != nil {
 			return "", fmt.Errorf("UndeployApplication: could not write all apps '%v': '%w'", u.Application, err)
 		}
-		dbApp, err := state.DBHandler.DBSelectApp(ctx, transaction, u.Application)
+		dbApp, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, u.Application)
 		if err != nil {
 			return "", fmt.Errorf("UndeployApplication: could not select app '%s': %v", u.Application, err)
 		}
+		logger.FromContext(ctx).Sugar().Warnf("before insert app fun")
 		err = state.DBHandler.InsertAppFun(ctx, transaction, dbApp.App, dbApp.EslVersion, db.AppStateChangeDelete, db.DBAppMetaData{Team: dbApp.Metadata.Team})
 		if err != nil {
 			return "", fmt.Errorf("UndeployApplication: could not insert app '%s': %v", u.Application, err)