feat(git): Allow exporting only strictly required data for Argo CD (#…

…2049)

Ref: SRX-2CSAEK

charts/kuberpult/templates/manifest-repo-export-service.yaml

@@ -231,6 +231,8 @@ spec:
           value: "{{ .Values.datadogProfiling.enabled }}"
         - name: KUBERPULT_RELEASE_VERSIONS_LIMIT
           value: "{{ .Values.git.releaseVersionsLimit }}"
+        - name: KUBERPULT_MINIMIZE_EXPORTED_DATA
+          value: "{{ .Values.git.minimizeExportedData }}"
         volumeMounts:
         - name: repository
           # The repository volume, an emptyDir, is mounted to the kp directory.

charts/kuberpult/tests/charts_test.go

@@ -1051,6 +1051,47 @@ manifestRepoExport:
 			},
 			ExpectedMissing: []core.EnvVar{},
 		},
+		{
+			Name: "Git MinimizeExportData mode explicitely set",
+			Values: `
+git:
+  url:  "testURL"
+  minimizeExportedData: true
+ingress:
+  domainName: "kuberpult-example.com"
+db:
+  dbOption: "postgreSQL"
+  writeEslTableOnly: false
+  sslMode: disable
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_MINIMIZE_EXPORTED_DATA",
+					Value: "true",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{},
+		},
+		{
+			Name: "Git MinimizeExportData mode default value",
+			Values: `
+git:
+  url:  "testURL"
+ingress:
+  domainName: "kuberpult-example.com"
+db:
+  dbOption: "postgreSQL"
+  writeEslTableOnly: false
+  sslMode: disable
+`,
+			ExpectedEnvs: []core.EnvVar{
+				{
+					Name:  "KUBERPULT_MINIMIZE_EXPORTED_DATA",
+					Value: "false",
+				},
+			},
+			ExpectedMissing: []core.EnvVar{},
+		},
 		{
 			Name: "DB ssl mode",
 			Values: `

charts/kuberpult/values.yaml

@@ -67,6 +67,12 @@ git:
   # Values outside of the range will fail the service during startup.
   releaseVersionsLimit: 20
 
+  # When minimizeExportedData==false, all data will be written to git, including meta information that is not relevant for argoCd, like locks.
+  # When true, only the files needed for Argo CD to work are written.
+  # Note that when this option is true, in case of data loss in the database, kuberpult can *not* recover all information from the git repo.
+  # Recommendation: Only activate this option, if frequent database backups are made and if the git repo takes up too many resources.
+  minimizeExportedData: false
+
 hub: europe-west3-docker.pkg.dev/fdc-public-docker-registry/kuberpult
 kubernetesEngine: "GKE"
 

docker-compose.yml

@@ -32,7 +32,7 @@ services:
       - KUBERPULT_DEX_RBAC_POLICY_PATH=/etc/policy.csv
       - KUBERPULT_DEX_RBAC_TEAM_PATH=/etc/team.csv
       - KUBERPULT_DEX_MOCK_ROLE=developer
-      - KUBERPULT_GIT_WRITE_COMMIT_DATA=true
+      - KUBERPULT_GIT_WRITE_COMMIT_DATA=false
       - KUBERPULT_MAXIMUM_QUEUE_SIZE=2
       - KUBERPULT_ALLOW_LONG_APP_NAMES=true
       - KUBERPULT_ARGO_CD_GENERATE_FILES=true
@@ -74,6 +74,7 @@ services:
       - KUBERPULT_DB_SSL_MODE=disable
       - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=5
       - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=1
+      - KUBERPULT_MINIMIZE_EXPORTED_DATA=true
     volumes:
       - ./services/cd-service:/kp/kuberpult
       - ./database:/kp/database

pkg/db/db.go

@@ -864,23 +864,50 @@ func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, t
 	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true)
 }
 
-func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) ([]*DBReleaseWithMetaData, error) {
+func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByAppOrderedByEslVersion")
 	defer span.Finish()
-	selectQuery := h.AdaptQuery(fmt.Sprintf(
-		"SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " +
-			" FROM releases " +
-			" WHERE appName=? AND deleted=?" +
-			" ORDER BY eslVersion DESC, releaseVersion DESC, created DESC;"))
+	selectQuery := h.AdaptQuery(`
+SELECT
+	releases.eslVersion,
+	releases.created,
+	releases.appName,
+	releases.metadata,
+	releases.releaseVersion,
+	releases.deleted,
+	releases.environments
+FROM (
+ 	SELECT max(eslVersion) as latestEslVersion, appName, releaseVersion
+ 	FROM releases
+ 	WHERE appName=?
+ 	GROUP BY appName, releaseVersion
+ 	) as c
+JOIN releases
+    ON  c.appName= releases.appName
+    AND c.latestEslVersion = releases.eslversion
+    AND c.releaseVersion = releases.releaseVersion
+ORDER BY c.releaseVersion DESC
+LIMIT 1;
+`,
+	)
 	span.SetTag("query", selectQuery)
+	span.SetTag("appName", app)
 	rows, err := tx.QueryContext(
 		ctx,
 		selectQuery,
 		app,
-		deleted,
 	)
 
-	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false)
+	result, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false)
+	span.SetTag("resultCount", len(result))
+	if err != nil {
+		span.Finish(tracer.WithError(err))
+		return nil, err
+	}
+	if result == nil {
+		return nil, nil
+	}
+	return result[0], nil
 }
 
 func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) {
@@ -2287,6 +2314,9 @@ type DBAppWithMetaData struct {
 func (h *DBHandler) DBInsertApplication(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error {
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBInsertApplication")
 	defer span.Finish()
+	log := logger.FromContext(ctx).Sugar()
+	log.Warnf("plain dbinsert app: %s/%v", appName, stateChange)
+
 	jsonToInsert, err := json.Marshal(metaData)
 	if err != nil {
 		return fmt.Errorf("could not marshal json data: %w", err)
@@ -2419,13 +2449,27 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro
 	if err != nil {
 		return nil, fmt.Errorf("could not query apps table from DB. Error: %w\n", err)
 	}
+	return processAppRow(ctx, rows)
+}
+
+func (h *DBHandler) DBSelectExistingApp(ctx context.Context, tx *sql.Tx, appName string) (*DBAppWithMetaData, error) {
+	app, err := h.DBSelectApp(ctx, tx, appName)
+	if err != nil {
+		return nil, err
+	}
+	if app == nil {
+		return nil, nil
+	}
+	return app, nil
+}
+
+func processAppRow(ctx context.Context, rows *sql.Rows) (*DBAppWithMetaData, error) {
 	defer func(rows *sql.Rows) {
 		err := rows.Close()
 		if err != nil {
 			logger.FromContext(ctx).Sugar().Warnf("row could not be closed: %v", err)
 		}
 	}(rows)
-
 	//exhaustruct:ignore
 	var row = &DBAppWithMetaData{}
 	if rows.Next() {
@@ -2443,10 +2487,13 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro
 			return nil, fmt.Errorf("Error during json unmarshal of apps. Error: %w. Data: %s\n", err, metadataStr)
 		}
 		row.Metadata = metaData
+		if row.StateChange == AppStateChangeDelete {
+			return nil, nil
+		}
 	} else {
 		row = nil
 	}
-	err = closeRows(rows)
+	err := closeRows(rows)
 	if err != nil {
 		return nil, err
 	}

pkg/db/overview.go

@@ -126,7 +126,7 @@ func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transacti
 	if env == nil {
 		return fmt.Errorf("could not find environment %s in overview", applicationLock.Env)
 	}
-	selectApp, err := h.DBSelectApp(ctx, transaction, applicationLock.App)
+	selectApp, err := h.DBSelectExistingApp(ctx, transaction, applicationLock.App)
 	if err != nil {
 		return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", applicationLock.App, err)
 	}

pkg/valid/validations.go

@@ -105,3 +105,11 @@ func ReadEnvVarUInt(envName string) (uint, error) {
 	}
 	return uint(i), nil
 }
+
+func ReadEnvVarBool(envName string) (bool, error) {
+	envValue, err := ReadEnvVar(envName)
+	if err != nil {
+		return false, err
+	}
+	return envValue == "true", nil
+}

services/cd-service/pkg/repository/repository.go

@@ -2478,12 +2478,11 @@ func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction
 	}
 
 	shouldDelete := stateChange == db.AppStateChangeDelete
-
 	if shouldDelete {
 		lApps := make([]*api.OverviewApplication, 0)
 
 		for _, curr := range cache.LightweightApps {
-			if curr.Name != appName {
+			if curr != nil && curr.Name != appName {
 				lApps = append(lApps, curr)
 			}
 		}

services/cd-service/pkg/repository/transformer.go

@@ -689,13 +689,13 @@ func (c *CreateApplicationVersion) Transform(
 	sortedKeys := sorting.SortKeys(c.Manifests)
 
 	if state.DBHandler.ShouldUseOtherTables() {
-		prevRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, c.Application, false, false)
+		prevRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, c.Application, false)
 		if err != nil {
 			return "", err
 		}
 		var v = db.InitialEslVersion - 1
-		if len(prevRelease) > 0 {
-			v = prevRelease[0].EslVersion
+		if prevRelease != nil {
+			v = prevRelease.EslVersion
 		}
 		isMinor, err := c.checkMinorFlags(ctx, transaction, state.DBHandler, version, state.MinorRegexes)
 		if err != nil {
@@ -1510,8 +1510,8 @@ func (u *UndeployApplication) GetDBEventType() db.EventType {
 	return db.EvtUndeployApplication
 }
 
-func (c *UndeployApplication) SetEslVersion(id db.TransformerID) {
-	c.TransformerEslVersion = id
+func (u *UndeployApplication) SetEslVersion(id db.TransformerID) {
+	u.TransformerEslVersion = id
 }
 
 func (u *UndeployApplication) Transform(
@@ -1633,7 +1633,7 @@ func (u *UndeployApplication) Transform(
 		if err != nil {
 			return "", fmt.Errorf("UndeployApplication: could not write all apps '%v': '%w'", u.Application, err)
 		}
-		dbApp, err := state.DBHandler.DBSelectApp(ctx, transaction, u.Application)
+		dbApp, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, u.Application)
 		if err != nil {
 			return "", fmt.Errorf("UndeployApplication: could not select app '%s': %v", u.Application, err)
 		}

services/cd-service/pkg/service/overview.go

@@ -126,7 +126,7 @@ func (o *OverviewServiceServer) GetAppDetails(
 		sort.Slice(result.Releases, func(i, j int) bool {
 			return result.Releases[j].Version < result.Releases[i].Version
 		})
-		if app, err := o.DBHandler.DBSelectApp(ctx, transaction, appName); err != nil {
+		if app, err := o.DBHandler.DBSelectExistingApp(ctx, transaction, appName); err != nil {
 			return nil, err
 		} else {
 			if app == nil {
@@ -190,31 +190,47 @@ func (o *OverviewServiceServer) GetAppDetails(
 			return nil, fmt.Errorf("could not obtain deployments for app %s: %w", appName, err)
 		}
 		for envName, currentDeployment := range deployments {
-			deployment := &api.Deployment{
-				Version:         uint64(*currentDeployment.Version),
-				QueuedVersion:   0,
-				UndeployVersion: false,
-				DeploymentMetaData: &api.Deployment_DeploymentMetaData{
-					CiLink:       currentDeployment.Metadata.CiLink,
-					DeployAuthor: currentDeployment.Metadata.DeployedByName,
-					DeployTime:   currentDeployment.Created.String(),
-				},
+			environment, err := o.DBHandler.DBSelectEnvironment(ctx, transaction, envName)
+			if err != nil {
+				return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err)
 			}
-			if queuedVersion, err := o.Repository.State().GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) {
-				return nil, err
-			} else {
-				if queuedVersion == nil {
-					deployment.QueuedVersion = 0
-				} else {
-					deployment.QueuedVersion = *queuedVersion
+			if environment == nil {
+				return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err)
+			}
+			foundApp := false // only apps that are active on that environment should be returned here
+			for _, appInEnv := range environment.Applications {
+				if appInEnv == appName {
+					foundApp = true
+					break
 				}
 			}
+			if foundApp {
+				deployment := &api.Deployment{
+					Version:         uint64(*currentDeployment.Version),
+					QueuedVersion:   0,
+					UndeployVersion: false,
+					DeploymentMetaData: &api.Deployment_DeploymentMetaData{
+						CiLink:       currentDeployment.Metadata.CiLink,
+						DeployAuthor: currentDeployment.Metadata.DeployedByName,
+						DeployTime:   currentDeployment.Created.String(),
+					},
+				}
+				if queuedVersion, err := o.Repository.State().GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) {
+					return nil, err
+				} else {
+					if queuedVersion == nil {
+						deployment.QueuedVersion = 0
+					} else {
+						deployment.QueuedVersion = *queuedVersion
+					}
+				}
 
-			rel := getReleaseFromVersion(releases, uint64(*currentDeployment.Version))
-			if rel != nil {
-				deployment.UndeployVersion = rel.Metadata.UndeployVersion
+				rel := getReleaseFromVersion(releases, uint64(*currentDeployment.Version))
+				if rel != nil {
+					deployment.UndeployVersion = rel.Metadata.UndeployVersion
+				}
+				response.Deployments[envName] = deployment
 			}
-			response.Deployments[envName] = deployment
 		}
 		result.UndeploySummary = deriveUndeploySummary(appName, response.Deployments)
 		result.Warnings = CalculateWarnings(deployments, appLocks, envGroups)
@@ -352,7 +368,17 @@ func (o *OverviewServiceServer) StreamOverview(in *api.GetOverviewRequest,
 		case <-o.Shutdown:
 			return nil
 		case <-ch:
-			ov := o.response.Load().(*api.GetOverviewResponse)
+			loaded := o.response.Load()
+			var ov *api.GetOverviewResponse = nil
+			if loaded == nil {
+				ov, err := o.getOverviewDB(stream.Context(), o.Repository.State())
+				if err != nil {
+					return fmt.Errorf("could not load overview")
+				}
+				o.response.Store(ov)
+			} else {
+				ov = loaded.(*api.GetOverviewResponse)
+			}
 
 			if err := stream.Send(ov); err != nil {
 				// if we don't log this here, the details will be lost - so this is an exception to the rule "either return an error or log it".
@@ -467,6 +493,10 @@ func (o *OverviewServiceServer) update(s *repository.State) {
 		logger.FromContext(o.Context).Error("error getting overview:", zap.Error(err))
 		return
 	}
+	if r == nil {
+		logger.FromContext(o.Context).Error("overview is nil")
+		return
+	}
 	o.response.Store(r)
 	o.notify.Notify()
 }

services/cd-service/pkg/service/overview_test.go

@@ -286,14 +286,6 @@ func TestOverviewAndAppDetails(t *testing.T) {
 								DeployAuthor: "test tester",
 							},
 						},
-						prod: {
-							Version:         3,
-							QueuedVersion:   0,
-							UndeployVersion: false,
-							DeploymentMetaData: &api.Deployment_DeploymentMetaData{
-								DeployAuthor: "test tester",
-							},
-						},
 					},
 					AppLocks:  map[string]*api.Locks{},
 					TeamLocks: map[string]*api.Locks{},