fix: do not consider the overview cache multiple times in a releasetr…

…ain (#1980)

This is a performance improvement that solves 2 issues with the overview
cache:
1) It reduces the amount of times we write the overview in a release
train from `n=number of apps` to `1 per env`. The 1 write contains
multiple changed apps.
2) When creating a new app or when undeploying, it does not force a
recalculation of the whole overview, but instead only updates the app.

Further cases that also force a complete recalculation of the overview
cache (like creating a new environment) are not handled in this PR.

Note that some functions like `CalculateWarnings` had to be moved to
another module but were not changed.

Ref: SRX-4WQBET

pkg/db/db.go

@@ -61,6 +61,8 @@ type DBConfig struct {
 	MaxOpenConnections uint
 }
 
+type InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error
+
 type DBHandler struct {
 	DbName         string
 	DriverName     string
@@ -75,6 +77,9 @@ type DBHandler struct {
 		3) DBHandler!=nil && WriteEslOnly==false: write everything to the database.
 	*/
 	WriteEslOnly bool
+
+	// InsertAppFun is intended to be used to add more to inserting an app: specifically to update the overview cache
+	InsertAppFun InsertAppFun
 }
 
 type EslVersion int64
@@ -109,14 +114,20 @@ func Connect(ctx context.Context, cfg DBConfig) (*DBHandler, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &DBHandler{
+	var handler = &DBHandler{
 		DbName:         cfg.DbName,
 		DriverName:     cfg.DriverName,
 		MigrationsPath: cfg.MigrationsPath,
 		DB:             db,
 		DBDriver:       &driver,
 		WriteEslOnly:   cfg.WriteEslOnly,
-	}, nil
+		InsertAppFun:   nil,
+	}
+	handler.InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error {
+		// by default, we just insert the app
+		return handler.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData)
+	}
+	return handler, nil
 }
 
 func GetDBConnection(cfg DBConfig) (*sql.DB, error) {
@@ -1797,10 +1808,6 @@ func (h *DBHandler) DBInsertApplication(ctx context.Context, transaction *sql.Tx
 	if err != nil {
 		return fmt.Errorf("could not insert app %s into DB. Error: %w\n", appName, err)
 	}
-	err = h.ForceOverviewRecalculation(ctx, transaction)
-	if err != nil {
-		return fmt.Errorf("could not update overview table. Error: %w\n", err)
-	}
 	return nil
 }
 
@@ -1921,7 +1928,7 @@ func (h *DBHandler) DBSelectApp(ctx context.Context, tx *sql.Tx, appName string)
 }
 
 // DBWriteDeployment writes one deployment, meaning "what should be deployed"
-func (h *DBHandler) DBWriteDeployment(ctx context.Context, tx *sql.Tx, deployment Deployment, previousEslVersion EslVersion) error {
+func (h *DBHandler) DBWriteDeployment(ctx context.Context, tx *sql.Tx, deployment Deployment, previousEslVersion EslVersion, skipOverview bool) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "DBWriteDeployment")
 	defer span.Finish()
 	if h == nil {
@@ -1959,9 +1966,11 @@ func (h *DBHandler) DBWriteDeployment(ctx context.Context, tx *sql.Tx, deploymen
 	if err != nil {
 		return fmt.Errorf("could not write deployment into DB. Error: %w\n", err)
 	}
-	err = h.UpdateOverviewDeployment(ctx, tx, deployment, *now)
-	if err != nil {
-		return fmt.Errorf("could not update overview table. Error: %w\n", err)
+	if !skipOverview {
+		err = h.UpdateOverviewDeployment(ctx, tx, deployment, *now)
+		if err != nil {
+			return fmt.Errorf("could not update overview table. Error: %w\n", err)
+		}
 	}
 	return nil
 }
@@ -4237,7 +4246,7 @@ func (h *DBHandler) DBSelectLatestDeploymentAttempt(ctx context.Context, tx *sql
 	return h.processDeploymentAttemptsRow(ctx, rows, err)
 }
 
-func (h *DBHandler) DBWriteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string, version *int64) error {
+func (h *DBHandler) DBWriteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string, version *int64, skipOverview bool) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "DBWriteDeploymentAttempt")
 	defer span.Finish()
 
@@ -4253,10 +4262,10 @@ func (h *DBHandler) DBWriteDeploymentAttempt(ctx context.Context, tx *sql.Tx, en
 		Env:        envName,
 		App:        appName,
 		Version:    version,
-	})
+	}, skipOverview)
 }
 
-func (h *DBHandler) DBDeleteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string) error {
+func (h *DBHandler) DBDeleteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string, skipOverview bool) error {
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBWriteDeploymentAttempt")
 	defer span.Finish()
 
@@ -4272,10 +4281,10 @@ func (h *DBHandler) DBDeleteDeploymentAttempt(ctx context.Context, tx *sql.Tx, e
 		Env:        envName,
 		App:        appName,
 		Version:    nil,
-	})
+	}, skipOverview)
 }
 
-func (h *DBHandler) dbWriteDeploymentAttemptInternal(ctx context.Context, tx *sql.Tx, deployment *QueuedDeployment) error {
+func (h *DBHandler) dbWriteDeploymentAttemptInternal(ctx context.Context, tx *sql.Tx, deployment *QueuedDeployment, skipOverview bool) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "dbWriteDeploymentAttemptInternal")
 	defer span.Finish()
 
@@ -4317,9 +4326,11 @@ func (h *DBHandler) dbWriteDeploymentAttemptInternal(ctx context.Context, tx *sq
 	if err != nil {
 		return fmt.Errorf("could not write deployment attempts table in DB. Error: %w\n", err)
 	}
-	err = h.UpdateOverviewDeploymentAttempt(ctx, tx, deployment)
-	if err != nil {
-		return fmt.Errorf("could not update overview table in DB. Error: %w\n", err)
+	if !skipOverview {
+		err = h.UpdateOverviewDeploymentAttempt(ctx, tx, deployment)
+		if err != nil {
+			return fmt.Errorf("could not update overview table in DB. Error: %w\n", err)
+		}
 	}
 	return nil
 }

pkg/db/db_test.go

@@ -867,7 +867,7 @@ func TestReadWriteDeployment(t *testing.T) {
 					Env:           tc.Env,
 					Version:       tc.VersionToDeploy,
 					TransformerID: 0,
-				}, 1)
+				}, 1, false)
 				if err != nil {
 					return err
 				}
@@ -1445,7 +1445,7 @@ func TestQueueApplicationVersion(t *testing.T) {
 			dbHandler := setupDB(t)
 			err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
 				for _, deployments := range tc.Deployments {
-					err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, deployments.Env, deployments.App, deployments.Version)
+					err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, deployments.Env, deployments.App, deployments.Version, false)
 					if err != nil {
 						return err
 					}
@@ -1507,12 +1507,12 @@ func TestQueueApplicationVersionDelete(t *testing.T) {
 
 			dbHandler := setupDB(t)
 			err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
-				err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName, tc.Version)
+				err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName, tc.Version, false)
 				if err != nil {
 					return err
 				}
 
-				err = dbHandler.DBDeleteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName)
+				err = dbHandler.DBDeleteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName, false)
 				if err != nil {
 					return err
 				}

pkg/db/overview.go

@@ -250,7 +250,7 @@ func (h *DBHandler) UpdateOverviewRelease(ctx context.Context, transaction *sql.
 	}
 	app := getApplicationByName(latestOverview.Applications, release.App)
 	if app == nil {
-		return fmt.Errorf("could not find application %s in overview", release.App)
+		return fmt.Errorf("could not find application '%s' in overview", release.App)
 	}
 	apiRelease := &api.Release{
 		PrNumber:        extractPrNumber(release.Metadata.SourceMessage),