fix(cd-service): fix apps in envs (#2064)

Ref: SRX-O2YISZ
* Retrieve list of environments' applications using releases table in
custom migration
* Set applications list to empty list when creating new environment
* When Undeploying an application delete it from applications list in
environments table

pkg/db/db.go

@@ -5602,16 +5602,17 @@ func (h *DBHandler) RunCustomMigrationEnvironments(ctx context.Context, getAllEn
 			return fmt.Errorf("could not get environments, error: %w", err)
 		}
 
-		allApplications, err := h.DBSelectAllApplications(ctx, transaction)
+		allEnvironmentNames := make([]string, 0)
+		allEnvsApps, err := h.FindEnvsAppsFromReleases(ctx, transaction)
 		if err != nil {
-			return fmt.Errorf("could not get all applications, error: %w", err)
+			return err
 		}
-
-		allEnvironmentNames := make([]string, 0)
 		for envName, config := range allEnvironments {
 			allEnvironmentNames = append(allEnvironmentNames, envName)
-
-			err = h.DBWriteEnvironment(ctx, transaction, envName, config, allApplications.Apps)
+			if allEnvsApps[envName] == nil {
+				allEnvsApps[envName] = make([]string, 0)
+			}
+			err = h.DBWriteEnvironment(ctx, transaction, envName, config, allEnvsApps[envName])
 			if err != nil {
 				return fmt.Errorf("unable to write manifest for environment %s to the database, error: %w", envName, err)
 			}
@@ -5648,18 +5649,24 @@ func (h *DBHandler) RunCustomMigrationEnvironmentApplications(ctx context.Contex
 		if err != nil {
 			return fmt.Errorf("could not get environments, error: %w", err)
 		}
-
-		allApplications, err := h.DBSelectAllApplications(ctx, transaction)
-		if err != nil {
-			return fmt.Errorf("could not get all applications, error: %w", err)
-		}
+		var allEnvsApps map[string][]string
 		for _, envName := range allEnvironments.Environments {
 			env, err := h.DBSelectEnvironment(ctx, transaction, envName)
 			if err != nil {
 				return fmt.Errorf("could not get env: %s, error: %w", envName, err)
 			}
+
 			if env.Applications == nil || len(env.Applications) == 0 {
-				err = h.DBWriteEnvironment(ctx, transaction, envName, env.Config, allApplications.Apps)
+				if allEnvsApps == nil {
+					allEnvsApps, err = h.FindEnvsAppsFromReleases(ctx, transaction)
+					if err != nil {
+						return fmt.Errorf("could not find all applications of all environments, error: %w", err)
+					}
+				}
+				if allEnvsApps[envName] == nil {
+					allEnvsApps[envName] = make([]string, 0)
+				}
+				err = h.DBWriteEnvironment(ctx, transaction, envName, env.Config, allEnvsApps[envName])
 				if err != nil {
 					return fmt.Errorf("unable to write manifest for environment %s to the database, error: %w", envName, err)
 				}
@@ -5669,6 +5676,26 @@ func (h *DBHandler) RunCustomMigrationEnvironmentApplications(ctx context.Contex
 	})
 }
 
+func (h *DBHandler) FindEnvsAppsFromReleases(ctx context.Context, tx *sql.Tx) (map[string][]string, error) {
+	envsApps := make(map[string][]string)
+	releases, err := h.DBSelectAllManifestsForAllReleases(ctx, tx)
+	if err != nil {
+		return nil, fmt.Errorf("could not get all environments for all releases, error: %w", err)
+	}
+	for app, versionEnvs := range releases {
+		envSet := make(map[string]struct{})
+		for _, envs := range versionEnvs {
+			for _, env := range envs {
+				envSet[env] = struct{}{}
+			}
+		}
+		for env := range envSet {
+			envsApps[env] = append(envsApps[env], app)
+		}
+	}
+	return envsApps, nil
+}
+
 func (h *DBHandler) RunCustomMigrationReleaseEnvironments(ctx context.Context) error {
 	span, ctx := tracer.StartSpanFromContext(ctx, "RunCustomMigrationReleaseEnvironments")
 	defer span.Finish()

pkg/db/db_test.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"sort"
 	"strconv"
 	"testing"
 	"time"
@@ -2708,6 +2709,209 @@ func TestReadWriteOverviewCache(t *testing.T) {
 	}
 }
 
+func TestFindEnvAppsFromReleases(t *testing.T) {
+	type TestCase struct {
+		Name             string
+		Releases         []DBReleaseWithMetaData
+		ExpectedEnvsApps map[string][]string
+	}
+	tcs := []TestCase{
+		{
+			Name: "Simple test: several releases",
+			Releases: []DBReleaseWithMetaData{
+				{
+					EslVersion:    1,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env1": "testmanifest",
+							"env2": "another test manifest",
+							"env3": "test",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+				{
+					EslVersion:    1,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app2",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env1": "testmanifest",
+							"env3": "test",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+				{
+					EslVersion:    1,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app3",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env1": "testmanifest",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+			},
+			ExpectedEnvsApps: map[string][]string{
+				"env1": {"app1", "app2", "app3"},
+				"env2": {"app1"},
+				"env3": {"app1", "app2"},
+			},
+		},
+		{
+			Name: "Several Releases for one app",
+			Releases: []DBReleaseWithMetaData{
+				{
+					EslVersion:    1,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env1": "testmanifest",
+							"env2": "another test manifest",
+							"env3": "test",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+				{
+					EslVersion:    1,
+					ReleaseNumber: 11,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env1": "testmanifest",
+							"env3": "test",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+				{
+					EslVersion:    1,
+					ReleaseNumber: 12,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env4": "testmanifest",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+			},
+			ExpectedEnvsApps: map[string][]string{
+				"env1": {"app1"},
+				"env2": {"app1"},
+				"env3": {"app1"},
+				"env4": {"app1"},
+			},
+		},
+		{
+			Name: "Releases with different esl versions",
+			Releases: []DBReleaseWithMetaData{
+				{
+					EslVersion:    1,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env1": "testmanifest",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+				{
+					EslVersion:    2,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env2": "testmanifest",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+				{
+					EslVersion:    3,
+					ReleaseNumber: 10,
+					Created:       time.Now(),
+					App:           "app1",
+					Manifests: DBReleaseManifests{
+						Manifests: map[string]string{
+							"env3": "testmanifest",
+						},
+					},
+					Metadata: DBReleaseMetaData{
+						SourceAuthor: "testauthor",
+					},
+				},
+			},
+			ExpectedEnvsApps: map[string][]string{
+				"env3": {"app1"},
+			},
+		},
+	}
+	for _, tc := range tcs {
+		tc := tc
+		t.Run(tc.Name, func(t *testing.T) {
+			t.Parallel()
+			ctx := testutil.MakeTestContext()
+			dbHandler := setupDB(t)
+
+			err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
+				for _, release := range tc.Releases {
+					err := dbHandler.DBInsertRelease(ctx, transaction, release, release.EslVersion-1)
+					if err != nil {
+						return err
+					}
+				}
+
+				envsAppsFromReleases, err := dbHandler.FindEnvsAppsFromReleases(ctx, transaction)
+				if err != nil {
+					return err
+				}
+				for env := range envsAppsFromReleases {
+					sort.Strings(envsAppsFromReleases[env])
+				}
+				if diff := cmp.Diff(tc.ExpectedEnvsApps, envsAppsFromReleases); diff != "" {
+					t.Errorf("response mismatch (-want, +got):\n%s", diff)
+				}
+				return nil
+			})
+			if err != nil {
+				t.Fatalf("error while running the transaction, error: %v", err)
+			}
+		})
+	}
+}
+
 // setupDB returns a new DBHandler with a tmp directory every time, so tests can are completely independent
 func setupDB(t *testing.T) *DBHandler {
 	ctx := context.Background()

services/cd-service/pkg/cmd/server.go

@@ -381,17 +381,17 @@ func RunServer() {
 			} else if err != nil {
 				logger.FromContext(ctx).Fatal("Error running custom database migrations", zap.Error(err))
 			}
-			logger.FromContext(ctx).Sugar().Warnf("Skipping custom migrations, because all tables contain data.")
-			err = dbHandler.RunCustomMigrationEnvironmentApplications(ctx)
+			logger.FromContext(ctx).Sugar().Warnf("Skipping git-related custom migrations, because all tables contain data.")
+			err = dbHandler.RunCustomMigrationReleaseEnvironments(ctx)
 			if err != nil {
 				return err
 			}
-			logger.FromContext(ctx).Sugar().Warnf("Applied custom migrations for environment applications")
-			err = dbHandler.RunCustomMigrationReleaseEnvironments(ctx)
+			logger.FromContext(ctx).Sugar().Warnf("Applied custom migration for release environments")
+			err = dbHandler.RunCustomMigrationEnvironmentApplications(ctx)
 			if err != nil {
 				return err
 			}
-			logger.FromContext(ctx).Sugar().Warnf("Applied custom migration for release environments")
+			logger.FromContext(ctx).Sugar().Warnf("Applied custom migrations for environment applications")
 		} else {
 			logger.FromContext(ctx).Sugar().Warnf("Skipping custom migrations, because KUBERPULT_DB_WRITE_ESL_TABLE_ONLY=false")
 		}

services/cd-service/pkg/repository/transformer.go

@@ -1635,7 +1635,26 @@ func (u *UndeployApplication) Transform(
 		if err != nil {
 			return "", fmt.Errorf("UndeployApplication: could not clear all deployments for app '%s': %v", u.Application, err)
 		}
-
+		allEnvs, err := state.DBHandler.DBSelectAllEnvironments(ctx, transaction)
+		if err != nil {
+			return "", fmt.Errorf("UndeployApplication: could not get all environments: %v", err)
+		}
+		for _, envName := range allEnvs.Environments {
+			env, err := state.DBHandler.DBSelectEnvironment(ctx, transaction, envName)
+			if err != nil {
+				return "", fmt.Errorf("UndeployApplication: could not get environment %s: %v", envName, err)
+			}
+			newEnvApps := make([]string, 0)
+			for _, app := range env.Applications {
+				if app != u.Application {
+					newEnvApps = append(newEnvApps, app)
+				}
+			}
+			err = state.DBHandler.DBWriteEnvironment(ctx, transaction, envName, env.Config, newEnvApps)
+			if err != nil {
+				return "", fmt.Errorf("UndeployApplication: could not write environment: %v", err)
+			}
+		}
 	} else {
 		// remove application
 		appDir := applicationDirectory(fs, u.Application)
@@ -2738,14 +2757,7 @@ func (c *CreateEnvironment) Transform(
 	}
 	if state.DBHandler.ShouldUseOtherTables() {
 		// write to environments table
-		allApplications, err := state.DBHandler.DBSelectAllApplications(ctx, transaction)
-		if err != nil {
-			return "", fmt.Errorf("unable to read all applications, error: %w", err)
-		}
 		environmentApplications := make([]string, 0)
-		if allApplications != nil {
-			environmentApplications = allApplications.Apps
-		}
 		err = state.DBInsertEnvironmentWithOverview(ctx, transaction, c.Environment, c.Config, environmentApplications)
 		if err != nil {
 			return "", fmt.Errorf("unable to write to the environment table, error: %w", err)

services/manifest-repo-export-service/pkg/repository/repository.go

@@ -1282,14 +1282,14 @@ func (s *State) GetEnvironmentConfigsForGroup(envGroup string) ([]string, error)
 }
 
 func (s *State) GetEnvironmentApplications(ctx context.Context, transaction *sql.Tx, environment string) ([]string, error) {
-	applications, err := s.DBHandler.DBSelectAllApplications(ctx, transaction)
+	envInfo, err := s.DBHandler.DBSelectEnvironment(ctx, transaction, environment)
 	if err != nil {
-		return nil, err
+		return make([]string, 0), err
 	}
-	if applications == nil {
+	if envInfo == nil || envInfo.Applications == nil {
 		return make([]string, 0), nil
 	}
-	return applications.Apps, nil
+	return envInfo.Applications, nil
 }
 
 // GetApplicationsFromFile returns apps from the filesystem