diff --git a/pkg/db/db.go b/pkg/db/db.go index b5ad7da06..3eeaabdb4 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -61,6 +61,8 @@ type DBConfig struct { MaxOpenConnections uint } +type InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error + type DBHandler struct { DbName string DriverName string @@ -75,6 +77,9 @@ type DBHandler struct { 3) DBHandler!=nil && WriteEslOnly==false: write everything to the database. */ WriteEslOnly bool + + // InsertAppFun is intended to be used to add more to inserting an app: specifically to update the overview cache + InsertAppFun InsertAppFun } type EslVersion int64 @@ -109,14 +114,20 @@ func Connect(ctx context.Context, cfg DBConfig) (*DBHandler, error) { if err != nil { return nil, err } - return &DBHandler{ + var handler = &DBHandler{ DbName: cfg.DbName, DriverName: cfg.DriverName, MigrationsPath: cfg.MigrationsPath, DB: db, DBDriver: &driver, WriteEslOnly: cfg.WriteEslOnly, - }, nil + InsertAppFun: nil, + } + handler.InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error { + // by default, we just insert the app + return handler.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData) + } + return handler, nil } func GetDBConnection(cfg DBConfig) (*sql.DB, error) { @@ -1797,10 +1808,6 @@ func (h *DBHandler) DBInsertApplication(ctx context.Context, transaction *sql.Tx if err != nil { return fmt.Errorf("could not insert app %s into DB. Error: %w\n", appName, err) } - err = h.ForceOverviewRecalculation(ctx, transaction) - if err != nil { - return fmt.Errorf("could not update overview table. Error: %w\n", err) - } return nil } @@ -1921,7 +1928,7 @@ func (h *DBHandler) DBSelectApp(ctx context.Context, tx *sql.Tx, appName string) } // DBWriteDeployment writes one deployment, meaning "what should be deployed" -func (h *DBHandler) DBWriteDeployment(ctx context.Context, tx *sql.Tx, deployment Deployment, previousEslVersion EslVersion) error { +func (h *DBHandler) DBWriteDeployment(ctx context.Context, tx *sql.Tx, deployment Deployment, previousEslVersion EslVersion, skipOverview bool) error { span, _ := tracer.StartSpanFromContext(ctx, "DBWriteDeployment") defer span.Finish() if h == nil { @@ -1959,9 +1966,11 @@ func (h *DBHandler) DBWriteDeployment(ctx context.Context, tx *sql.Tx, deploymen if err != nil { return fmt.Errorf("could not write deployment into DB. Error: %w\n", err) } - err = h.UpdateOverviewDeployment(ctx, tx, deployment, *now) - if err != nil { - return fmt.Errorf("could not update overview table. Error: %w\n", err) + if !skipOverview { + err = h.UpdateOverviewDeployment(ctx, tx, deployment, *now) + if err != nil { + return fmt.Errorf("could not update overview table. Error: %w\n", err) + } } return nil } @@ -4237,7 +4246,7 @@ func (h *DBHandler) DBSelectLatestDeploymentAttempt(ctx context.Context, tx *sql return h.processDeploymentAttemptsRow(ctx, rows, err) } -func (h *DBHandler) DBWriteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string, version *int64) error { +func (h *DBHandler) DBWriteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string, version *int64, skipOverview bool) error { span, _ := tracer.StartSpanFromContext(ctx, "DBWriteDeploymentAttempt") defer span.Finish() @@ -4253,10 +4262,10 @@ func (h *DBHandler) DBWriteDeploymentAttempt(ctx context.Context, tx *sql.Tx, en Env: envName, App: appName, Version: version, - }) + }, skipOverview) } -func (h *DBHandler) DBDeleteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string) error { +func (h *DBHandler) DBDeleteDeploymentAttempt(ctx context.Context, tx *sql.Tx, envName, appName string, skipOverview bool) error { span, ctx := tracer.StartSpanFromContext(ctx, "DBWriteDeploymentAttempt") defer span.Finish() @@ -4272,10 +4281,10 @@ func (h *DBHandler) DBDeleteDeploymentAttempt(ctx context.Context, tx *sql.Tx, e Env: envName, App: appName, Version: nil, - }) + }, skipOverview) } -func (h *DBHandler) dbWriteDeploymentAttemptInternal(ctx context.Context, tx *sql.Tx, deployment *QueuedDeployment) error { +func (h *DBHandler) dbWriteDeploymentAttemptInternal(ctx context.Context, tx *sql.Tx, deployment *QueuedDeployment, skipOverview bool) error { span, _ := tracer.StartSpanFromContext(ctx, "dbWriteDeploymentAttemptInternal") defer span.Finish() @@ -4317,9 +4326,11 @@ func (h *DBHandler) dbWriteDeploymentAttemptInternal(ctx context.Context, tx *sq if err != nil { return fmt.Errorf("could not write deployment attempts table in DB. Error: %w\n", err) } - err = h.UpdateOverviewDeploymentAttempt(ctx, tx, deployment) - if err != nil { - return fmt.Errorf("could not update overview table in DB. Error: %w\n", err) + if !skipOverview { + err = h.UpdateOverviewDeploymentAttempt(ctx, tx, deployment) + if err != nil { + return fmt.Errorf("could not update overview table in DB. Error: %w\n", err) + } } return nil } diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index b63f842e8..14e1516be 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -867,7 +867,7 @@ func TestReadWriteDeployment(t *testing.T) { Env: tc.Env, Version: tc.VersionToDeploy, TransformerID: 0, - }, 1) + }, 1, false) if err != nil { return err } @@ -1445,7 +1445,7 @@ func TestQueueApplicationVersion(t *testing.T) { dbHandler := setupDB(t) err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error { for _, deployments := range tc.Deployments { - err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, deployments.Env, deployments.App, deployments.Version) + err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, deployments.Env, deployments.App, deployments.Version, false) if err != nil { return err } @@ -1507,12 +1507,12 @@ func TestQueueApplicationVersionDelete(t *testing.T) { dbHandler := setupDB(t) err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error { - err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName, tc.Version) + err := dbHandler.DBWriteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName, tc.Version, false) if err != nil { return err } - err = dbHandler.DBDeleteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName) + err = dbHandler.DBDeleteDeploymentAttempt(ctx, transaction, tc.Env, tc.AppName, false) if err != nil { return err } diff --git a/pkg/db/overview.go b/pkg/db/overview.go index d63cef8f1..34a40c84a 100644 --- a/pkg/db/overview.go +++ b/pkg/db/overview.go @@ -250,7 +250,7 @@ func (h *DBHandler) UpdateOverviewRelease(ctx context.Context, transaction *sql. } app := getApplicationByName(latestOverview.Applications, release.App) if app == nil { - return fmt.Errorf("could not find application %s in overview", release.App) + return fmt.Errorf("could not find application '%s' in overview", release.App) } apiRelease := &api.Release{ PrNumber: extractPrNumber(release.Metadata.SourceMessage), diff --git a/pkg/db/overview_test.go b/pkg/db/overview_test.go index 809b7772a..8287bd8fe 100644 --- a/pkg/db/overview_test.go +++ b/pkg/db/overview_test.go @@ -1241,3 +1241,307 @@ func TestForceOverviewRecalculation(t *testing.T) { }) } } + +func makeApps(apps ...*api.Environment_Application) map[string]*api.Environment_Application { + var result map[string]*api.Environment_Application = map[string]*api.Environment_Application{} + for i := 0; i < len(apps); i++ { + app := apps[i] + result[app.Name] = app + } + return result +} + +func makeEnv(envName string, groupName string, upstream *api.EnvironmentConfig_Upstream, apps map[string]*api.Environment_Application) *api.Environment { + return &api.Environment{ + Name: envName, + Config: &api.EnvironmentConfig{ + Upstream: upstream, + EnvironmentGroup: &groupName, + }, + Locks: map[string]*api.Lock{}, + + Applications: apps, + DistanceToUpstream: 0, + Priority: api.Priority_UPSTREAM, // we are 1 away from prod, hence pre-prod + } +} + +func makeApp(appName string, version uint64) *api.Environment_Application { + return &api.Environment_Application{ + Name: appName, + Version: version, + Locks: nil, + QueuedVersion: 0, + UndeployVersion: false, + ArgoCd: nil, + } +} +func makeEnvGroup(envGroupName string, environments []*api.Environment) *api.EnvironmentGroup { + return &api.EnvironmentGroup{ + EnvironmentGroupName: envGroupName, + Environments: environments, + DistanceToUpstream: 0, + } +} + +func makeUpstreamLatest() *api.EnvironmentConfig_Upstream { + f := true + return &api.EnvironmentConfig_Upstream{ + Latest: &f, + } +} + +func makeUpstreamEnv(upstream string) *api.EnvironmentConfig_Upstream { + return &api.EnvironmentConfig_Upstream{ + Environment: &upstream, + } +} + +func TestCalculateWarnings(t *testing.T) { + var dev = "dev" + tcs := []struct { + Name string + AppName string + Groups []*api.EnvironmentGroup + ExpectedWarnings []*api.Warning + }{ + { + Name: "no envs - no warning", + AppName: "foo", + Groups: []*api.EnvironmentGroup{ + makeEnvGroup(dev, []*api.Environment{ + makeEnv("dev-de", dev, makeUpstreamLatest(), nil), + })}, + ExpectedWarnings: []*api.Warning{}, + }, + { + Name: "app deployed in higher version on upstream should warn", + AppName: "foo", + Groups: []*api.EnvironmentGroup{ + makeEnvGroup(dev, []*api.Environment{ + makeEnv("prod", dev, makeUpstreamEnv("dev"), + makeApps(makeApp("foo", 2))), + }), + makeEnvGroup(dev, []*api.Environment{ + makeEnv("dev", dev, makeUpstreamLatest(), + makeApps(makeApp("foo", 1))), + }), + }, + ExpectedWarnings: []*api.Warning{ + { + WarningType: &api.Warning_UnusualDeploymentOrder{ + UnusualDeploymentOrder: &api.UnusualDeploymentOrder{ + UpstreamVersion: 1, + UpstreamEnvironment: "dev", + ThisVersion: 2, + ThisEnvironment: "prod", + }, + }, + }, + }, + }, + { + Name: "app deployed in same version on upstream should not warn", + AppName: "foo", + Groups: []*api.EnvironmentGroup{ + makeEnvGroup(dev, []*api.Environment{ + makeEnv("prod", dev, makeUpstreamEnv("dev"), + makeApps(makeApp("foo", 2))), + }), + makeEnvGroup(dev, []*api.Environment{ + makeEnv("dev", dev, makeUpstreamLatest(), + makeApps(makeApp("foo", 2))), + }), + }, + ExpectedWarnings: []*api.Warning{}, + }, + { + Name: "app deployed in no version on upstream should warn", + AppName: "foo", + Groups: []*api.EnvironmentGroup{ + makeEnvGroup(dev, []*api.Environment{ + makeEnv("prod", dev, makeUpstreamEnv("dev"), + makeApps(makeApp("foo", 1))), + }), + makeEnvGroup(dev, []*api.Environment{ + makeEnv("dev", dev, makeUpstreamLatest(), + makeApps()), + }), + }, + ExpectedWarnings: []*api.Warning{ + { + WarningType: &api.Warning_UpstreamNotDeployed{ + UpstreamNotDeployed: &api.UpstreamNotDeployed{ + UpstreamEnvironment: "dev", + ThisVersion: 1, + ThisEnvironment: "prod", + }, + }, + }, + }, + }, + } + for _, tc := range tcs { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + actualWarnings := CalculateWarnings(testutil.MakeTestContext(), tc.AppName, tc.Groups) + if len(actualWarnings) != len(tc.ExpectedWarnings) { + t.Errorf("Different number of warnings. got: %s\nwant: %s", actualWarnings, tc.ExpectedWarnings) + } + for i := 0; i < len(actualWarnings); i++ { + actualWarning := actualWarnings[i] + expectedWarning := tc.ExpectedWarnings[i] + if diff := cmp.Diff(actualWarning.String(), expectedWarning.String()); diff != "" { + t.Errorf("Different warning at index [%d]:\ngot: %s\nwant: %s", i, actualWarning, expectedWarning) + } + } + }) + } +} + +func groupFromEnvs(environments []*api.Environment) []*api.EnvironmentGroup { + return []*api.EnvironmentGroup{ + { + EnvironmentGroupName: "group1", + Environments: environments, + }, + } +} + +func TestDeriveUndeploySummary(t *testing.T) { + var tcs = []struct { + Name string + AppName string + groups []*api.EnvironmentGroup + ExpectedResult api.UndeploySummary + }{ + { + Name: "No Environments", + AppName: "foo", + groups: []*api.EnvironmentGroup{}, + ExpectedResult: api.UndeploySummary_UNDEPLOY, + }, + { + Name: "one Environment but no Application", + AppName: "foo", + groups: groupFromEnvs([]*api.Environment{ + { + Applications: map[string]*api.Environment_Application{ + "bar": { // different app + UndeployVersion: true, + Version: 666, + }, + }, + }, + }), + ExpectedResult: api.UndeploySummary_UNDEPLOY, + }, + { + Name: "One Env with undeploy", + AppName: "foo", + groups: groupFromEnvs([]*api.Environment{ + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: true, + Version: 666, + }, + }, + }, + }), + ExpectedResult: api.UndeploySummary_UNDEPLOY, + }, + { + Name: "One Env with normal version", + AppName: "foo", + groups: groupFromEnvs([]*api.Environment{ + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: false, + Version: 666, + }, + }, + }, + }), + ExpectedResult: api.UndeploySummary_NORMAL, + }, + { + Name: "Two Envs all undeploy", + AppName: "foo", + groups: groupFromEnvs([]*api.Environment{ + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: true, + Version: 666, + }, + }, + }, + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: true, + Version: 666, + }, + }, + }, + }), + ExpectedResult: api.UndeploySummary_UNDEPLOY, + }, + { + Name: "Two Envs all normal", + AppName: "foo", + groups: groupFromEnvs([]*api.Environment{ + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: false, + Version: 666, + }, + }, + }, + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: false, + Version: 666, + }, + }, + }, + }), + ExpectedResult: api.UndeploySummary_NORMAL, + }, + { + Name: "Two Envs all different", + AppName: "foo", + groups: groupFromEnvs([]*api.Environment{ + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: true, + Version: 666, + }, + }, + }, + { + Applications: map[string]*api.Environment_Application{ + "foo": { + UndeployVersion: false, + Version: 666, + }, + }, + }, + }), + ExpectedResult: api.UndeploySummary_MIXED, + }, + } + for _, tc := range tcs { + t.Run(tc.Name, func(t *testing.T) { + actualResult := deriveUndeploySummary(tc.AppName, tc.groups) + if !cmp.Equal(tc.ExpectedResult, actualResult) { + t.Fatal("Output mismatch (-want +got):\n", cmp.Diff(tc.ExpectedResult, actualResult)) + } + }) + } +} diff --git a/services/cd-service/pkg/cmd/server.go b/services/cd-service/pkg/cmd/server.go index 69166b786..f520d45a3 100755 --- a/services/cd-service/pkg/cmd/server.go +++ b/services/cd-service/pkg/cmd/server.go @@ -18,6 +18,7 @@ package cmd import ( "context" + "database/sql" "fmt" "net/http" "os" @@ -344,6 +345,11 @@ func RunServer() { logger.FromContext(ctx).Fatal("Could not pull repository to perform custom migrations", zap.Error(err)) } logger.FromContext(ctx).Sugar().Warnf("running custom migrations, because KUBERPULT_DB_WRITE_ESL_TABLE_ONLY=false") + + // we overwrite InsertApp in order to also update the overview: + dbHandler.InsertAppFun = func(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion db.EslVersion, stateChange db.AppStateChange, metaData db.DBAppMetaData) error { + return repo.State().DBInsertApplicationWithOverview(ctx, transaction, appName, previousEslVersion, stateChange, metaData) + } migErr := dbHandler.RunCustomMigrations( ctx, repo.State().GetAppsAndTeams, diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index 02a13d536..101864a9f 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -1680,19 +1680,19 @@ func (s *State) GetQueuedVersionFromManifest(environment string, application str return s.readSymlink(environment, application, queueFileName) } -func (s *State) DeleteQueuedVersionFromDB(ctx context.Context, transaction *sql.Tx, environment string, application string) error { - return s.DBHandler.DBDeleteDeploymentAttempt(ctx, transaction, environment, application) +func (s *State) DeleteQueuedVersionFromDB(ctx context.Context, transaction *sql.Tx, environment string, application string, skipOverview bool) error { + return s.DBHandler.DBDeleteDeploymentAttempt(ctx, transaction, environment, application, skipOverview) } -func (s *State) DeleteQueuedVersion(ctx context.Context, transaction *sql.Tx, environment string, application string) error { +func (s *State) DeleteQueuedVersion(ctx context.Context, transaction *sql.Tx, environment string, application string, skipOverview bool) error { if s.DBHandler.ShouldUseOtherTables() { - return s.DeleteQueuedVersionFromDB(ctx, transaction, environment, application) + return s.DeleteQueuedVersionFromDB(ctx, transaction, environment, application, skipOverview) } queuedVersion := s.Filesystem.Join("environments", environment, "applications", application, queueFileName) return s.Filesystem.Remove(queuedVersion) } -func (s *State) DeleteQueuedVersionIfExists(ctx context.Context, transaction *sql.Tx, environment string, application string) error { +func (s *State) DeleteQueuedVersionIfExists(ctx context.Context, transaction *sql.Tx, environment string, application string, skipOverview bool) error { queuedVersion, err := s.GetQueuedVersion(ctx, transaction, environment, application) if err != nil { return err @@ -1700,7 +1700,7 @@ func (s *State) DeleteQueuedVersionIfExists(ctx context.Context, transaction *sq if queuedVersion == nil { return nil // nothing to do } - return s.DeleteQueuedVersion(ctx, transaction, environment, application) + return s.DeleteQueuedVersion(ctx, transaction, environment, application, skipOverview) } func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, transaction *sql.Tx, environment string, application string) (*uint64, error) { @@ -2077,7 +2077,7 @@ func (s *State) WriteCurrentlyDeployed(ctx context.Context, transaction *sql.Tx, CiLink: "", }, } - err = dbHandler.DBWriteDeployment(ctx, transaction, deployment, 0) + err = dbHandler.DBWriteDeployment(ctx, transaction, deployment, 0, true) if err != nil { return fmt.Errorf("error writing Deployment to DB for app %s in env %s: %w", deployment.App, deployment.Env, err) } @@ -2260,10 +2260,16 @@ func (s *State) WriteAllQueuedAppVersions(ctx context.Context, transaction *sql. } else { versionIntPtr = nil } - err = dbHandler.DBWriteDeploymentAttempt(ctx, transaction, envName, currentApp, versionIntPtr) + err = dbHandler.DBWriteDeploymentAttempt(ctx, transaction, envName, currentApp, versionIntPtr, true) if err != nil { + var deref int64 + if versionIntPtr == nil { + deref = 0 + } else { + deref = *versionIntPtr + } return fmt.Errorf("error writing existing queued application version '%d' to DB for app '%s' on environment '%s': %w", - *versionIntPtr, currentApp, envName, err) + deref, currentApp, envName, err) } } } @@ -2330,6 +2336,322 @@ func (s *State) WriteAllCommitEvents(ctx context.Context, transaction *sql.Tx, d return nil } +func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion db.EslVersion, stateChange db.AppStateChange, metaData db.DBAppMetaData) error { + h := s.DBHandler + err := h.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData) + if err != nil { + return err + } + + cache, err := h.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return err + } + if cache == nil { + logger.FromContext(ctx).Sugar().Warnf("overview was nil, will skip update for app %s", appName) + return nil + } + + shouldDelete := stateChange == db.AppStateChangeDelete + err = s.UpdateTopLevelAppInOverview(ctx, transaction, appName, cache, shouldDelete) + if err != nil { + return err + } + + for envGroupIndex := range cache.EnvironmentGroups { + envGroup := cache.EnvironmentGroups[envGroupIndex] + + for i := range envGroup.Environments { + env := envGroup.Environments[i] + + if shouldDelete { + delete(env.Applications, appName) + } else { + envApp, err := s.UpdateOneAppEnvInOverview(ctx, transaction, appName, env.Name, nil) + if err != nil { + return err + } + + if env.Applications == nil { + env.Applications = map[string]*api.Environment_Application{} + } + env.Applications[appName] = envApp + } + } + } + + err = h.WriteOverviewCache(ctx, transaction, cache) + if err != nil { + return err + } + + return nil +} + +func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sql.Tx, appName string, result *api.GetOverviewResponse, deleteApp bool) error { + if deleteApp { + delete(result.Applications, appName) + return nil + } + app := api.Application{ + UndeploySummary: 0, + Warnings: nil, + Name: appName, + Releases: []*api.Release{}, + SourceRepoUrl: "", + Team: "", + } + if rels, err := s.GetAllApplicationReleases(ctx, transaction, appName); err != nil { + logger.FromContext(ctx).Sugar().Warnf("app without releases: %v", err) + // continue, apps are not required to have releases + } else { + for _, id := range rels { + if rel, err := s.GetApplicationRelease(ctx, transaction, appName, id); err != nil { + return err + } else { + if rel == nil { + // ignore + } else { + release := rel.ToProto() + release.Version = id + release.UndeployVersion = rel.UndeployVersion + app.Releases = append(app.Releases, release) + } + } + } + } + if team, err := s.GetApplicationTeamOwner(ctx, transaction, appName); err != nil { + return err + } else { + app.Team = team + } + if result == nil { + return nil + } + app.UndeploySummary = deriveUndeploySummary(appName, result.EnvironmentGroups) + app.Warnings = CalculateWarnings(ctx, app.Name, result.EnvironmentGroups) + if result.Applications == nil { + result.Applications = map[string]*api.Application{} + } + result.Applications[appName] = &app + return nil +} + +func getEnvironmentByName(groups []*api.EnvironmentGroup, envNameToReturn string) *api.Environment { + for _, currentGroup := range groups { + for _, currentEnv := range currentGroup.Environments { + if currentEnv.Name == envNameToReturn { + return currentEnv + } + } + } + return nil +} + +/* +CalculateWarnings returns warnings for the User to be displayed in the UI. +For really unusual configurations, these will be logged and not returned. +*/ +func CalculateWarnings(_ context.Context, appName string, groups []*api.EnvironmentGroup) []*api.Warning { + result := make([]*api.Warning, 0) + for e := 0; e < len(groups); e++ { + group := groups[e] + for i := 0; i < len(groups[e].Environments); i++ { + env := group.Environments[i] + if env == nil || env.Config == nil || env.Config.Upstream == nil || env.Config.Upstream.Environment == nil { + // if the env has no upstream, there's nothing to warn about + continue + } + upstreamEnvName := env.Config.GetUpstream().Environment + upstreamEnv := getEnvironmentByName(groups, *upstreamEnvName) + if upstreamEnv == nil { + // this is already checked on startup and therefore shouldn't happen here + continue + } + + appInEnv := env.Applications[appName] + if appInEnv == nil { + // appName is not deployed here, ignore it + continue + } + versionInEnv := appInEnv.Version + appInUpstreamEnv := upstreamEnv.Applications[appName] + if appInUpstreamEnv == nil { + // appName is not deployed upstream... that's unusual! + var warning = api.Warning{ + WarningType: &api.Warning_UpstreamNotDeployed{ + UpstreamNotDeployed: &api.UpstreamNotDeployed{ + UpstreamEnvironment: *upstreamEnvName, + ThisVersion: versionInEnv, + ThisEnvironment: env.Name, + }, + }, + } + result = append(result, &warning) + continue + } + versionInUpstreamEnv := appInUpstreamEnv.Version + + if versionInEnv > versionInUpstreamEnv && len(appInEnv.Locks) == 0 { + var warning = api.Warning{ + WarningType: &api.Warning_UnusualDeploymentOrder{ + UnusualDeploymentOrder: &api.UnusualDeploymentOrder{ + UpstreamVersion: versionInUpstreamEnv, + UpstreamEnvironment: *upstreamEnvName, + ThisVersion: versionInEnv, + ThisEnvironment: env.Name, + }, + }, + } + result = append(result, &warning) + } + } + } + return result +} + +func deriveUndeploySummary(appName string, groups []*api.EnvironmentGroup) api.UndeploySummary { + var allNormal = true + var allUndeploy = true + for _, group := range groups { + for _, environment := range group.Environments { + var app, exists = environment.Applications[appName] + if !exists { + continue + } + if app.Version == 0 { + // if the app exists but nothing is deployed, we ignore this + continue + } + if app.UndeployVersion { + allNormal = false + } else { + allUndeploy = false + } + } + } + if allUndeploy { + return api.UndeploySummary_UNDEPLOY + } + if allNormal { + return api.UndeploySummary_NORMAL + } + return api.UndeploySummary_MIXED + +} + +func (s *State) UpdateOneAppEnvInOverview(ctx context.Context, transaction *sql.Tx, appName string, envName string, configParam *config.EnvironmentConfig) (*api.Environment_Application, error) { + var envConfig = configParam + if envConfig == nil { + var err error + envConfig, err = s.GetEnvironmentConfig(ctx, transaction, envName) + if err != nil { + return nil, fmt.Errorf("could not get environment to update app %s in env %s: %w", appName, envName, err) + } + } + + app := api.Environment_Application{ + Version: 0, + QueuedVersion: 0, + UndeployVersion: false, + ArgoCd: nil, + Name: appName, + Locks: map[string]*api.Lock{}, + TeamLocks: map[string]*api.Lock{}, + Team: "", + DeploymentMetaData: &api.Environment_Application_DeploymentMetaData{ + DeployAuthor: "", + DeployTime: "", + }, + } + teamName, err := s.GetTeamName(ctx, transaction, appName) + if err == nil { + app.Team = teamName + if teamLocks, teamErr := s.GetEnvironmentTeamLocks(ctx, transaction, envName, teamName); teamErr != nil { + return nil, teamErr + } else { + for lockId, lock := range teamLocks { + app.TeamLocks[lockId] = &api.Lock{ + Message: lock.Message, + LockId: lockId, + CreatedAt: timestamppb.New(lock.CreatedAt), + CreatedBy: &api.Actor{ + Name: lock.CreatedBy.Name, + Email: lock.CreatedBy.Email, + }, + } + } + } + } // Err != nil means no team name was found so no need to parse team locks + + var version *uint64 + version, err = s.GetEnvironmentApplicationVersion(ctx, transaction, envName, appName) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + + if version == nil { + app.Version = 0 + } else { + app.Version = *version + } + } + + if queuedVersion, err := s.GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + if queuedVersion == nil { + app.QueuedVersion = 0 + } else { + app.QueuedVersion = *queuedVersion + } + } + app.UndeployVersion = false + if version != nil { + if release, err := s.GetApplicationRelease(ctx, transaction, appName, *version); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if release != nil { + app.UndeployVersion = release.UndeployVersion + } + } + + if appLocks, err := s.GetEnvironmentApplicationLocks(ctx, transaction, envName, appName); err != nil { + return nil, err + } else { + for lockId, lock := range appLocks { + app.Locks[lockId] = &api.Lock{ + Message: lock.Message, + LockId: lockId, + CreatedAt: timestamppb.New(lock.CreatedAt), + CreatedBy: &api.Actor{ + Name: lock.CreatedBy.Name, + Email: lock.CreatedBy.Email, + }, + } + } + } + if envConfig != nil && envConfig.ArgoCd != nil { + if syncWindows, err := mapper.TransformSyncWindows(envConfig.ArgoCd.SyncWindows, appName); err != nil { + return nil, err + } else { + app.ArgoCd = &api.Environment_Application_ArgoCD{ + SyncWindows: syncWindows, + } + } + } + deployAuthor, deployTime, err := s.GetDeploymentMetaData(ctx, transaction, envName, appName) + if err != nil { + return nil, err + } + app.DeploymentMetaData.DeployAuthor = deployAuthor + if deployTime.IsZero() { + app.DeploymentMetaData.DeployTime = "" + } else { + app.DeploymentMetaData.DeployTime = fmt.Sprintf("%d", deployTime.Unix()) + } + return &app, nil +} + func (s *State) GetAppsAndTeams() (map[string]string, error) { result, err := s.GetApplicationsFromFile() if err != nil { @@ -2835,7 +3157,7 @@ func (s *State) ProcessQueue(ctx context.Context, transaction *sql.Tx, fs billy. if currentlyDeployedVersion != nil && *queuedVersion == *currentlyDeployedVersion { // delete queue, it's outdated! But if we can't, that's not really a problem, as it would be overwritten // whenever the next deployment happens: - err = s.DeleteQueuedVersion(ctx, transaction, environment, application) + err = s.DeleteQueuedVersion(ctx, transaction, environment, application, false) return fmt.Sprintf("deleted queued version %d because it was already deployed. app=%q env=%q", *queuedVersion, application, environment), err } } diff --git a/services/cd-service/pkg/repository/repository_test.go b/services/cd-service/pkg/repository/repository_test.go index c4f91bd06..3c2968d39 100644 --- a/services/cd-service/pkg/repository/repository_test.go +++ b/services/cd-service/pkg/repository/repository_test.go @@ -21,6 +21,7 @@ import ( "database/sql" "errors" "fmt" + "google.golang.org/protobuf/testing/protocmp" "net/http" "net/http/httptest" "os" @@ -2164,3 +2165,226 @@ func setupRepositoryTestAux(t *testing.T, commits uint) (Repository, error) { } return repo, nil } + +func TestUpdateOverviewCache(t *testing.T) { + tcs := []struct { + Name string + InitialCache *api.GetOverviewResponse + StateChange db.AppStateChange + ExpectedOverview *api.GetOverviewResponse + }{ + { + Name: "overview=nil", + StateChange: db.AppStateChangeCreate, + InitialCache: nil, + ExpectedOverview: &api.GetOverviewResponse{ + Applications: map[string]*api.Application{ + "app1": { + Name: "app1", + Releases: nil, + SourceRepoUrl: "", + Team: "", + UndeploySummary: api.UndeploySummary_UNDEPLOY, + Warnings: nil, + }, + }, + EnvironmentGroups: []*api.EnvironmentGroup{}, + }, + }, + { + Name: "overview creates new app", + StateChange: db.AppStateChangeCreate, + InitialCache: &api.GetOverviewResponse{ + Applications: nil, + EnvironmentGroups: []*api.EnvironmentGroup{ + { + EnvironmentGroupName: "dev", + Environments: []*api.Environment{ + { + Name: "dev", + Config: nil, + Locks: nil, + Applications: nil, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + GitRevision: "123", + Branch: "main", + ManifestRepoUrl: "https://example.com", + }, + ExpectedOverview: &api.GetOverviewResponse{ + Applications: map[string]*api.Application{ + "app1": { + Name: "app1", + Releases: nil, + SourceRepoUrl: "", + Team: "", + UndeploySummary: api.UndeploySummary_UNDEPLOY, + Warnings: nil, + }, + }, + EnvironmentGroups: []*api.EnvironmentGroup{ + { + EnvironmentGroupName: "dev", + Environments: []*api.Environment{ + { + Name: "dev", + Config: nil, + Locks: nil, + Applications: map[string]*api.Environment_Application{ + "app1": { + Name: "app1", + Version: 0, + Locks: nil, + QueuedVersion: 0, + UndeployVersion: false, + ArgoCd: nil, + DeploymentMetaData: &api.Environment_Application_DeploymentMetaData{ + DeployAuthor: "", + DeployTime: "", + }, + TeamLocks: nil, + Team: "", + }, + }, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + GitRevision: "123", + Branch: "main", + ManifestRepoUrl: "https://example.com", + }, + }, + { + Name: "overview deletes an app", + StateChange: db.AppStateChangeDelete, + InitialCache: &api.GetOverviewResponse{ + Applications: map[string]*api.Application{ + "app1": { + Name: "app1", + Releases: nil, + SourceRepoUrl: "", + Team: "", + UndeploySummary: api.UndeploySummary_UNDEPLOY, + Warnings: nil, + }, + }, + EnvironmentGroups: []*api.EnvironmentGroup{ + { + EnvironmentGroupName: "dev", + Environments: []*api.Environment{ + { + Name: "dev", + Config: nil, + Locks: nil, + Applications: map[string]*api.Environment_Application{ + "app1": { + Name: "app1", + Version: 0, + Locks: nil, + QueuedVersion: 0, + UndeployVersion: false, + ArgoCd: nil, + DeploymentMetaData: &api.Environment_Application_DeploymentMetaData{ + DeployAuthor: "", + DeployTime: "", + }, + TeamLocks: nil, + Team: "", + }, + }, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + GitRevision: "123", + Branch: "main", + ManifestRepoUrl: "https://example.com", + }, + ExpectedOverview: &api.GetOverviewResponse{ + Applications: nil, + EnvironmentGroups: []*api.EnvironmentGroup{ + { + EnvironmentGroupName: "dev", + Environments: []*api.Environment{ + { + Name: "dev", + Config: nil, + Locks: nil, + Applications: nil, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + DistanceToUpstream: 0, + Priority: 0, + }, + }, + GitRevision: "123", + Branch: "main", + ManifestRepoUrl: "https://example.com", + }, + }, + } + + for _, tc := range tcs { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + ctx := testutil.MakeTestContext() + + repo := SetupRepositoryTestWithDBOptions(t, false) + state := repo.State() + dbHandler := state.DBHandler + + err := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error { + + // GIVEN: + err := dbHandler.WriteOverviewCache( + ctx, + transaction, + tc.InitialCache, + ) + if err != nil { + return err + } + + // WHEN: + metaData := db.DBAppMetaData{ + Team: "", + } + err = state.DBInsertApplicationWithOverview(ctx, transaction, "app1", db.InitialEslVersion-1, tc.StateChange, metaData) + if err != nil { + return err + } + + actualOverview, err := dbHandler.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return err + } + if diff := cmp.Diff(tc.ExpectedOverview, actualOverview, protocmp.Transform()); diff != "" { + t.Fatalf("overview mismatch (-want, +got):\n%s", diff) + } + + return nil + }) + if err != nil { + t.Fatalf("transaction error: %v", err) + } + }) + } +} diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index 8303e5b95..16c728231 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -504,7 +504,8 @@ func (c *CreateApplicationVersion) Transform( } ver = app.EslVersion + 1 } - err = state.DBHandler.DBInsertApplication( + + err = state.DBHandler.InsertAppFun( ctx, transaction, c.Application, @@ -761,6 +762,7 @@ func (c *CreateApplicationVersion) Transform( Author: c.SourceAuthor, CiLink: c.CiLink, TransformerEslVersion: c.TransformerEslVersion, + SkipOverview: false, } err := t.Execute(d, transaction) if err != nil { @@ -1364,6 +1366,7 @@ func (c *CreateUndeployApplicationVersion) Transform( Author: "", TransformerEslVersion: c.TransformerEslVersion, CiLink: "", + SkipOverview: false, } err := t.Execute(d, transaction) if err != nil { @@ -1505,7 +1508,7 @@ func (u *UndeployApplication) Transform( deployment.Version = nil deployment.Metadata.DeployedByName = user.Name deployment.Metadata.DeployedByEmail = user.Email - err = state.DBHandler.DBWriteDeployment(ctx, transaction, *deployment, deployment.EslVersion) + err = state.DBHandler.DBWriteDeployment(ctx, transaction, *deployment, deployment.EslVersion, false) if err != nil { return "", err } @@ -1573,7 +1576,7 @@ func (u *UndeployApplication) Transform( if err != nil { return "", fmt.Errorf("UndeployApplication: could not select app '%s': %v", u.Application, err) } - err = state.DBHandler.DBInsertApplication(ctx, transaction, dbApp.App, dbApp.EslVersion, db.AppStateChangeDelete, db.DBAppMetaData{Team: dbApp.Metadata.Team}) + err = state.DBHandler.InsertAppFun(ctx, transaction, dbApp.App, dbApp.EslVersion, db.AppStateChangeDelete, db.DBAppMetaData{Team: dbApp.Metadata.Team}) if err != nil { return "", fmt.Errorf("UndeployApplication: could not insert app '%s': %v", u.Application, err) } @@ -2721,9 +2724,10 @@ func (c *CreateEnvironment) Transform( } type QueueApplicationVersion struct { - Environment string - Application string - Version uint64 + Environment string + Application string + Version uint64 + SkipOverview bool } func (c *QueueApplicationVersion) Transform( @@ -2733,8 +2737,8 @@ func (c *QueueApplicationVersion) Transform( transaction *sql.Tx, ) (string, error) { if state.DBHandler.ShouldUseOtherTables() { - version := (int64(c.Version)) - err := state.DBHandler.DBWriteDeploymentAttempt(ctx, transaction, c.Environment, c.Application, &version) + version := int64(c.Version) + err := state.DBHandler.DBWriteDeploymentAttempt(ctx, transaction, c.Environment, c.Application, &version, c.SkipOverview) if err != nil { return "", err } @@ -2768,6 +2772,7 @@ type DeployApplicationVersion struct { Author string `json:"author"` CiLink string `json:"cilink"` TransformerEslVersion db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslVersion + SkipOverview bool `json:"-"` } func (c *DeployApplicationVersion) GetDBEventType() db.EventType { @@ -2896,9 +2901,10 @@ func (c *DeployApplicationVersion) Transform( switch c.LockBehaviour { case api.LockBehavior_RECORD: q := QueueApplicationVersion{ - Environment: c.Environment, - Application: c.Application, - Version: c.Version, + Environment: c.Environment, + Application: c.Application, + Version: c.Version, + SkipOverview: c.SkipOverview, } return q.Transform(ctx, state, t, transaction) case api.LockBehavior_FAIL: @@ -2961,7 +2967,7 @@ func (c *DeployApplicationVersion) Transform( } else { previousVersion = existingDeployment.EslVersion } - err = state.DBHandler.DBWriteDeployment(ctx, transaction, newDeployment, previousVersion) + err = state.DBHandler.DBWriteDeployment(ctx, transaction, newDeployment, previousVersion, c.SkipOverview) if err != nil { return "", fmt.Errorf("could not write deployment for %v - %v", newDeployment, err) } @@ -3033,7 +3039,7 @@ func (c *DeployApplicationVersion) Transform( ReleaseVersionsLimit: state.ReleaseVersionsLimit, CloudRunClient: state.CloudRunClient, } - err = s.DeleteQueuedVersionIfExists(ctx, transaction, c.Environment, c.Application) + err = s.DeleteQueuedVersionIfExists(ctx, transaction, c.Environment, c.Application, c.SkipOverview) if err != nil { return "", err } @@ -3983,6 +3989,17 @@ func (c *envReleaseTrain) Transform( } sort.Strings(appNames) + var overview *api.GetOverviewResponse + var envOfOverview *api.Environment + if state.DBHandler.ShouldUseOtherTables() { + var err error + overview, err = state.DBHandler.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return "", grpc.InternalError(ctx, fmt.Errorf("unexpected error for env=%s while reading overview cache: %w", c.Env, err)) + } + envOfOverview = getEnvOfOverview(overview, c.Env) + } + for _, appName := range appNames { appPrognosis := prognosis.AppsPrognoses[appName] if appPrognosis.SkipCause != nil { @@ -4003,10 +4020,23 @@ func (c *envReleaseTrain) Transform( Author: "", TransformerEslVersion: c.TransformerEslVersion, CiLink: c.CiLink, + SkipOverview: true, } if err := t.Execute(d, transaction); err != nil { return "", grpc.InternalError(ctx, fmt.Errorf("unexpected error while deploying app %q to env %q: %w", appName, c.Env, err)) } + + if envOfOverview != nil { + err := state.UpdateTopLevelAppInOverview(ctx, transaction, appName, overview, false) + if err != nil { + return "", grpc.InternalError(ctx, fmt.Errorf("unexpected error while updating top level app %q to env %q: %w", appName, c.Env, err)) + } + envApp, err := state.UpdateOneAppEnvInOverview(ctx, transaction, appName, c.Env, &envConfig) + if err != nil { + return "", grpc.InternalError(ctx, fmt.Errorf("unexpected error while updating top level app %q to env %q: %w", appName, c.Env, err)) + } + envOfOverview.Applications[appName] = envApp + } } teamInfo := "" if c.Parent.Team != "" { @@ -4023,14 +4053,35 @@ func (c *envReleaseTrain) Transform( if checker.SkipCause != nil { deployedApps += 1 } + } + if state.DBHandler.ShouldUseOtherTables() { + err := state.DBHandler.WriteOverviewCache(ctx, transaction, overview) + if err != nil { + return "", grpc.InternalError(ctx, fmt.Errorf("unexpected error for env=%s while writing overview cache: %w", c.Env, err)) + } } + return fmt.Sprintf("Release Train to '%s' environment:\n\n"+ "The release train deployed %d services from '%s' to '%s'%s", c.Env, deployedApps, source, c.Env, teamInfo, ), nil } +func getEnvOfOverview(overview *api.GetOverviewResponse, envName string) *api.Environment { + if overview == nil || overview.EnvironmentGroups == nil { + return nil + } + for _, envGroup := range overview.EnvironmentGroups { + for _, env := range envGroup.Environments { + if env.Name == envName { + return env + } + } + } + return nil +} + // skippedServices is a helper Transformer to generate the "skipped // services" commit log. type skippedServices struct { diff --git a/services/cd-service/pkg/repository/transformer_db_test.go b/services/cd-service/pkg/repository/transformer_db_test.go index 4580f0d96..acc28f2a4 100644 --- a/services/cd-service/pkg/repository/transformer_db_test.go +++ b/services/cd-service/pkg/repository/transformer_db_test.go @@ -1292,7 +1292,7 @@ func TestDeleteQueueApplicationVersion(t *testing.T) { if err != nil { t.Fatalf("expected no error, got %v", err) } - err2 := state.DeleteQueuedVersion(ctx, transaction, envProduction, testAppName) + err2 := state.DeleteQueuedVersion(ctx, transaction, envProduction, testAppName, true) if err2 != nil { t.Fatalf("expected no error, got %v", err2) } diff --git a/services/cd-service/pkg/service/batch.go b/services/cd-service/pkg/service/batch.go index 257e3822d..580524926 100644 --- a/services/cd-service/pkg/service/batch.go +++ b/services/cd-service/pkg/service/batch.go @@ -237,6 +237,7 @@ func (d *BatchServer) processAction( Author: "", CiLink: "", //Only gets populated when a release is created or release train is conducted. TransformerEslVersion: 0, + SkipOverview: false, }, nil, nil case *api.BatchAction_DeleteEnvFromApp: act := action.DeleteEnvFromApp diff --git a/services/cd-service/pkg/service/overview.go b/services/cd-service/pkg/service/overview.go index 54f251565..c94a13ec6 100644 --- a/services/cd-service/pkg/service/overview.go +++ b/services/cd-service/pkg/service/overview.go @@ -21,12 +21,10 @@ import ( "database/sql" "errors" "fmt" - "os" + "github.com/freiheit-com/kuberpult/pkg/mapper" "sync" "sync/atomic" - "github.com/freiheit-com/kuberpult/pkg/mapper" - "github.com/freiheit-com/kuberpult/pkg/grpc" "github.com/freiheit-com/kuberpult/pkg/logger" "go.uber.org/zap" @@ -177,105 +175,11 @@ func (o *OverviewServiceServer) getOverview( return nil, err } else { for _, appName := range apps { - teamName, err := s.GetTeamName(ctx, transaction, appName) - app := api.Environment_Application{ - Version: 0, - QueuedVersion: 0, - UndeployVersion: false, - ArgoCd: nil, - Name: appName, - Locks: map[string]*api.Lock{}, - TeamLocks: map[string]*api.Lock{}, - Team: teamName, - DeploymentMetaData: &api.Environment_Application_DeploymentMetaData{ - DeployAuthor: "", - DeployTime: "", - }, - } - if err == nil { - if teamLocks, teamErr := s.GetEnvironmentTeamLocks(ctx, transaction, envName, teamName); teamErr != nil { - return nil, teamErr - } else { - for lockId, lock := range teamLocks { - app.TeamLocks[lockId] = &api.Lock{ - Message: lock.Message, - LockId: lockId, - CreatedAt: timestamppb.New(lock.CreatedAt), - CreatedBy: &api.Actor{ - Name: lock.CreatedBy.Name, - Email: lock.CreatedBy.Email, - }, - } - } - } - } // Err != nil means no team name was found so no need to parse team locks - - var version *uint64 - version, err = s.GetEnvironmentApplicationVersion(ctx, transaction, envName, appName) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else { - - if version == nil { - app.Version = 0 - } else { - app.Version = *version - } - } - - if queuedVersion, err := s.GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else { - if queuedVersion == nil { - app.QueuedVersion = 0 - } else { - app.QueuedVersion = *queuedVersion - } - } - app.UndeployVersion = false - if version != nil { - if release, err := s.GetApplicationRelease(ctx, transaction, appName, *version); err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else if release != nil { - app.UndeployVersion = release.UndeployVersion - } - } - - if appLocks, err := s.GetEnvironmentApplicationLocks(ctx, transaction, envName, appName); err != nil { - return nil, err - } else { - for lockId, lock := range appLocks { - app.Locks[lockId] = &api.Lock{ - Message: lock.Message, - LockId: lockId, - CreatedAt: timestamppb.New(lock.CreatedAt), - CreatedBy: &api.Actor{ - Name: lock.CreatedBy.Name, - Email: lock.CreatedBy.Email, - }, - } - } - } - if config.ArgoCd != nil { - if syncWindows, err := mapper.TransformSyncWindows(config.ArgoCd.SyncWindows, appName); err != nil { - return nil, err - } else { - app.ArgoCd = &api.Environment_Application_ArgoCD{ - SyncWindows: syncWindows, - } - } - } - deployAuthor, deployTime, err := s.GetDeploymentMetaData(ctx, transaction, envName, appName) - if err != nil { - return nil, err - } - app.DeploymentMetaData.DeployAuthor = deployAuthor - if deployTime.IsZero() { - app.DeploymentMetaData.DeployTime = "" - } else { - app.DeploymentMetaData.DeployTime = fmt.Sprintf("%d", deployTime.Unix()) + app, err2 := s.UpdateOneAppEnvInOverview(ctx, transaction, appName, envName, &config) + if err2 != nil { + return nil, err2 } - env.Applications[appName] = &app + env.Applications[appName] = app } } envInGroup.Applications = env.Applications @@ -285,40 +189,10 @@ func (o *OverviewServiceServer) getOverview( return nil, err } else { for _, appName := range apps { - app := api.Application{ - UndeploySummary: 0, - Warnings: nil, - Name: appName, - Releases: []*api.Release{}, - SourceRepoUrl: "", - Team: "", - } - if rels, err := s.GetAllApplicationReleases(ctx, transaction, appName); err != nil { - return nil, err - } else { - for _, id := range rels { - if rel, err := s.GetApplicationRelease(ctx, transaction, appName, id); err != nil { - return nil, err - } else { - if rel == nil { - // ignore - } else { - release := rel.ToProto() - release.Version = id - release.UndeployVersion = rel.UndeployVersion - app.Releases = append(app.Releases, release) - } - } - } - } - if team, err := s.GetApplicationTeamOwner(ctx, transaction, appName); err != nil { - return nil, err - } else { - app.Team = team + err2 := s.UpdateTopLevelAppInOverview(ctx, transaction, appName, &result, false) + if err2 != nil { + return nil, err2 } - app.UndeploySummary = deriveUndeploySummary(appName, result.EnvironmentGroups) - app.Warnings = CalculateWarnings(ctx, app.Name, result.EnvironmentGroups) - result.Applications[appName] = &app } } @@ -326,98 +200,6 @@ func (o *OverviewServiceServer) getOverview( return &result, nil } -/* -CalculateWarnings returns warnings for the User to be displayed in the UI. -For really unusual configurations, these will be logged and not returned. -*/ -func CalculateWarnings(ctx context.Context, appName string, groups []*api.EnvironmentGroup) []*api.Warning { - result := make([]*api.Warning, 0) - for e := 0; e < len(groups); e++ { - group := groups[e] - for i := 0; i < len(groups[e].Environments); i++ { - env := group.Environments[i] - if env.Config.Upstream == nil || env.Config.Upstream.Environment == nil { - // if the env has no upstream, there's nothing to warn about - continue - } - upstreamEnvName := env.Config.GetUpstream().Environment - upstreamEnv := getEnvironmentByName(groups, *upstreamEnvName) - if upstreamEnv == nil { - // this is already checked on startup and therefore shouldn't happen here - continue - } - - appInEnv := env.Applications[appName] - if appInEnv == nil { - // appName is not deployed here, ignore it - continue - } - versionInEnv := appInEnv.Version - appInUpstreamEnv := upstreamEnv.Applications[appName] - if appInUpstreamEnv == nil { - // appName is not deployed upstream... that's unusual! - var warning = api.Warning{ - WarningType: &api.Warning_UpstreamNotDeployed{ - UpstreamNotDeployed: &api.UpstreamNotDeployed{ - UpstreamEnvironment: *upstreamEnvName, - ThisVersion: versionInEnv, - ThisEnvironment: env.Name, - }, - }, - } - result = append(result, &warning) - continue - } - versionInUpstreamEnv := appInUpstreamEnv.Version - - if versionInEnv > versionInUpstreamEnv && len(appInEnv.Locks) == 0 { - var warning = api.Warning{ - WarningType: &api.Warning_UnusualDeploymentOrder{ - UnusualDeploymentOrder: &api.UnusualDeploymentOrder{ - UpstreamVersion: versionInUpstreamEnv, - UpstreamEnvironment: *upstreamEnvName, - ThisVersion: versionInEnv, - ThisEnvironment: env.Name, - }, - }, - } - result = append(result, &warning) - } - } - } - return result -} - -func deriveUndeploySummary(appName string, groups []*api.EnvironmentGroup) api.UndeploySummary { - var allNormal = true - var allUndeploy = true - for _, group := range groups { - for _, environment := range group.Environments { - var app, exists = environment.Applications[appName] - if !exists { - continue - } - if app.Version == 0 { - // if the app exists but nothing is deployed, we ignore this - continue - } - if app.UndeployVersion { - allNormal = false - } else { - allUndeploy = false - } - } - } - if allUndeploy { - return api.UndeploySummary_UNDEPLOY - } - if allNormal { - return api.UndeploySummary_NORMAL - } - return api.UndeploySummary_MIXED - -} - func getEnvironmentInGroup(groups []*api.EnvironmentGroup, groupNameToReturn string, envNameToReturn string) *api.Environment { for _, currentGroup := range groups { if currentGroup.EnvironmentGroupName == groupNameToReturn { @@ -431,17 +213,6 @@ func getEnvironmentInGroup(groups []*api.EnvironmentGroup, groupNameToReturn str return nil } -func getEnvironmentByName(groups []*api.EnvironmentGroup, envNameToReturn string) *api.Environment { - for _, currentGroup := range groups { - for _, currentEnv := range currentGroup.Environments { - if currentEnv.Name == envNameToReturn { - return currentEnv - } - } - } - return nil -} - func (o *OverviewServiceServer) StreamOverview(in *api.GetOverviewRequest, stream api.OverviewService_StreamOverviewServer) error { ch, unsubscribe := o.subscribe() diff --git a/services/cd-service/pkg/service/overview_test.go b/services/cd-service/pkg/service/overview_test.go index 92eb9df28..c36c23c10 100644 --- a/services/cd-service/pkg/service/overview_test.go +++ b/services/cd-service/pkg/service/overview_test.go @@ -50,164 +50,6 @@ func (m *mockOverviewService_StreamOverviewServer) Context() context.Context { return m.Ctx } -func makeApps(apps ...*api.Environment_Application) map[string]*api.Environment_Application { - var result map[string]*api.Environment_Application = map[string]*api.Environment_Application{} - for i := 0; i < len(apps); i++ { - app := apps[i] - result[app.Name] = app - } - return result -} - -func makeEnv(envName string, groupName string, upstream *api.EnvironmentConfig_Upstream, apps map[string]*api.Environment_Application) *api.Environment { - return &api.Environment{ - Name: envName, - Config: &api.EnvironmentConfig{ - Upstream: upstream, - EnvironmentGroup: &groupName, - }, - Locks: map[string]*api.Lock{}, - - Applications: apps, - DistanceToUpstream: 0, - Priority: api.Priority_UPSTREAM, // we are 1 away from prod, hence pre-prod - } -} - -func makeApp(appName string, version uint64) *api.Environment_Application { - return &api.Environment_Application{ - Name: appName, - Version: version, - Locks: nil, - QueuedVersion: 0, - UndeployVersion: false, - ArgoCd: nil, - } -} -func makeEnvGroup(envGroupName string, environments []*api.Environment) *api.EnvironmentGroup { - return &api.EnvironmentGroup{ - EnvironmentGroupName: envGroupName, - Environments: environments, - DistanceToUpstream: 0, - } -} - -func makeUpstreamLatest() *api.EnvironmentConfig_Upstream { - f := true - return &api.EnvironmentConfig_Upstream{ - Latest: &f, - } -} - -func makeUpstreamEnv(upstream string) *api.EnvironmentConfig_Upstream { - return &api.EnvironmentConfig_Upstream{ - Environment: &upstream, - } -} - -func TestCalculateWarnings(t *testing.T) { - var dev = "dev" - tcs := []struct { - Name string - AppName string - Groups []*api.EnvironmentGroup - ExpectedWarnings []*api.Warning - }{ - { - Name: "no envs - no warning", - AppName: "foo", - Groups: []*api.EnvironmentGroup{ - makeEnvGroup(dev, []*api.Environment{ - makeEnv("dev-de", dev, makeUpstreamLatest(), nil), - })}, - ExpectedWarnings: []*api.Warning{}, - }, - { - Name: "app deployed in higher version on upstream should warn", - AppName: "foo", - Groups: []*api.EnvironmentGroup{ - makeEnvGroup(dev, []*api.Environment{ - makeEnv("prod", dev, makeUpstreamEnv("dev"), - makeApps(makeApp("foo", 2))), - }), - makeEnvGroup(dev, []*api.Environment{ - makeEnv("dev", dev, makeUpstreamLatest(), - makeApps(makeApp("foo", 1))), - }), - }, - ExpectedWarnings: []*api.Warning{ - { - WarningType: &api.Warning_UnusualDeploymentOrder{ - UnusualDeploymentOrder: &api.UnusualDeploymentOrder{ - UpstreamVersion: 1, - UpstreamEnvironment: "dev", - ThisVersion: 2, - ThisEnvironment: "prod", - }, - }, - }, - }, - }, - { - Name: "app deployed in same version on upstream should not warn", - AppName: "foo", - Groups: []*api.EnvironmentGroup{ - makeEnvGroup(dev, []*api.Environment{ - makeEnv("prod", dev, makeUpstreamEnv("dev"), - makeApps(makeApp("foo", 2))), - }), - makeEnvGroup(dev, []*api.Environment{ - makeEnv("dev", dev, makeUpstreamLatest(), - makeApps(makeApp("foo", 2))), - }), - }, - ExpectedWarnings: []*api.Warning{}, - }, - { - Name: "app deployed in no version on upstream should warn", - AppName: "foo", - Groups: []*api.EnvironmentGroup{ - makeEnvGroup(dev, []*api.Environment{ - makeEnv("prod", dev, makeUpstreamEnv("dev"), - makeApps(makeApp("foo", 1))), - }), - makeEnvGroup(dev, []*api.Environment{ - makeEnv("dev", dev, makeUpstreamLatest(), - makeApps()), - }), - }, - ExpectedWarnings: []*api.Warning{ - { - WarningType: &api.Warning_UpstreamNotDeployed{ - UpstreamNotDeployed: &api.UpstreamNotDeployed{ - UpstreamEnvironment: "dev", - ThisVersion: 1, - ThisEnvironment: "prod", - }, - }, - }, - }, - }, - } - for _, tc := range tcs { - tc := tc - t.Run(tc.Name, func(t *testing.T) { - actualWarnings := CalculateWarnings(testutil.MakeTestContext(), tc.AppName, tc.Groups) - if len(actualWarnings) != len(tc.ExpectedWarnings) { - t.Errorf("Different number of warnings. got: %s\nwant: %s", actualWarnings, tc.ExpectedWarnings) - } - for i := 0; i < len(actualWarnings); i++ { - actualWarning := actualWarnings[i] - expectedWarning := tc.ExpectedWarnings[i] - if diff := cmp.Diff(actualWarning.String(), expectedWarning.String()); diff != "" { - t.Errorf("Different warning at index [%d]:\ngot: %s\nwant: %s", i, actualWarning, expectedWarning) - } - } - }) - } - -} - func TestOverviewService(t *testing.T) { var dev = "dev" var upstreamLatest = true @@ -976,150 +818,3 @@ func TestOverviewServiceFromCommit(t *testing.T) { }) } } - -func groupFromEnvs(environments []*api.Environment) []*api.EnvironmentGroup { - return []*api.EnvironmentGroup{ - { - EnvironmentGroupName: "group1", - Environments: environments, - }, - } -} - -func TestDeriveUndeploySummary(t *testing.T) { - var tcs = []struct { - Name string - AppName string - groups []*api.EnvironmentGroup - ExpectedResult api.UndeploySummary - }{ - { - Name: "No Environments", - AppName: "foo", - groups: []*api.EnvironmentGroup{}, - ExpectedResult: api.UndeploySummary_UNDEPLOY, - }, - { - Name: "one Environment but no Application", - AppName: "foo", - groups: groupFromEnvs([]*api.Environment{ - { - Applications: map[string]*api.Environment_Application{ - "bar": { // different app - UndeployVersion: true, - Version: 666, - }, - }, - }, - }), - ExpectedResult: api.UndeploySummary_UNDEPLOY, - }, - { - Name: "One Env with undeploy", - AppName: "foo", - groups: groupFromEnvs([]*api.Environment{ - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: true, - Version: 666, - }, - }, - }, - }), - ExpectedResult: api.UndeploySummary_UNDEPLOY, - }, - { - Name: "One Env with normal version", - AppName: "foo", - groups: groupFromEnvs([]*api.Environment{ - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: false, - Version: 666, - }, - }, - }, - }), - ExpectedResult: api.UndeploySummary_NORMAL, - }, - { - Name: "Two Envs all undeploy", - AppName: "foo", - groups: groupFromEnvs([]*api.Environment{ - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: true, - Version: 666, - }, - }, - }, - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: true, - Version: 666, - }, - }, - }, - }), - ExpectedResult: api.UndeploySummary_UNDEPLOY, - }, - { - Name: "Two Envs all normal", - AppName: "foo", - groups: groupFromEnvs([]*api.Environment{ - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: false, - Version: 666, - }, - }, - }, - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: false, - Version: 666, - }, - }, - }, - }), - ExpectedResult: api.UndeploySummary_NORMAL, - }, - { - Name: "Two Envs all different", - AppName: "foo", - groups: groupFromEnvs([]*api.Environment{ - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: true, - Version: 666, - }, - }, - }, - { - Applications: map[string]*api.Environment_Application{ - "foo": { - UndeployVersion: false, - Version: 666, - }, - }, - }, - }), - ExpectedResult: api.UndeploySummary_MIXED, - }, - } - for _, tc := range tcs { - t.Run(tc.Name, func(t *testing.T) { - actualResult := deriveUndeploySummary(tc.AppName, tc.groups) - if !cmp.Equal(tc.ExpectedResult, actualResult) { - t.Fatal("Output mismatch (-want +got):\n", cmp.Diff(tc.ExpectedResult, actualResult)) - } - }) - } -} diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 0cdef3b74..d592e7bdc 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -845,7 +845,7 @@ func TestReleaseTrain(t *testing.T) { Env: "production", Version: &v, TransformerID: 5, - }, 10) + }, 10, false) if err != nil { return err } diff --git a/services/manifest-repo-export-service/pkg/service/version_test.go b/services/manifest-repo-export-service/pkg/service/version_test.go index 8041790e7..4b318bbfe 100644 --- a/services/manifest-repo-export-service/pkg/service/version_test.go +++ b/services/manifest-repo-export-service/pkg/service/version_test.go @@ -254,7 +254,7 @@ func TestVersion(t *testing.T) { App: "test", Env: "development", Version: &version, - }, 0) + }, 0, false) err = repo.Apply(ctx, transaction, tc.Setup...) if err != nil { return err