From 5f3c279c15f52bf383dbc02a05bb6a4239b9fb06 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Wed, 16 Oct 2024 11:32:13 +0200 Subject: [PATCH 01/17] feat(git): Allow exporting only strictly required data for Argo CD Ref: SRX-2CSAEK --- .../manifest-repo-export-service.yaml | 2 + charts/kuberpult/tests/charts_test.go | 41 ++ charts/kuberpult/values.yaml | 6 + docker-compose.yml | 3 +- pkg/db/db.go | 52 ++- pkg/db/overview.go | 59 ++- pkg/valid/validations.go | 8 + .../cd-service/pkg/repository/repository.go | 30 +- .../cd-service/pkg/repository/transformer.go | 7 +- services/cd-service/pkg/service/overview.go | 79 ++-- .../pkg/cmd/server.go | 17 +- .../pkg/repository/repository.go | 149 +++++-- .../pkg/repository/transformer.go | 367 +++++++++++------- .../pkg/repository/transformer_test.go | 2 +- .../pkg/service/git_test.go | 16 +- .../pkg/service/version_test.go | 13 +- 16 files changed, 622 insertions(+), 229 deletions(-) diff --git a/charts/kuberpult/templates/manifest-repo-export-service.yaml b/charts/kuberpult/templates/manifest-repo-export-service.yaml index 20af96bb5..df793a1a4 100644 --- a/charts/kuberpult/templates/manifest-repo-export-service.yaml +++ b/charts/kuberpult/templates/manifest-repo-export-service.yaml @@ -231,6 +231,8 @@ spec: value: "{{ .Values.datadogProfiling.enabled }}" - name: KUBERPULT_RELEASE_VERSIONS_LIMIT value: "{{ .Values.git.releaseVersionsLimit }}" + - name: KUBERPULT_MINIMIZE_EXPORTED_DATA + value: "{{ .Values.git.minimizeExportedData }}" volumeMounts: - name: repository # The repository volume, an emptyDir, is mounted to the kp directory. diff --git a/charts/kuberpult/tests/charts_test.go b/charts/kuberpult/tests/charts_test.go index 8c27edb6b..d5ab7c17d 100644 --- a/charts/kuberpult/tests/charts_test.go +++ b/charts/kuberpult/tests/charts_test.go @@ -1051,6 +1051,47 @@ manifestRepoExport: }, ExpectedMissing: []core.EnvVar{}, }, + { + Name: "Git MinimizeExportData mode explicitely set", + Values: ` +git: + url: "testURL" + minimizeExportedData: true +ingress: + domainName: "kuberpult-example.com" +db: + dbOption: "postgreSQL" + writeEslTableOnly: false + sslMode: disable +`, + ExpectedEnvs: []core.EnvVar{ + { + Name: "KUBERPULT_MINIMIZE_EXPORTED_DATA", + Value: "true", + }, + }, + ExpectedMissing: []core.EnvVar{}, + }, + { + Name: "Git MinimizeExportData mode default value", + Values: ` +git: + url: "testURL" +ingress: + domainName: "kuberpult-example.com" +db: + dbOption: "postgreSQL" + writeEslTableOnly: false + sslMode: disable +`, + ExpectedEnvs: []core.EnvVar{ + { + Name: "KUBERPULT_MINIMIZE_EXPORTED_DATA", + Value: "false", + }, + }, + ExpectedMissing: []core.EnvVar{}, + }, { Name: "DB ssl mode", Values: ` diff --git a/charts/kuberpult/values.yaml b/charts/kuberpult/values.yaml index e0c2638a6..845e9913d 100644 --- a/charts/kuberpult/values.yaml +++ b/charts/kuberpult/values.yaml @@ -67,6 +67,12 @@ git: # Values outside of the range will fail the service during startup. releaseVersionsLimit: 20 + # When minimizeExportedData==false, all data will be written to git, including meta information that is not relevant for argoCd, like locks. + # When true, only the files needed for Argo CD to work are written. + # Note that when this option is true, in case of data loss in the database, kuberpult can *not* recover all information from the git repo. + # Recommendation: Only activate this option, if frequent database backups are made and if the git repo takes up too many resources. + minimizeExportedData: false + hub: europe-west3-docker.pkg.dev/fdc-public-docker-registry/kuberpult kubernetesEngine: "GKE" diff --git a/docker-compose.yml b/docker-compose.yml index 1f8abb0f7..ea82915f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: - KUBERPULT_DEX_RBAC_POLICY_PATH=/etc/policy.csv - KUBERPULT_DEX_RBAC_TEAM_PATH=/etc/team.csv - KUBERPULT_DEX_MOCK_ROLE=developer - - KUBERPULT_GIT_WRITE_COMMIT_DATA=true + - KUBERPULT_GIT_WRITE_COMMIT_DATA=false - KUBERPULT_MAXIMUM_QUEUE_SIZE=2 - KUBERPULT_ALLOW_LONG_APP_NAMES=true - KUBERPULT_ARGO_CD_GENERATE_FILES=true @@ -74,6 +74,7 @@ services: - KUBERPULT_DB_SSL_MODE=disable - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=5 - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=1 + - KUBERPULT_MINIMIZE_EXPORTED_DATA=false volumes: - ./services/cd-service:/kp/kuberpult - ./database:/kp/database diff --git a/pkg/db/db.go b/pkg/db/db.go index 7645df285..326e29299 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -883,6 +883,33 @@ func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) } +func (h *DBHandler) DBSelectLastReleasesByApp(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { + span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByApp") + defer span.Finish() + selectQuery := h.AdaptQuery(fmt.Sprintf( + "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + + " FROM releases " + + " WHERE appName=? AND deleted=?" + + " ORDER BY eslVersion DESC, releaseVersion DESC" + + " LIMIT 1;")) + span.SetTag("query", selectQuery) + rows, err := tx.QueryContext( + ctx, + selectQuery, + app, + deleted, + ) + + releases, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) + if err != nil { + return nil, err + } + if len(releases) == 0 { + return nil, nil + } + return releases[0], nil +} + func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectAllReleasesOfApp") defer span.Finish() @@ -2287,6 +2314,9 @@ type DBAppWithMetaData struct { func (h *DBHandler) DBInsertApplication(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error { span, ctx := tracer.StartSpanFromContext(ctx, "DBInsertApplication") defer span.Finish() + log := logger.FromContext(ctx).Sugar() + log.Warnf("plain dbinsert app: %s/%v", appName, stateChange) + jsonToInsert, err := json.Marshal(metaData) if err != nil { return fmt.Errorf("could not marshal json data: %w", err) @@ -2419,6 +2449,25 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro if err != nil { return nil, fmt.Errorf("could not query apps table from DB. Error: %w\n", err) } + return processAppRow(ctx, rows) +} + +func (h *DBHandler) DBSelectExistingApp(ctx context.Context, tx *sql.Tx, appName string) (*DBAppWithMetaData, error) { + app, err := h.DBSelectApp(ctx, tx, appName) + if err != nil { + return nil, err + } + if app == nil { + return nil, nil + } + if app.StateChange == AppStateChangeDelete { + return nil, nil + } + return app, nil +} + +func processAppRow(ctx context.Context, rows *sql.Rows) (*DBAppWithMetaData, error) { + //exhaustruct:ignore defer func(rows *sql.Rows) { err := rows.Close() if err != nil { @@ -2426,7 +2475,6 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro } }(rows) - //exhaustruct:ignore var row = &DBAppWithMetaData{} if rows.Next() { var metadataStr string @@ -2446,7 +2494,7 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro } else { row = nil } - err = closeRows(rows) + err := closeRows(rows) if err != nil { return nil, err } diff --git a/pkg/db/overview.go b/pkg/db/overview.go index 3d4224e92..8b50e07f9 100644 --- a/pkg/db/overview.go +++ b/pkg/db/overview.go @@ -114,6 +114,63 @@ func (h *DBHandler) UpdateOverviewEnvironmentLock(ctx context.Context, transacti return nil } +func (h *DBHandler) UpdateOverviewDeployment(ctx context.Context, transaction *sql.Tx, deployment Deployment, createdTime time.Time) error { + latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return err + } + if h.IsOverviewEmpty(latestOverview) { + return nil + } + env := getEnvironmentByName(latestOverview.EnvironmentGroups, deployment.Env) + if env == nil { + return fmt.Errorf("could not find environment %s in overview", deployment.Env) + } + + selectApp, err := h.DBSelectExistingApp(ctx, transaction, deployment.App) + if err != nil { + return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", deployment.App, err) + } + if selectApp == nil { + return fmt.Errorf("could not find application '%s' in apps table: got no result", deployment.App) + } + + err = h.WriteOverviewCache(ctx, transaction, latestOverview) + if err != nil { + return err + } + return nil +} + +func (h *DBHandler) UpdateOverviewDeploymentAttempt(ctx context.Context, transaction *sql.Tx, queuedDeployment *QueuedDeployment) error { + latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return err + } + if h.IsOverviewEmpty(latestOverview) { + return nil + } + if queuedDeployment == nil { + return nil + } + env := getEnvironmentByName(latestOverview.EnvironmentGroups, queuedDeployment.Env) + if env == nil { + return fmt.Errorf("could not find environment %s in overview", queuedDeployment.Env) + } + selectApp, err := h.DBSelectExistingApp(ctx, transaction, queuedDeployment.App) + if err != nil { + return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", queuedDeployment.App, err) + } + if selectApp == nil { + return fmt.Errorf("could not find application '%s' in apps table: got no result", queuedDeployment.App) + } + err = h.WriteOverviewCache(ctx, transaction, latestOverview) + if err != nil { + return err + } + return nil +} + func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transaction *sql.Tx, applicationLock ApplicationLock, createdTime time.Time) error { latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) if err != nil { @@ -126,7 +183,7 @@ func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transacti if env == nil { return fmt.Errorf("could not find environment %s in overview", applicationLock.Env) } - selectApp, err := h.DBSelectApp(ctx, transaction, applicationLock.App) + selectApp, err := h.DBSelectExistingApp(ctx, transaction, applicationLock.App) if err != nil { return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", applicationLock.App, err) } diff --git a/pkg/valid/validations.go b/pkg/valid/validations.go index 1dcfec142..9ac76a12b 100644 --- a/pkg/valid/validations.go +++ b/pkg/valid/validations.go @@ -105,3 +105,11 @@ func ReadEnvVarUInt(envName string) (uint, error) { } return uint(i), nil } + +func ReadEnvVarBool(envName string) (bool, error) { + envValue, err := ReadEnvVar(envName) + if err != nil { + return false, err + } + return envValue == "true", nil +} diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index b62ee9588..fc399446b 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -2462,6 +2462,9 @@ func (s *State) WriteAllCommitEvents(ctx context.Context, transaction *sql.Tx, d } func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion db.EslVersion, stateChange db.AppStateChange, metaData db.DBAppMetaData) error { + log := logger.FromContext(ctx).Sugar() + log.Warnf("dbinsert app with overview: %s/%v", appName, stateChange) + h := s.DBHandler err := h.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData) if err != nil { @@ -2478,12 +2481,11 @@ func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction } shouldDelete := stateChange == db.AppStateChangeDelete - if shouldDelete { lApps := make([]*api.OverviewApplication, 0) for _, curr := range cache.LightweightApps { - if curr.Name != appName { + if curr != nil && curr.Name != appName { lApps = append(lApps, curr) } } @@ -2533,6 +2535,30 @@ func (s *State) DBInsertEnvironmentWithOverview(ctx context.Context, tx *sql.Tx, return nil } +func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sql.Tx, appName string, result *api.GetOverviewResponse, deleteApp bool, allReleasesOfAllApps map[string][]int64) error { + if deleteApp { + removeOverviewAppFromLightweightApps(result, appName) + return nil + } + team, err := s.GetApplicationTeamOwner(ctx, transaction, appName) + if err != nil { + return fmt.Errorf("could not obtain application team owner to update top level app in overview: %w", err) + } + result.LightweightApps = append(result.LightweightApps, &api.OverviewApplication{Name: appName, Team: team}) + return nil +} + +func removeOverviewAppFromLightweightApps(result *api.GetOverviewResponse, appName string) { + lApps := make([]*api.OverviewApplication, max(len(result.LightweightApps)-1, 0)) + + for _, curr := range result.LightweightApps { + if curr != nil && curr.Name != appName { + lApps = append(lApps, curr) + } + } + result.LightweightApps = lApps +} + func getEnvironmentInGroup(groups []*api.EnvironmentGroup, groupNameToReturn string, envNameToReturn string) *api.Environment { for _, currentGroup := range groups { if currentGroup.EnvironmentGroupName == groupNameToReturn { diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index a68ad452f..1b5135e22 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -1493,8 +1493,8 @@ func (u *UndeployApplication) GetDBEventType() db.EventType { return db.EvtUndeployApplication } -func (c *UndeployApplication) SetEslVersion(id db.TransformerID) { - c.TransformerEslVersion = id +func (u *UndeployApplication) SetEslVersion(id db.TransformerID) { + u.TransformerEslVersion = id } func (u *UndeployApplication) Transform( @@ -1616,10 +1616,11 @@ func (u *UndeployApplication) Transform( if err != nil { return "", fmt.Errorf("UndeployApplication: could not write all apps '%v': '%w'", u.Application, err) } - dbApp, err := state.DBHandler.DBSelectApp(ctx, transaction, u.Application) + dbApp, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, u.Application) if err != nil { return "", fmt.Errorf("UndeployApplication: could not select app '%s': %v", u.Application, err) } + logger.FromContext(ctx).Sugar().Warnf("before insert app fun") err = state.DBHandler.InsertAppFun(ctx, transaction, dbApp.App, dbApp.EslVersion, db.AppStateChangeDelete, db.DBAppMetaData{Team: dbApp.Metadata.Team}) if err != nil { return "", fmt.Errorf("UndeployApplication: could not insert app '%s': %v", u.Application, err) diff --git a/services/cd-service/pkg/service/overview.go b/services/cd-service/pkg/service/overview.go index b06de293c..ccc590098 100644 --- a/services/cd-service/pkg/service/overview.go +++ b/services/cd-service/pkg/service/overview.go @@ -126,7 +126,7 @@ func (o *OverviewServiceServer) GetAppDetails( sort.Slice(result.Releases, func(i, j int) bool { return result.Releases[j].Version < result.Releases[i].Version }) - if app, err := o.DBHandler.DBSelectApp(ctx, transaction, appName); err != nil { + if app, err := o.DBHandler.DBSelectExistingApp(ctx, transaction, appName); err != nil { return nil, err } else { if app == nil { @@ -190,31 +190,47 @@ func (o *OverviewServiceServer) GetAppDetails( return nil, fmt.Errorf("could not obtain deployments for app %s: %w", appName, err) } for envName, currentDeployment := range deployments { - deployment := &api.Deployment{ - Version: uint64(*currentDeployment.Version), - QueuedVersion: 0, - UndeployVersion: false, - DeploymentMetaData: &api.Deployment_DeploymentMetaData{ - CiLink: currentDeployment.Metadata.CiLink, - DeployAuthor: currentDeployment.Metadata.DeployedByName, - DeployTime: currentDeployment.Created.String(), - }, + environment, err := o.DBHandler.DBSelectEnvironment(ctx, transaction, envName) + if err != nil { + return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err) } - if queuedVersion, err := o.Repository.State().GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else { - if queuedVersion == nil { - deployment.QueuedVersion = 0 - } else { - deployment.QueuedVersion = *queuedVersion + if environment == nil { + return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err) + } + foundApp := false + for _, appInEnv := range environment.Applications { + if appInEnv == appName { + foundApp = true + break } } + if foundApp { + deployment := &api.Deployment{ + Version: uint64(*currentDeployment.Version), + QueuedVersion: 0, + UndeployVersion: false, + DeploymentMetaData: &api.Deployment_DeploymentMetaData{ + CiLink: currentDeployment.Metadata.CiLink, + DeployAuthor: currentDeployment.Metadata.DeployedByName, + DeployTime: currentDeployment.Created.String(), + }, + } + if queuedVersion, err := o.Repository.State().GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + if queuedVersion == nil { + deployment.QueuedVersion = 0 + } else { + deployment.QueuedVersion = *queuedVersion + } + } - rel := getReleaseFromVersion(releases, uint64(*currentDeployment.Version)) - if rel != nil { - deployment.UndeployVersion = rel.Metadata.UndeployVersion + rel := getReleaseFromVersion(releases, uint64(*currentDeployment.Version)) + if rel != nil { + deployment.UndeployVersion = rel.Metadata.UndeployVersion + } + response.Deployments[envName] = deployment } - response.Deployments[envName] = deployment } result.UndeploySummary = deriveUndeploySummary(appName, response.Deployments) result.Warnings = CalculateWarnings(deployments, appLocks, envGroups) @@ -224,6 +240,11 @@ func (o *OverviewServiceServer) GetAppDetails( return nil, err } response.Application = resultApp + + //span2, ctx := tracer.StartSpanFromContext(ctx, "Delay") + //time.Sleep(1000 * time.Millisecond) + //defer span2.Finish() + return response, nil } @@ -352,7 +373,17 @@ func (o *OverviewServiceServer) StreamOverview(in *api.GetOverviewRequest, case <-o.Shutdown: return nil case <-ch: - ov := o.response.Load().(*api.GetOverviewResponse) + loaded := o.response.Load() + var ov *api.GetOverviewResponse = nil + if loaded == nil { + ov, err := o.getOverviewDB(stream.Context(), o.Repository.State()) + if err != nil { + return fmt.Errorf("could not load overview") + } + o.response.Store(ov) + } else { + ov = loaded.(*api.GetOverviewResponse) + } if err := stream.Send(ov); err != nil { // if we don't log this here, the details will be lost - so this is an exception to the rule "either return an error or log it". @@ -467,6 +498,10 @@ func (o *OverviewServiceServer) update(s *repository.State) { logger.FromContext(o.Context).Error("error getting overview:", zap.Error(err)) return } + if r == nil { + logger.FromContext(o.Context).Error("overview is nil") + return + } o.response.Store(r) o.notify.Notify() } diff --git a/services/manifest-repo-export-service/pkg/cmd/server.go b/services/manifest-repo-export-service/pkg/cmd/server.go index 97242c622..fa1b5010a 100755 --- a/services/manifest-repo-export-service/pkg/cmd/server.go +++ b/services/manifest-repo-export-service/pkg/cmd/server.go @@ -154,6 +154,11 @@ func Run(ctx context.Context) error { return fmt.Errorf("error parsing KUBERPULT_RELEASE_VERSIONS_LIMIT, error: %w", err) } + minimizeExportedData, err := valid.ReadEnvVarBool("KUBERPULT_MINIMIZE_EXPORTED_DATA") + if err != nil { + return err + } + var eslProcessingIdleTimeSeconds uint64 if val, exists := os.LookupEnv("KUBERPULT_ESL_PROCESSING_BACKOFF"); !exists { log.Infof("environment variable KUBERPULT_ESL_PROCESSING_BACKOFF is not set, using default backoff of 10 seconds") @@ -237,11 +242,13 @@ func Run(ctx context.Context) error { Certificates: repository.Certificates{ KnownHostsFile: gitSshKnownHosts, }, - Branch: gitBranch, - NetworkTimeout: time.Duration(networkTimeoutSeconds) * time.Second, - ReleaseVersionLimit: uint(releaseVersionLimit), - ArgoCdGenerateFiles: argoCdGenerateFiles, - DBHandler: dbHandler, + Branch: gitBranch, + NetworkTimeout: time.Duration(networkTimeoutSeconds) * time.Second, + ReleaseVersionLimit: uint(releaseVersionLimit), + ArgoCdGenerateFiles: argoCdGenerateFiles, + MinimizeExportedData: minimizeExportedData, + + DBHandler: dbHandler, } repo, err := repository.New(ctx, cfg) diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 783336058..86b83f5f0 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -131,9 +131,12 @@ type RepositoryConfig struct { // network timeout NetworkTimeout time.Duration + DBHandler *db.DBHandler + ArgoCdGenerateFiles bool ReleaseVersionLimit uint - DBHandler *db.DBHandler + + MinimizeExportedData bool } func openOrCreate(path string) (*git.Repository, error) { @@ -275,7 +278,10 @@ func New(ctx context.Context, cfg RepositoryConfig) (Repository, error) { } // Check configuration for errors and abort early if any: - _, err = state.GetEnvironmentConfigsAndValidate(ctx) + err = state.DBHandler.WithTransaction(ctx, true, func(ctx context.Context, transaction *sql.Tx) error { + _, err = state.GetEnvironmentConfigsAndValidate(ctx, transaction) + return err + }) if err != nil { return nil, err } @@ -443,7 +449,7 @@ func (r *repository) ApplyTransformersInternal(ctx context.Context, transaction } return nil, nil, nil, &applyErr } - if msg, subChanges, err := RunTransformer(ctxWithTime, transformer, state, transaction); err != nil { + if msg, subChanges, err := RunTransformer(ctxWithTime, transformer, state, transaction, r.config.MinimizeExportedData); err != nil { applyErr := TransformerBatchApplyError{ TransformerError: err, Index: 0, @@ -697,7 +703,7 @@ func (r *repository) afterTransform(ctx context.Context, transaction *sql.Tx, st span, ctx := tracer.StartSpanFromContext(ctx, "afterTransform") defer span.Finish() - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return err } @@ -727,11 +733,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, transaction *sql.Tx, appData := []argocd.AppData{} sort.Strings(apps) for _, appName := range apps { - if err != nil { - return err - } - //team, err := state.DBHandler.DBSelectApp().GetApplicationTeamOwner(ctx, transaction, appName) - oneAppData, err := state.DBHandler.DBSelectApp(ctx, transaction, appName) + oneAppData, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, appName) if err != nil { return fmt.Errorf("updateArgoCdApps: could not select app '%s' in db %v", appName, err) } @@ -1051,6 +1053,76 @@ func (s *State) GetEnvironmentTeamLocks(environment, team string) (map[string]Lo return result, nil } } + +func (s *State) GetEnvironmentApplicationLocksFromDB(ctx context.Context, transaction *sql.Tx, environment, application string) (map[string]Lock, error) { + if transaction == nil { + return nil, fmt.Errorf("GetEnvironmentApplicationLocksFromDB: No transaction provided") + } + activeLockIds, err := s.DBHandler.DBSelectAllAppLocks(ctx, transaction, environment, application) + if err != nil { + return nil, err + } + var lockIds []string + if activeLockIds != nil { + lockIds = activeLockIds.AppLocks + } + locks, err := s.DBHandler.DBSelectAppLockSet(ctx, transaction, environment, application, lockIds) + + if err != nil { + return nil, err + } + result := make(map[string]Lock, len(locks)) + for _, lock := range locks { + genericLock := Lock{ + Message: lock.Metadata.Message, + CreatedBy: Actor{ + Name: lock.Metadata.CreatedByName, + Email: lock.Metadata.CreatedByEmail, + }, + CreatedAt: lock.Created, + } + result[lock.LockID] = genericLock + } + return result, nil +} + +func (s *State) GetEnvironmentTeamLocksFromDB(ctx context.Context, transaction *sql.Tx, environment, team string) (map[string]Lock, error) { + if team == "" { + return map[string]Lock{}, nil + } + + if transaction == nil { + return nil, fmt.Errorf("GetEnvironmentTeamLocksFromDB: No transaction provided") + } + activeLockIDs, err := s.DBHandler.DBSelectAllTeamLocks(ctx, transaction, environment, team) + if err != nil { + return nil, err + } + + var lockIds []string + if activeLockIDs != nil { + lockIds = activeLockIDs.TeamLocks + } + locks, err := s.DBHandler.DBSelectTeamLockSet(ctx, transaction, environment, team, lockIds) + + if err != nil { + return nil, err + } + result := make(map[string]Lock, len(locks)) + for _, lock := range locks { + genericLock := Lock{ + Message: lock.Metadata.Message, + CreatedBy: Actor{ + Name: lock.Metadata.CreatedByName, + Email: lock.Metadata.CreatedByEmail, + }, + CreatedAt: lock.Created, + } + result[lock.LockID] = genericLock + } + return result, nil +} + func (s *State) GetDeploymentMetaData(environment, application string) (string, time.Time, error) { base := s.Filesystem.Join("environments", environment, "applications", application) author, err := readFile(s.Filesystem, s.Filesystem.Join(base, "deployed_by")) @@ -1203,9 +1275,28 @@ func envExists(envConfigs map[string]config.EnvironmentConfig, envNameToSearchFo return false } -func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context) (map[string]config.EnvironmentConfig, error) { +func (s *State) GetAllEnvironmentConfigsFromDB(ctx context.Context, transaction *sql.Tx) (map[string]config.EnvironmentConfig, error) { + dbAllEnvs, err := s.DBHandler.DBSelectAllEnvironments(ctx, transaction) + if err != nil { + return nil, fmt.Errorf("unable to retrieve all environments, error: %w", err) + } + if dbAllEnvs == nil { + return nil, nil + } + envs, err := s.DBHandler.DBSelectEnvironmentsBatch(ctx, transaction, dbAllEnvs.Environments) + if err != nil { + return nil, fmt.Errorf("unable to retrieve manifests for environments %v from the database, error: %w", dbAllEnvs.Environments, err) + } + ret := make(map[string]config.EnvironmentConfig) + for _, env := range *envs { + ret[env.Name] = env.Config + } + return ret, nil +} + +func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context, transaction *sql.Tx) (map[string]config.EnvironmentConfig, error) { logger := logger.FromContext(ctx) - envConfigs, err := s.GetEnvironmentConfigs() + envConfigs, err := s.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return nil, err } @@ -1233,22 +1324,22 @@ func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context) (map[strin return envConfigs, err } -func (s *State) GetEnvironmentConfigs() (map[string]config.EnvironmentConfig, error) { - envs, err := s.Filesystem.ReadDir("environments") - if err != nil { - return nil, err - } - result := map[string]config.EnvironmentConfig{} - for _, env := range envs { - c, err := s.GetEnvironmentConfig(env.Name()) - if err != nil { - return nil, err - - } - result[env.Name()] = *c - } - return result, nil -} +//func (s *State) GetEnvironmentConfigs() (map[string]config.EnvironmentConfig, error) { +//envs, err := s.Filesystem.ReadDir("environments") +//if err != nil { +// return nil, err +//} +//result := map[string]config.EnvironmentConfig{} +//for _, env := range envs { +// c, err := s.GetEnvironmentConfig(env.Name()) +// if err != nil { +// return nil, err +// +// } +// result[env.Name()] = *c +//} +//return result, nil +//} func (s *State) GetEnvironmentConfig(environmentName string) (*config.EnvironmentConfig, error) { fileName := s.Filesystem.Join("environments", environmentName, "config.json") @@ -1261,8 +1352,8 @@ func (s *State) GetEnvironmentConfig(environmentName string) (*config.Environmen return &config, nil } -func (s *State) GetEnvironmentConfigsForGroup(envGroup string) ([]string, error) { - allEnvConfigs, err := s.GetEnvironmentConfigs() +func (s *State) GetEnvironmentConfigsForGroup(ctx context.Context, transaction *sql.Tx, envGroup string) ([]string, error) { + allEnvConfigs, err := s.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return nil, err } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 9583893f4..cc8e63a1c 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -81,22 +81,27 @@ func versionToString(Version uint64) string { return strconv.FormatUint(Version, 10) } +// releasesDirectory returns applications//releases/ func releasesDirectory(fs billy.Filesystem, application string) string { return fs.Join("applications", application, "releases") } +// applicationDirectory returns applications// func applicationDirectory(fs billy.Filesystem, application string) string { return fs.Join("applications", application) } +// environmentApplicationDirectory returns environments//applications// func environmentApplicationDirectory(fs billy.Filesystem, environment, application string) string { return fs.Join("environments", environment, "applications", application) } +// releasesDirectoryWithVersion returns applications//releases/ func releasesDirectoryWithVersion(fs billy.Filesystem, application string, version uint64) string { return fs.Join(releasesDirectory(fs, application), versionToString(version)) } +// environmentApplicationDirectory returns applications//releases//environments/ func manifestDirectoryWithReleasesVersion(fs billy.Filesystem, application string, version uint64) string { return fs.Join(releasesDirectoryWithVersion(fs, application, version), "environments") } @@ -122,6 +127,8 @@ type TransformerContext interface { Execute(t Transformer, transaction *sql.Tx) error AddAppEnv(app string, env string, team string) DeleteEnvFromApp(app string, env string) + ShouldMinimizeGitData() bool + ShouldMaximizeGitData() bool } type TransformerMetadata struct { @@ -133,7 +140,12 @@ func (t *TransformerMetadata) GetMetadata() *TransformerMetadata { return t } -func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *sql.Tx) (string, *TransformerResult, error) { +func GetNoOpMessage(t Transformer) (string, error) { + evt := t.GetDBEventType() + return fmt.Sprintf("Empty Commit\nNo files changed in %s", evt), nil +} + +func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *sql.Tx, minimizeExportedData bool) (string, *TransformerResult, error) { runner := transformerRunner{ ChangedApps: nil, DeletedRootApps: nil, @@ -141,6 +153,8 @@ func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *s Context: ctx, State: s, Stack: [][]string{nil}, + + MinimizeGitData: minimizeExportedData, } if err := runner.Execute(t, transaction); err != nil { return "", nil, err @@ -193,6 +207,8 @@ type transformerRunner struct { ChangedApps []AppEnv DeletedRootApps []RootApp Commits *CommitIds + + MinimizeGitData bool } func (r *transformerRunner) Execute(t Transformer, transaction *sql.Tx) error { @@ -235,6 +251,14 @@ func (r *transformerRunner) DeleteEnvFromApp(app string, env string) { }) } +func (r *transformerRunner) ShouldMinimizeGitData() bool { + return r.MinimizeGitData +} + +func (r *transformerRunner) ShouldMaximizeGitData() bool { + return !r.MinimizeGitData +} + type RawNode struct{ *yaml3.Node } func (n *RawNode) UnmarshalYAML(node *yaml3.Node) error { @@ -322,7 +346,7 @@ type DeployApplicationVersionSource struct { func (c *DeployApplicationVersion) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { @@ -340,30 +364,31 @@ func (c *DeployApplicationVersion) Transform( if c.LockBehaviour != api.LockBehavior_IGNORE { // Check that the environment is not locked var ( - envLocks, appLocks, teamLocks map[string]Lock + appLocks, envLocks, teamLocks map[string]Lock err error ) - envLocks, err = state.GetEnvironmentLocks(c.Environment) + envLocks, err = state.GetEnvironmentLocksFromDB(ctx, transaction, c.Environment) if err != nil { return "", err } - appLocks, err = state.GetEnvironmentApplicationLocks(c.Environment, c.Application) + appLocks, err = state.GetEnvironmentApplicationLocksFromDB(ctx, transaction, c.Environment, c.Application) if err != nil { return "", err } - teamName, err := state.GetTeamName(c.Application) + app, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, c.Application) if err != nil { return "", err } + var teamName = "" + if app != nil && app.Metadata.Team != "" { + teamName = app.Metadata.Team + } - if errors.Is(err, os.ErrNotExist) { - teamLocks = map[string]Lock{} //If we dont find the team file, there is no team for application, meaning there can't be any team locks - } else { - teamLocks, err = state.GetEnvironmentTeamLocks(c.Environment, string(teamName)) - if err != nil { - return "", err - } + teamLocks, err = state.GetEnvironmentTeamLocksFromDB(ctx, transaction, c.Environment, teamName) + //teamLocks, err = state.GetEnvironmentTeamLocks(c.Environment, teamName) + if err != nil { + return "", err } if len(envLocks) > 0 || len(appLocks) > 0 || len(teamLocks) > 0 { @@ -375,7 +400,7 @@ func (c *DeployApplicationVersion) Transform( Version: c.Version, TransformerEslVersion: c.TransformerEslVersion, } - return q.Transform(ctx, state, t, nil) + return q.Transform(ctx, state, tCtx, transaction) case api.LockBehavior_FAIL: return "", &LockedError{ EnvironmentApplicationLocks: appLocks, @@ -386,18 +411,21 @@ func (c *DeployApplicationVersion) Transform( } } applicationDir := fsys.Join("environments", c.Environment, "applications", c.Application) - versionFile := fsys.Join(applicationDir, "version") - // Create a symlink to the release if err := fsys.MkdirAll(applicationDir, 0777); err != nil { return "", err } + versionFile := fsys.Join(applicationDir, "version") if err := fsys.Remove(versionFile); err != nil && !errors.Is(err, os.ErrNotExist) { return "", err } - if err := fsys.Symlink(fsys.Join("..", "..", "..", "..", releaseDir), versionFile); err != nil { - return "", err + + if tCtx.ShouldMaximizeGitData() { + if err := fsys.Symlink(fsys.Join("..", "..", "..", "..", releaseDir), versionFile); err != nil { + return "", err + } } + // Copy the manifest for argocd manifestsDir := fsys.Join(applicationDir, "manifests") if err := fsys.MkdirAll(manifestsDir, 0777); err != nil { @@ -414,30 +442,33 @@ func (c *DeployApplicationVersion) Transform( if err := util.WriteFile(fsys, manifestFilename, manifestContent, 0666); err != nil { return "", err } + teamOwner, err := state.GetApplicationTeamOwner(ctx, transaction, c.Application) if err != nil { return "", err } - t.AddAppEnv(c.Application, c.Environment, teamOwner) + tCtx.AddAppEnv(c.Application, c.Environment, teamOwner) existingDeployment, err := state.DBHandler.DBSelectLatestDeployment(ctx, transaction, c.Application, c.Environment) if err != nil { return "", fmt.Errorf("error while retrieving deployment: %v", err) } - logger.FromContext(ctx).Sugar().Warnf("writing deployed name...") - if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by"), []byte(existingDeployment.Metadata.DeployedByName), 0666); err != nil { - return "", err - } + if tCtx.ShouldMaximizeGitData() { + logger.FromContext(ctx).Sugar().Warnf("writing deployed name...") + if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by"), []byte(existingDeployment.Metadata.DeployedByName), 0666); err != nil { + return "", err + } - logger.FromContext(ctx).Sugar().Warnf("writing deployed email...") - if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by_email"), []byte(existingDeployment.Metadata.DeployedByEmail), 0666); err != nil { - return "", err - } + logger.FromContext(ctx).Sugar().Warnf("writing deployed email...") + if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by_email"), []byte(existingDeployment.Metadata.DeployedByEmail), 0666); err != nil { + return "", err + } - logger.FromContext(ctx).Sugar().Warnf("writing deployed at...") - if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_at_utc"), []byte(existingDeployment.Created.UTC().String()), 0666); err != nil { - return "", err + logger.FromContext(ctx).Sugar().Warnf("writing deployed at...") + if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_at_utc"), []byte(existingDeployment.Created.UTC().String()), 0666); err != nil { + return "", err + } } err = state.DeleteQueuedVersionIfExists(c.Environment, c.Application) @@ -453,7 +484,7 @@ func (c *DeployApplicationVersion) Transform( }, TransformerEslVersion: c.TransformerEslVersion, } - if err := t.Execute(d, transaction); err != nil { + if err := tCtx.Execute(d, transaction); err != nil { return "", err } return fmt.Sprintf("deployed version %d of %q to %q", c.Version, c.Application, c.Environment), nil @@ -505,9 +536,12 @@ func (c *CreateEnvironmentLock) GetDBEventType() db.EventType { func (c *CreateEnvironmentLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem envDir := fs.Join("environments", c.Environment) if _, err := fs.Stat(envDir); err != nil { @@ -591,9 +625,12 @@ func (c *DeleteEnvironmentLock) GetDBEventType() db.EventType { func (c *DeleteEnvironmentLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, _ *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem s := State{ Commit: nil, @@ -644,9 +681,12 @@ func (c *CreateEnvironmentApplicationLock) GetDBEventType() db.EventType { func (c *CreateEnvironmentApplicationLock) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem envDir := fs.Join("environments", c.Environment) if _, err := fs.Stat(envDir); err != nil { @@ -706,10 +746,12 @@ func (c *DeleteEnvironmentApplicationLock) SetEslVersion(eslVersion db.Transform func (c *DeleteEnvironmentApplicationLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { - + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem queueMessage := "" lockDir := fs.Join("environments", c.Environment, "applications", c.Application, "locks", c.LockId) @@ -763,7 +805,7 @@ func (c *CreateApplicationVersion) GetDBEventType() db.EventType { func (c *CreateApplicationVersion) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { version := c.Version @@ -774,8 +816,11 @@ func (c *CreateApplicationVersion) Transform( releaseDir := releasesDirectoryWithVersion(fs, c.Application, version) appDir := applicationDirectory(fs, c.Application) - if err := fs.MkdirAll(releaseDir, 0777); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + + if tCtx.ShouldMaximizeGitData() { + if err := fs.MkdirAll(releaseDir, 0777); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } var checkForInvalidCommitId = func(commitId, helperText string) { @@ -789,54 +834,56 @@ func (c *CreateApplicationVersion) Transform( checkForInvalidCommitId(c.SourceCommitId, "Source") checkForInvalidCommitId(c.PreviousCommit, "Previous") - if c.SourceCommitId != "" { - c.SourceCommitId = strings.ToLower(c.SourceCommitId) - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceCommitId), []byte(c.SourceCommitId), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if tCtx.ShouldMaximizeGitData() { + if c.SourceCommitId != "" { + c.SourceCommitId = strings.ToLower(c.SourceCommitId) + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceCommitId), []byte(c.SourceCommitId), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } - } - if c.SourceAuthor != "" { - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceAuthor), []byte(c.SourceAuthor), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if c.SourceAuthor != "" { + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceAuthor), []byte(c.SourceAuthor), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } - } - if c.SourceMessage != "" { - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceMessage), []byte(c.SourceMessage), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if c.SourceMessage != "" { + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceMessage), []byte(c.SourceMessage), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } - } - if c.DisplayVersion != "" { - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldDisplayVersion), []byte(c.DisplayVersion), 0666); err != nil { + if c.DisplayVersion != "" { + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldDisplayVersion), []byte(c.DisplayVersion), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } + } + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil { return "", GetCreateReleaseGeneralFailure(err) } - } - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } - if c.Team != "" { - //util.WriteFile has a bug where it does not truncate the old file content. If two application versions with the same - //team are deployed, team names simply get concatenated. Just remove the file beforehand. - //This bug can't be fixed because it is part of the util library - teamFileLoc := fs.Join(appDir, fieldTeam) - if _, err := fs.Stat(teamFileLoc); err == nil { //If path to file exists - err := fs.Remove(teamFileLoc) - if err != nil { + if c.Team != "" { + //util.WriteFile has a bug where it does not truncate the old file content. If two application versions with the same + //team are deployed, team names simply get concatenated. Just remove the file beforehand. + //This bug can'tCtx be fixed because it is part of the util library + teamFileLoc := fs.Join(appDir, fieldTeam) + if _, err := fs.Stat(teamFileLoc); err == nil { //If path to file exists + err := fs.Remove(teamFileLoc) + if err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } + } + if err := util.WriteFile(fs, teamFileLoc, []byte(c.Team), 0666); err != nil { return "", GetCreateReleaseGeneralFailure(err) } } - if err := util.WriteFile(fs, teamFileLoc, []byte(c.Team), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } } - isLatest, err := isLatestVersion(state, c.Application, version) + isLatest, err := isLatestVersion(ctx, state, transaction, c.Application, version) if err != nil { return "", GetCreateReleaseGeneralFailure(err) } if !isLatest { // check that we can actually backfill this version - oldVersions, err := findOldApplicationVersions(ctx, transaction, state, c.Application) + oldVersions, err := findOldApplicationVersions(ctx, transaction, state, int(state.ReleaseVersionsLimit), c.Application) if err != nil { return "", GetCreateReleaseGeneralFailure(err) } @@ -869,7 +916,7 @@ func (c *CreateApplicationVersion) Transform( } } - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return "", err } @@ -878,9 +925,6 @@ func (c *CreateApplicationVersion) Transform( env := sortedKeys[i] man := c.Manifests[env] - if err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } envDir := fs.Join(releaseDir, "environments", env) config, found := configs[env] @@ -889,18 +933,20 @@ func (c *CreateApplicationVersion) Transform( hasUpstream = config.Upstream != nil } - if err = fs.MkdirAll(envDir, 0777); err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } - if err := util.WriteFile(fs, fs.Join(envDir, "manifests.yaml"), []byte(man), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if tCtx.ShouldMaximizeGitData() { + if err = fs.MkdirAll(envDir, 0777); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } + if err := util.WriteFile(fs, fs.Join(envDir, "manifests.yaml"), []byte(man), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } teamOwner, err := state.GetApplicationTeamOwner(ctx, transaction, c.Application) if err != nil { return "", err } - t.AddAppEnv(c.Application, env, teamOwner) + tCtx.AddAppEnv(c.Application, env, teamOwner) if hasUpstream && config.Upstream.Latest && isLatest { d := &DeployApplicationVersion{ SourceTrain: nil, @@ -917,7 +963,7 @@ func (c *CreateApplicationVersion) Transform( AuthorEmail: "", }, } - err := t.Execute(d, transaction) + err := tCtx.Execute(d, transaction) if err != nil { _, ok := err.(*LockedError) if ok { @@ -1002,13 +1048,13 @@ func writeNextPrevInfo(ctx context.Context, sourceCommitId string, otherCommitId return nil } -func isLatestVersion(state *State, application string, version uint64) (bool, error) { - rels, err := state.GetApplicationReleasesFromFile(application) +func isLatestVersion(ctx context.Context, state *State, transaction *sql.Tx, application string, version uint64) (bool, error) { + rels, err := state.DBHandler.DBSelectReleasesByAppLatestEslVersion(ctx, transaction, application, true) if err != nil { return false, err } for _, r := range rels { - if r > version { + if r.ReleaseNumber > version { return false, nil } } @@ -1017,13 +1063,14 @@ func isLatestVersion(state *State, application string, version uint64) (bool, er // Finds old releases for an application: Checks for the oldest release that is currently deployed on any environment // Releases older that the oldest deployed release are eligible for deletion. releaseVersionsLimit -func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state *State, name string) ([]uint64, error) { +func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state *State, versionLimit int, appName string) ([]uint64, error) { // 1) get release in each env: - envConfigs, err := state.GetEnvironmentConfigs() + envConfigs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) + //envConfigs, err := state.GetEnvironmentConfigs() if err != nil { return nil, err } - versions, err := state.GetApplicationReleasesFromFile(name) + versions, err := state.GetApplicationReleasesFromFile(appName) if err != nil { return nil, err } @@ -1036,7 +1083,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state // Use the latest version as oldest deployed version oldestDeployedVersion := versions[len(versions)-1] for env := range envConfigs { - version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, name) + version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, appName) if err != nil { return nil, err } @@ -1050,13 +1097,13 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state return versions[i] >= oldestDeployedVersion }) - if positionOfOldestVersion < (int(state.ReleaseVersionsLimit) - 1) { + if positionOfOldestVersion < (versionLimit - 1) { return nil, nil } indexToKeep := positionOfOldestVersion - 1 majorsCount := 0 for ; indexToKeep >= 0; indexToKeep-- { - release, err := state.DBHandler.DBSelectReleaseByVersion(ctx, transaction, name, versions[indexToKeep], false) + release, err := state.DBHandler.DBSelectReleaseByVersion(ctx, transaction, appName, versions[indexToKeep], false) if err != nil { return nil, err } @@ -1066,7 +1113,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state } else if !release.Metadata.IsMinor && !release.Metadata.IsPrepublish { majorsCount += 1 } - if majorsCount >= int(state.ReleaseVersionsLimit) { + if majorsCount >= versionLimit { break } } @@ -1126,9 +1173,12 @@ func (c *CreateEnvironmentTeamLock) GetDBEventType() db.EventType { func (c *CreateEnvironmentTeamLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, tx *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } if !valid.EnvironmentName(c.Environment) { return "", status.Error(codes.InvalidArgument, fmt.Sprintf("cannot create environment team lock: invalid environment: '%s'", c.Environment)) @@ -1216,9 +1266,13 @@ func (c *DeleteEnvironmentTeamLock) GetDBEventType() db.EventType { func (c *DeleteEnvironmentTeamLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, _ *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } + if !valid.EnvironmentName(c.Environment) { return "", status.Error(codes.InvalidArgument, fmt.Sprintf("cannot delete environment team lock: invalid environment: '%s'", c.Environment)) } @@ -1270,11 +1324,14 @@ func (c *CreateEnvironment) GetDBEventType() db.EventType { } func (c *CreateEnvironment) Transform( - ctx context.Context, + _ context.Context, state *State, - t TransformerContext, - transaction *sql.Tx, + tCtx TransformerContext, + _ *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem envDir := fs.Join("environments", c.Environment) if err := fs.MkdirAll(envDir, 0777); err != nil { @@ -1385,11 +1442,20 @@ func (c *CleanupOldApplicationVersions) GetDBEventType() db.EventType { func (c *CleanupOldApplicationVersions) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - oldVersions, err := findOldApplicationVersions(ctx, transaction, state, c.Application) + var versionLimit = int(state.ReleaseVersionsLimit) + var err error = nil + var oldVersions []uint64 + if tCtx.ShouldMinimizeGitData() { + versionLimit = 0 + //oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) + oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) + } else { + oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) + } if err != nil { return "", fmt.Errorf("cleanup: could not get application releases for app '%s': %w", c.Application, err) } @@ -1479,20 +1545,23 @@ func (u *ReleaseTrain) Transform( ctx context.Context, state *State, t TransformerContext, - tx *sql.Tx, + transaction *sql.Tx, ) (string, error) { //Gets deployments generated by the releasetrain with elsVersion u.TransformerEslVersion from the database and simply deploys them - deployments, err := state.DBHandler.DBSelectDeploymentsByTransformerID(ctx, tx, u.TransformerEslVersion, 100) + deployments, err := state.DBHandler.DBSelectDeploymentsByTransformerID(ctx, transaction, u.TransformerEslVersion, 100) if err != nil { return "", err } - skippedDeployments, err := state.DBHandler.DBSelectAllLockPreventedEventsForTransformerID(ctx, tx, u.TransformerEslVersion) + skippedDeployments, err := state.DBHandler.DBSelectAllLockPreventedEventsForTransformerID(ctx, transaction, u.TransformerEslVersion) if err != nil { return "", err } var targetGroupName = u.Target - configs, _ := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) + if err != nil { + return "", err + } var envGroupConfigs, isEnvGroup = getEnvironmentGroupsEnvironmentsOrEnvironment(configs, targetGroupName, u.TargetType) for _, currentDeployment := range deployments { envConfig := envGroupConfigs[currentDeployment.Env] @@ -1515,7 +1584,7 @@ func (u *ReleaseTrain) Transform( }, TransformerEslVersion: u.TransformerEslVersion, Author: "", - }, tx); err != nil { + }, transaction); err != nil { return "", err } } @@ -1559,47 +1628,49 @@ type DeleteEnvFromApp struct { TransformerEslVersion db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslVersion } -func (u *DeleteEnvFromApp) GetEslVersion() db.TransformerID { - return u.TransformerEslVersion +func (c *DeleteEnvFromApp) GetEslVersion() db.TransformerID { + return c.TransformerEslVersion } func (c *DeleteEnvFromApp) SetEslVersion(eslVersion db.TransformerID) { c.TransformerEslVersion = eslVersion } -func (u *DeleteEnvFromApp) GetDBEventType() db.EventType { +func (c *DeleteEnvFromApp) GetDBEventType() db.EventType { return db.EvtDeleteEnvFromApp } -func (u *DeleteEnvFromApp) Transform( +func (c *DeleteEnvFromApp) Transform( ctx context.Context, state *State, - t TransformerContext, - transsaction *sql.Tx, + tCtx TransformerContext, + _ *sql.Tx, ) (string, error) { fs := state.Filesystem thisSprintf := func(format string, a ...any) string { - return fmt.Sprintf("DeleteEnvFromApp app '%s' on env '%s': %s", u.Application, u.Environment, fmt.Sprintf(format, a...)) + return fmt.Sprintf("DeleteEnvFromApp app '%s' on env '%s': %s", c.Application, c.Environment, fmt.Sprintf(format, a...)) } - if u.Application == "" { + if c.Application == "" { return "", fmt.Errorf(thisSprintf("Need to provide the application")) } - if u.Environment == "" { + if c.Environment == "" { return "", fmt.Errorf(thisSprintf("Need to provide the environment")) } - envAppDir := environmentApplicationDirectory(fs, u.Environment, u.Application) + envAppDir := environmentApplicationDirectory(fs, c.Environment, c.Application) entries, err := fs.ReadDir(envAppDir) + msg := fmt.Sprintf("Attempted to remove environment '%v' from application '%v' but it did not exist", c.Environment, c.Application) if err != nil { - return "", wrapFileError(err, envAppDir, thisSprintf("Could not open application directory. Does the app exist?")) + return "", wrapFileError(err, envAppDir, thisSprintf("Could not open application directory")) } if entries == nil { // app was never deployed on this env, so that's unusual - but for idempotency we treat it just like a success case: - return fmt.Sprintf("Attempted to remove environment '%v' from application '%v' but it did not exist.", u.Environment, u.Application), nil + logger.FromContext(ctx).Warn(msg) + return msg, nil } err = fs.Remove(envAppDir) @@ -1607,8 +1678,8 @@ func (u *DeleteEnvFromApp) Transform( return "", wrapFileError(err, envAppDir, thisSprintf("Cannot delete app.'")) } - t.DeleteEnvFromApp(u.Application, u.Environment) - return fmt.Sprintf("Environment '%v' was removed from application '%v' successfully.", u.Environment, u.Application), nil + tCtx.DeleteEnvFromApp(c.Application, c.Environment) + return fmt.Sprintf("Environment '%v' was removed from application '%v' successfully.", c.Environment, c.Application), nil } type CreateUndeployApplicationVersion struct { @@ -1619,15 +1690,15 @@ type CreateUndeployApplicationVersion struct { TransformerEslVersion db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslVersion } -func (u *CreateUndeployApplicationVersion) GetEslVersion() db.TransformerID { - return u.TransformerEslVersion +func (c *CreateUndeployApplicationVersion) GetEslVersion() db.TransformerID { + return c.TransformerEslVersion } func (c *CreateUndeployApplicationVersion) SetEslVersion(eslVersion db.TransformerID) { c.TransformerEslVersion = eslVersion } -func (u *CreateUndeployApplicationVersion) GetDBEventType() db.EventType { +func (c *CreateUndeployApplicationVersion) GetDBEventType() db.EventType { return db.EvtCreateUndeployApplicationVersion } @@ -1638,20 +1709,23 @@ func (c *CreateUndeployApplicationVersion) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - lastRelease, err := state.GetLastRelease(ctx, fs, c.Application) + lastRelease, err := state.DBHandler.DBSelectReleasesByAppLatestEslVersion(ctx, transaction, c.Application, false) + //lastRelease, err := state.GetLastRelease(ctx, fs, c.Application) if err != nil { - return "", fmt.Errorf("Could not get last reelase for app '%v': %v\n", c.Application, err) + return "", fmt.Errorf("Could not get last relase for app '%v': %v\n", c.Application, err) } - if lastRelease == 0 { - return "", fmt.Errorf("cannot undeploy non-existing application '%v'", c.Application) + var nextReleaseNumber uint64 = 0 + if len(lastRelease) == 0 { + return "", fmt.Errorf("cannot undeploy application '%v'", c.Application) } + nextReleaseNumber = lastRelease[0].ReleaseNumber - releaseDir := releasesDirectoryWithVersion(fs, c.Application, lastRelease+1) + releaseDir := releasesDirectoryWithVersion(fs, c.Application, nextReleaseNumber) if err = fs.MkdirAll(releaseDir, 0777); err != nil { return "", err } - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return "", err } @@ -1663,9 +1737,6 @@ func (c *CreateUndeployApplicationVersion) Transform( return "", err } for env := range configs { - if err != nil { - return "", err - } envDir := fs.Join(releaseDir, "environments", env) config, found := configs[env] @@ -1694,7 +1765,7 @@ func (c *CreateUndeployApplicationVersion) Transform( SourceTrain: nil, Environment: env, Application: c.Application, - Version: lastRelease + 1, + Version: nextReleaseNumber, // the train should queue deployments, instead of giving up: LockBehaviour: api.LockBehavior_RECORD, Authentication: c.Authentication, @@ -1717,7 +1788,7 @@ func (c *CreateUndeployApplicationVersion) Transform( } } } - return fmt.Sprintf("created undeploy-version %d of '%v'", lastRelease+1, c.Application), nil + return fmt.Sprintf("created undeploy-version %d of '%v'", nextReleaseNumber, c.Application), nil } type UndeployApplication struct { @@ -1735,8 +1806,8 @@ func (u *UndeployApplication) GetDBEventType() db.EventType { return db.EvtUndeployApplication } -func (c *UndeployApplication) SetEslVersion(id db.TransformerID) { - c.TransformerEslVersion = id +func (u *UndeployApplication) SetEslVersion(id db.TransformerID) { + u.TransformerEslVersion = id } func (u *UndeployApplication) Transform( @@ -1746,29 +1817,23 @@ func (u *UndeployApplication) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - lastRelease, err := state.GetLastRelease(ctx, fs, u.Application) + lastRelease, err := state.DBHandler.DBSelectLastReleasesByApp(ctx, transaction, u.Application, false, true) if err != nil { return "", err } - if lastRelease == 0 { - return "", fmt.Errorf("UndeployApplication: error cannot undeploy non-existing application '%v'", u.Application) - } - isUndeploy, err := state.IsUndeployVersion(u.Application, lastRelease) - if err != nil { - return "", err + if lastRelease == nil { + return "", fmt.Errorf("cannot undeploy application without releases '%v'", u.Application) } + isUndeploy := lastRelease.Metadata.UndeployVersion if !isUndeploy { - return "", fmt.Errorf("UndeployApplication: error last release is not un-deployed application version of '%v'", u.Application) + return "", fmt.Errorf("UndeployApplication: last release is not un-deployed application version of '%v'", u.Application) } appDir := applicationDirectory(fs, u.Application) - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) // we use ALL envs, to be sure if err != nil { - return "", err + return "", fmt.Errorf("could not get environment configs: %w", err) } for env := range configs { - if err != nil { - return "", err - } envAppDir := environmentApplicationDirectory(fs, env, u.Application) entries, err := fs.ReadDir(envAppDir) if err != nil { @@ -1824,14 +1889,15 @@ func (u *UndeployApplication) Transform( } } } - if err = fs.Remove(appDir); err != nil { - return "", err + + if err = fs.Remove(appDir); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", wrapFileError(err, appDir, "UndeployApplication: could not remove application directory") } for env := range configs { appDir := environmentApplicationDirectory(fs, env, u.Application) teamOwner, err := state.GetApplicationTeamOwner(ctx, transaction, u.Application) if err != nil { - return "", err + return "", fmt.Errorf("could not find team for app %s: %w", u.Application, err) } t.AddAppEnv(u.Application, env, teamOwner) // remove environment application @@ -1839,6 +1905,7 @@ func (u *UndeployApplication) Transform( return "", fmt.Errorf("UndeployApplication: unexpected error application '%v' environment '%v': '%w'", u.Application, env, err) } } + return fmt.Sprintf("application '%v' was deleted successfully", u.Application), nil } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 8b752281a..512336834 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -95,7 +95,7 @@ func setupRepositoryTestWithPath(t *testing.T) (Repository, string) { CommitterEmail: "kuberpult@freiheit.com", CommitterName: "kuberpult", ArgoCdGenerateFiles: true, - ReleaseVersionLimit: 2, + ReleaseVersionLimit: 0, } if dbConfig != nil { diff --git a/services/manifest-repo-export-service/pkg/service/git_test.go b/services/manifest-repo-export-service/pkg/service/git_test.go index f4fd50b88..c2a8e30fe 100644 --- a/services/manifest-repo-export-service/pkg/service/git_test.go +++ b/services/manifest-repo-export-service/pkg/service/git_test.go @@ -54,11 +54,12 @@ func setupRepositoryTestWithoutDB(t *testing.T) (rp.Repository, error) { t.Logf("test created dir: %s", localDir) repoCfg := rp.RepositoryConfig{ - URL: remoteDir, - Path: localDir, - CommitterEmail: "kuberpult@freiheit.com", - CommitterName: "kuberpult", - ArgoCdGenerateFiles: true, + URL: remoteDir, + Path: localDir, + CommitterEmail: "kuberpult@freiheit.com", + CommitterName: "kuberpult", + ArgoCdGenerateFiles: true, + MinimizeExportedData: false, } repoCfg.DBHandler = nil @@ -1031,8 +1032,9 @@ func TestGetCommitInfo(t *testing.T) { } config := rp.RepositoryConfig{ - ArgoCdGenerateFiles: true, - DBHandler: repo.State().DBHandler, + ArgoCdGenerateFiles: true, + DBHandler: repo.State().DBHandler, + MinimizeExportedData: false, } sv := &GitServer{ Repository: repo, diff --git a/services/manifest-repo-export-service/pkg/service/version_test.go b/services/manifest-repo-export-service/pkg/service/version_test.go index 4b318bbfe..99e6ca3ee 100644 --- a/services/manifest-repo-export-service/pkg/service/version_test.go +++ b/services/manifest-repo-export-service/pkg/service/version_test.go @@ -68,12 +68,13 @@ func setupRepositoryTestWithPath(t *testing.T) (repository.Repository, string) { } repoCfg := repository.RepositoryConfig{ - URL: remoteDir, - Path: localDir, - CommitterEmail: "kuberpult@freiheit.com", - CommitterName: "kuberpult", - ArgoCdGenerateFiles: true, - ReleaseVersionLimit: 2, + URL: remoteDir, + Path: localDir, + CommitterEmail: "kuberpult@freiheit.com", + CommitterName: "kuberpult", + ArgoCdGenerateFiles: true, + ReleaseVersionLimit: 2, + MinimizeExportedData: false, } if dbConfig != nil { From 01c051e79880a8f30563733df9f3e05374b0053b Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Tue, 5 Nov 2024 13:16:13 +0100 Subject: [PATCH 02/17] cleanup --- pkg/db/overview.go | 57 ------------------- .../cd-service/pkg/repository/repository.go | 27 --------- .../cd-service/pkg/repository/transformer.go | 1 - services/cd-service/pkg/service/overview.go | 7 +-- .../pkg/repository/repository.go | 17 ------ .../pkg/repository/transformer.go | 7 --- 6 files changed, 1 insertion(+), 115 deletions(-) diff --git a/pkg/db/overview.go b/pkg/db/overview.go index 8b50e07f9..470eec455 100644 --- a/pkg/db/overview.go +++ b/pkg/db/overview.go @@ -114,63 +114,6 @@ func (h *DBHandler) UpdateOverviewEnvironmentLock(ctx context.Context, transacti return nil } -func (h *DBHandler) UpdateOverviewDeployment(ctx context.Context, transaction *sql.Tx, deployment Deployment, createdTime time.Time) error { - latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) - if err != nil { - return err - } - if h.IsOverviewEmpty(latestOverview) { - return nil - } - env := getEnvironmentByName(latestOverview.EnvironmentGroups, deployment.Env) - if env == nil { - return fmt.Errorf("could not find environment %s in overview", deployment.Env) - } - - selectApp, err := h.DBSelectExistingApp(ctx, transaction, deployment.App) - if err != nil { - return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", deployment.App, err) - } - if selectApp == nil { - return fmt.Errorf("could not find application '%s' in apps table: got no result", deployment.App) - } - - err = h.WriteOverviewCache(ctx, transaction, latestOverview) - if err != nil { - return err - } - return nil -} - -func (h *DBHandler) UpdateOverviewDeploymentAttempt(ctx context.Context, transaction *sql.Tx, queuedDeployment *QueuedDeployment) error { - latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) - if err != nil { - return err - } - if h.IsOverviewEmpty(latestOverview) { - return nil - } - if queuedDeployment == nil { - return nil - } - env := getEnvironmentByName(latestOverview.EnvironmentGroups, queuedDeployment.Env) - if env == nil { - return fmt.Errorf("could not find environment %s in overview", queuedDeployment.Env) - } - selectApp, err := h.DBSelectExistingApp(ctx, transaction, queuedDeployment.App) - if err != nil { - return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", queuedDeployment.App, err) - } - if selectApp == nil { - return fmt.Errorf("could not find application '%s' in apps table: got no result", queuedDeployment.App) - } - err = h.WriteOverviewCache(ctx, transaction, latestOverview) - if err != nil { - return err - } - return nil -} - func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transaction *sql.Tx, applicationLock ApplicationLock, createdTime time.Time) error { latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) if err != nil { diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index fc399446b..2434892f1 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -2462,9 +2462,6 @@ func (s *State) WriteAllCommitEvents(ctx context.Context, transaction *sql.Tx, d } func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion db.EslVersion, stateChange db.AppStateChange, metaData db.DBAppMetaData) error { - log := logger.FromContext(ctx).Sugar() - log.Warnf("dbinsert app with overview: %s/%v", appName, stateChange) - h := s.DBHandler err := h.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData) if err != nil { @@ -2535,30 +2532,6 @@ func (s *State) DBInsertEnvironmentWithOverview(ctx context.Context, tx *sql.Tx, return nil } -func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sql.Tx, appName string, result *api.GetOverviewResponse, deleteApp bool, allReleasesOfAllApps map[string][]int64) error { - if deleteApp { - removeOverviewAppFromLightweightApps(result, appName) - return nil - } - team, err := s.GetApplicationTeamOwner(ctx, transaction, appName) - if err != nil { - return fmt.Errorf("could not obtain application team owner to update top level app in overview: %w", err) - } - result.LightweightApps = append(result.LightweightApps, &api.OverviewApplication{Name: appName, Team: team}) - return nil -} - -func removeOverviewAppFromLightweightApps(result *api.GetOverviewResponse, appName string) { - lApps := make([]*api.OverviewApplication, max(len(result.LightweightApps)-1, 0)) - - for _, curr := range result.LightweightApps { - if curr != nil && curr.Name != appName { - lApps = append(lApps, curr) - } - } - result.LightweightApps = lApps -} - func getEnvironmentInGroup(groups []*api.EnvironmentGroup, groupNameToReturn string, envNameToReturn string) *api.Environment { for _, currentGroup := range groups { if currentGroup.EnvironmentGroupName == groupNameToReturn { diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index 1b5135e22..3428def5e 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -1620,7 +1620,6 @@ func (u *UndeployApplication) Transform( if err != nil { return "", fmt.Errorf("UndeployApplication: could not select app '%s': %v", u.Application, err) } - logger.FromContext(ctx).Sugar().Warnf("before insert app fun") err = state.DBHandler.InsertAppFun(ctx, transaction, dbApp.App, dbApp.EslVersion, db.AppStateChangeDelete, db.DBAppMetaData{Team: dbApp.Metadata.Team}) if err != nil { return "", fmt.Errorf("UndeployApplication: could not insert app '%s': %v", u.Application, err) diff --git a/services/cd-service/pkg/service/overview.go b/services/cd-service/pkg/service/overview.go index ccc590098..c1513914c 100644 --- a/services/cd-service/pkg/service/overview.go +++ b/services/cd-service/pkg/service/overview.go @@ -197,7 +197,7 @@ func (o *OverviewServiceServer) GetAppDetails( if environment == nil { return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err) } - foundApp := false + foundApp := false // only apps that are active on that environment should be returned here for _, appInEnv := range environment.Applications { if appInEnv == appName { foundApp = true @@ -240,11 +240,6 @@ func (o *OverviewServiceServer) GetAppDetails( return nil, err } response.Application = resultApp - - //span2, ctx := tracer.StartSpanFromContext(ctx, "Delay") - //time.Sleep(1000 * time.Millisecond) - //defer span2.Finish() - return response, nil } diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 86b83f5f0..97a4f5e11 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -1324,23 +1324,6 @@ func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context, transactio return envConfigs, err } -//func (s *State) GetEnvironmentConfigs() (map[string]config.EnvironmentConfig, error) { -//envs, err := s.Filesystem.ReadDir("environments") -//if err != nil { -// return nil, err -//} -//result := map[string]config.EnvironmentConfig{} -//for _, env := range envs { -// c, err := s.GetEnvironmentConfig(env.Name()) -// if err != nil { -// return nil, err -// -// } -// result[env.Name()] = *c -//} -//return result, nil -//} - func (s *State) GetEnvironmentConfig(environmentName string) (*config.EnvironmentConfig, error) { fileName := s.Filesystem.Join("environments", environmentName, "config.json") var config config.EnvironmentConfig diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index cc8e63a1c..cd031b4d3 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -386,7 +386,6 @@ func (c *DeployApplicationVersion) Transform( } teamLocks, err = state.GetEnvironmentTeamLocksFromDB(ctx, transaction, c.Environment, teamName) - //teamLocks, err = state.GetEnvironmentTeamLocks(c.Environment, teamName) if err != nil { return "", err } @@ -455,17 +454,14 @@ func (c *DeployApplicationVersion) Transform( } if tCtx.ShouldMaximizeGitData() { - logger.FromContext(ctx).Sugar().Warnf("writing deployed name...") if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by"), []byte(existingDeployment.Metadata.DeployedByName), 0666); err != nil { return "", err } - logger.FromContext(ctx).Sugar().Warnf("writing deployed email...") if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by_email"), []byte(existingDeployment.Metadata.DeployedByEmail), 0666); err != nil { return "", err } - logger.FromContext(ctx).Sugar().Warnf("writing deployed at...") if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_at_utc"), []byte(existingDeployment.Created.UTC().String()), 0666); err != nil { return "", err } @@ -1451,7 +1447,6 @@ func (c *CleanupOldApplicationVersions) Transform( var oldVersions []uint64 if tCtx.ShouldMinimizeGitData() { versionLimit = 0 - //oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) } else { oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) @@ -1710,7 +1705,6 @@ func (c *CreateUndeployApplicationVersion) Transform( ) (string, error) { fs := state.Filesystem lastRelease, err := state.DBHandler.DBSelectReleasesByAppLatestEslVersion(ctx, transaction, c.Application, false) - //lastRelease, err := state.GetLastRelease(ctx, fs, c.Application) if err != nil { return "", fmt.Errorf("Could not get last relase for app '%v': %v\n", c.Application, err) } @@ -1905,7 +1899,6 @@ func (u *UndeployApplication) Transform( return "", fmt.Errorf("UndeployApplication: unexpected error application '%v' environment '%v': '%w'", u.Application, env, err) } } - return fmt.Sprintf("application '%v' was deleted successfully", u.Application), nil } From 522da8de623eeb47206a349d5ccc71836f5471b7 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Tue, 5 Nov 2024 13:46:14 +0100 Subject: [PATCH 03/17] linter --- .../pkg/repository/transformer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index cd031b4d3..29ca9bd40 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -1442,13 +1442,12 @@ func (c *CleanupOldApplicationVersions) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - var versionLimit = int(state.ReleaseVersionsLimit) - var err error = nil + var err error var oldVersions []uint64 if tCtx.ShouldMinimizeGitData() { - versionLimit = 0 oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) } else { + var versionLimit = int(state.ReleaseVersionsLimit) oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) } if err != nil { @@ -1708,7 +1707,7 @@ func (c *CreateUndeployApplicationVersion) Transform( if err != nil { return "", fmt.Errorf("Could not get last relase for app '%v': %v\n", c.Application, err) } - var nextReleaseNumber uint64 = 0 + var nextReleaseNumber uint64 if len(lastRelease) == 0 { return "", fmt.Errorf("cannot undeploy application '%v'", c.Application) } From dfde5687827665e0fa55581d9c92e27dfb544b7f Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Tue, 5 Nov 2024 13:59:46 +0100 Subject: [PATCH 04/17] remove outdated test --- .../pkg/repository/repository.go | 11 -- .../pkg/repository/repository_test.go | 146 ++++++------------ 2 files changed, 43 insertions(+), 114 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 97a4f5e11..83a3797a0 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -1324,17 +1324,6 @@ func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context, transactio return envConfigs, err } -func (s *State) GetEnvironmentConfig(environmentName string) (*config.EnvironmentConfig, error) { - fileName := s.Filesystem.Join("environments", environmentName, "config.json") - var config config.EnvironmentConfig - if err := decodeJsonFile(s.Filesystem, fileName, &config); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("%s : %w", fileName, InvalidJson) - } - } - return &config, nil -} - func (s *State) GetEnvironmentConfigsForGroup(ctx context.Context, transaction *sql.Tx, envGroup string) ([]string, error) { allEnvConfigs, err := s.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { diff --git a/services/manifest-repo-export-service/pkg/repository/repository_test.go b/services/manifest-repo-export-service/pkg/repository/repository_test.go index 918c09390..4c209bf8f 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository_test.go +++ b/services/manifest-repo-export-service/pkg/repository/repository_test.go @@ -17,9 +17,10 @@ Copyright freiheit.com*/ package repository import ( + "context" "fmt" + "github.com/freiheit-com/kuberpult/pkg/db" "github.com/freiheit-com/kuberpult/pkg/testutil" - "os" "os/exec" "path" "testing" @@ -31,108 +32,6 @@ import ( git "github.com/libgit2/git2go/v34" ) -func TestConfigValidity(t *testing.T) { - tcs := []struct { - Name string - ConfigContent string - ErrorExpected bool - }{ - { - Name: "Initialization with valid config.json file works", - ConfigContent: "{\"upstream\": {\"latest\": true }}", - ErrorExpected: false, - }, - { - Name: "Initialization with invalid config.json file throws error", - ConfigContent: "{\"upstream\": \"latest\": true }}", - ErrorExpected: true, - }, - } - for _, tc := range tcs { - tc := tc - t.Run(tc.Name, func(t *testing.T) { - t.Parallel() - // create a remote - workdir := t.TempDir() - remoteDir := path.Join(workdir, "remote") - cmd := exec.Command("git", "init", "--bare", remoteDir) - cmd.Start() - cmd.Wait() - - workdir = t.TempDir() - cmd = exec.Command("git", "clone", remoteDir, workdir) // Clone git dir - _, err := cmd.Output() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - t.Logf("stderr: %s\n", exitErr.Stderr) - } - t.Fatal(err) - } - - if err := os.MkdirAll(path.Join(workdir, "environments", "development"), 0700); err != nil { - t.Fatal(err) - } - - configFilePath := path.Join(workdir, "environments", "development", "config.json") - if err := os.WriteFile(configFilePath, []byte(tc.ConfigContent), 0666); err != nil { - t.Fatal(err) - } - cmd = exec.Command("git", "add", configFilePath) // Add a new file to git - cmd.Dir = workdir - _, err = cmd.Output() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - t.Logf("stderr: %s\n", exitErr.Stderr) - } - t.Fatal(err) - } - cmd = exec.Command("git", "commit", "-m", "valid config") // commit the new file - cmd.Dir = workdir - cmd.Env = []string{ - "GIT_AUTHOR_NAME=kuberpult", - "GIT_COMMITTER_NAME=kuberpult", - "EMAIL=test@kuberpult.com", - } - _, err = cmd.Output() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - t.Logf("stderr: %s\n", exitErr.Stderr) - } - t.Fatal(err) - } - cmd = exec.Command("git", "push", "origin", "HEAD") // push the new commit - cmd.Dir = workdir - _, err = cmd.Output() - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - t.Logf("stderr: %s\n", exitErr.Stderr) - } - t.Fatal(err) - } - - _, err = New( - testutil.MakeTestContext(), - RepositoryConfig{ - URL: remoteDir, - Path: t.TempDir(), - ArgoCdGenerateFiles: true, - }, - ) - - if tc.ErrorExpected { - if err == nil { - t.Errorf("Initialized even though config.json was incorrect") - } - } else { - if err != nil { - t.Errorf("Initialization failed with valid config.json") - } - } - - }) - } -} - func TestRetrySsh(t *testing.T) { tcs := []struct { Name string @@ -331,6 +230,47 @@ func setupRepositoryTest(t *testing.T) Repository { return repo } +func SetupRepositoryTestWithDB(t *testing.T) *db.DBHandler { + ctx := context.Background() + migrationsPath, err := testutil.CreateMigrationsPath(2) + if err != nil { + t.Fatalf("CreateMigrationsPath error: %v", err) + } + dbConfig := &db.DBConfig{ + MigrationsPath: migrationsPath, + DriverName: "sqlite3", + } + + dir := t.TempDir() + remoteDir := path.Join(dir, "remote") + localDir := path.Join(dir, "local") + cmd := exec.Command("git", "init", "--bare", remoteDir) + err = cmd.Start() + if err != nil { + t.Fatalf("error starting %v", err) + return nil + } + err = cmd.Wait() + if err != nil { + t.Fatalf("error waiting %v", err) + return nil + } + t.Logf("test created dir: %s", localDir) + + dbConfig.DbHost = dir + + migErr := db.RunDBMigrations(ctx, *dbConfig) + if migErr != nil { + t.Fatal(migErr) + } + + dbHandler, err := db.Connect(ctx, *dbConfig) + if err != nil { + t.Fatal(err) + } + return dbHandler +} + func TestGetTagsNoTags(t *testing.T) { name := "No tags to be returned at all" From bb7f20310f8d0b492e83b4c283962de13e261455 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Tue, 5 Nov 2024 14:23:32 +0100 Subject: [PATCH 05/17] continue-test-fixes --- .../pkg/repository/repository.go | 11 ---- .../pkg/repository/repository_test.go | 53 +++++++------------ 2 files changed, 20 insertions(+), 44 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 83a3797a0..babf65bf8 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -19,7 +19,6 @@ package repository import ( "context" "database/sql" - "encoding/json" "errors" "fmt" "io" @@ -1566,16 +1565,6 @@ func names(fs billy.Filesystem, path string) ([]string, error) { return result, nil } -func decodeJsonFile(fs billy.Filesystem, path string, out interface{}) error { - if file, err := fs.Open(path); err != nil { - return wrapFileError(err, path, "could not decode json file") - } else { - defer file.Close() - dec := json.NewDecoder(file) - return dec.Decode(out) - } -} - func readFile(fs billy.Filesystem, path string) ([]byte, error) { if file, err := fs.Open(path); err != nil { return nil, err diff --git a/services/manifest-repo-export-service/pkg/repository/repository_test.go b/services/manifest-repo-export-service/pkg/repository/repository_test.go index 4c209bf8f..f9b6dc413 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository_test.go +++ b/services/manifest-repo-export-service/pkg/repository/repository_test.go @@ -187,7 +187,7 @@ func TestDeleteDirIfEmpty(t *testing.T) { tc := tc t.Run(tc.Name, func(t *testing.T) { t.Parallel() - repo := setupRepositoryTest(t) + repo, _ := SetupRepositoryTestWithDB(t) state := repo.State() err := state.Filesystem.MkdirAll(tc.CreateThisDir, 0777) if err != nil { @@ -206,33 +206,9 @@ func TestDeleteDirIfEmpty(t *testing.T) { } } -func setupRepositoryTest(t *testing.T) Repository { - dir := t.TempDir() - remoteDir := path.Join(dir, "remote") - localDir := path.Join(dir, "local") - cmd := exec.Command("git", "init", "--bare", remoteDir) - cmd.Start() - cmd.Wait() - t.Logf("test created dir: %s", localDir) - repo, err := New( - testutil.MakeTestContext(), - RepositoryConfig{ - URL: remoteDir, - Path: localDir, - CommitterEmail: "kuberpult@freiheit.com", - CommitterName: "kuberpult", - ArgoCdGenerateFiles: true, - }, - ) - if err != nil { - t.Fatal(err) - } - return repo -} - -func SetupRepositoryTestWithDB(t *testing.T) *db.DBHandler { +func SetupRepositoryTestWithDB(t *testing.T) (Repository, *db.DBHandler) { ctx := context.Background() - migrationsPath, err := testutil.CreateMigrationsPath(2) + migrationsPath, err := testutil.CreateMigrationsPath(4) if err != nil { t.Fatalf("CreateMigrationsPath error: %v", err) } @@ -248,17 +224,14 @@ func SetupRepositoryTestWithDB(t *testing.T) *db.DBHandler { err = cmd.Start() if err != nil { t.Fatalf("error starting %v", err) - return nil + return nil, nil } err = cmd.Wait() if err != nil { t.Fatalf("error waiting %v", err) - return nil + return nil, nil } - t.Logf("test created dir: %s", localDir) - dbConfig.DbHost = dir - migErr := db.RunDBMigrations(ctx, *dbConfig) if migErr != nil { t.Fatal(migErr) @@ -268,7 +241,21 @@ func SetupRepositoryTestWithDB(t *testing.T) *db.DBHandler { if err != nil { t.Fatal(err) } - return dbHandler + repo, err := New( + testutil.MakeTestContext(), + RepositoryConfig{ + URL: remoteDir, + Path: localDir, + CommitterEmail: "kuberpult@freiheit.com", + CommitterName: "kuberpult", + ArgoCdGenerateFiles: true, + DBHandler: dbHandler, + }, + ) + if err != nil { + t.Fatal(err) + } + return repo, dbHandler } func TestGetTagsNoTags(t *testing.T) { From eceeecd8df7a7118324bbbe8be6864b0f9fee853 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Wed, 6 Nov 2024 16:55:46 +0100 Subject: [PATCH 06/17] fix linter --- pkg/db/db.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index 326e29299..f2b88f852 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -2467,14 +2467,13 @@ func (h *DBHandler) DBSelectExistingApp(ctx context.Context, tx *sql.Tx, appName } func processAppRow(ctx context.Context, rows *sql.Rows) (*DBAppWithMetaData, error) { - //exhaustruct:ignore defer func(rows *sql.Rows) { err := rows.Close() if err != nil { logger.FromContext(ctx).Sugar().Warnf("row could not be closed: %v", err) } }(rows) - + //exhaustruct:ignore var row = &DBAppWithMetaData{} if rows.Next() { var metadataStr string From 7b743c46fbfa07766de3522122cef72dd9f69d15 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Wed, 6 Nov 2024 17:21:08 +0100 Subject: [PATCH 07/17] wip --- .../pkg/repository/repository.go | 3 + .../pkg/repository/repository_test.go | 121 ++++++++++++------ 2 files changed, 82 insertions(+), 42 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index babf65bf8..288eaadfe 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -276,6 +276,9 @@ func New(ctx context.Context, cfg RepositoryConfig) (Repository, error) { return nil, err } + if state == nil || state.DBHandler == nil { + return nil, fmt.Errorf("no database configured") + } // Check configuration for errors and abort early if any: err = state.DBHandler.WithTransaction(ctx, true, func(ctx context.Context, transaction *sql.Tx) error { _, err = state.GetEnvironmentConfigsAndValidate(ctx, transaction) diff --git a/services/manifest-repo-export-service/pkg/repository/repository_test.go b/services/manifest-repo-export-service/pkg/repository/repository_test.go index f9b6dc413..da7b5b4d7 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository_test.go +++ b/services/manifest-repo-export-service/pkg/repository/repository_test.go @@ -187,7 +187,7 @@ func TestDeleteDirIfEmpty(t *testing.T) { tc := tc t.Run(tc.Name, func(t *testing.T) { t.Parallel() - repo, _ := SetupRepositoryTestWithDB(t) + repo, _, _ := SetupRepositoryTestWithDB(t) state := repo.State() err := state.Filesystem.MkdirAll(tc.CreateThisDir, 0777) if err != nil { @@ -206,7 +206,7 @@ func TestDeleteDirIfEmpty(t *testing.T) { } } -func SetupRepositoryTestWithDB(t *testing.T) (Repository, *db.DBHandler) { +func SetupRepositoryTestWithDB(t *testing.T) (Repository, *db.DBHandler, *RepositoryConfig) { ctx := context.Background() migrationsPath, err := testutil.CreateMigrationsPath(4) if err != nil { @@ -224,12 +224,12 @@ func SetupRepositoryTestWithDB(t *testing.T) (Repository, *db.DBHandler) { err = cmd.Start() if err != nil { t.Fatalf("error starting %v", err) - return nil, nil + return nil, nil, nil } err = cmd.Wait() if err != nil { t.Fatalf("error waiting %v", err) - return nil, nil + return nil, nil, nil } dbConfig.DbHost = dir migErr := db.RunDBMigrations(ctx, *dbConfig) @@ -241,21 +241,22 @@ func SetupRepositoryTestWithDB(t *testing.T) (Repository, *db.DBHandler) { if err != nil { t.Fatal(err) } + config := RepositoryConfig{ + URL: "file://" + remoteDir, + Path: localDir, + CommitterEmail: "kuberpult@freiheit.com", + CommitterName: "kuberpult", + ArgoCdGenerateFiles: true, + DBHandler: dbHandler, + } repo, err := New( testutil.MakeTestContext(), - RepositoryConfig{ - URL: remoteDir, - Path: localDir, - CommitterEmail: "kuberpult@freiheit.com", - CommitterName: "kuberpult", - ArgoCdGenerateFiles: true, - DBHandler: dbHandler, - }, + config, ) if err != nil { t.Fatal(err) } - return repo, dbHandler + return repo, dbHandler, &config } func TestGetTagsNoTags(t *testing.T) { @@ -263,36 +264,72 @@ func TestGetTagsNoTags(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - dir := t.TempDir() - remoteDir := path.Join(dir, "remote") - localDir := path.Join(dir, "local") - repoConfig := RepositoryConfig{ - URL: "file://" + remoteDir, - Path: localDir, - Branch: "master", - ArgoCdGenerateFiles: true, - } - cmd := exec.Command("git", "init", "--bare", remoteDir) - cmd.Start() - cmd.Wait() - _, err := New( - testutil.MakeTestContext(), - repoConfig, - ) - if err != nil { - t.Fatal(err) - } - tags, err := GetTags( - repoConfig, - localDir, - testutil.MakeTestContext(), - ) - if err != nil { - t.Fatalf("new: expected no error, got '%e'", err) - } - if len(tags) != 0 { - t.Fatalf("expected %v tags but got %v", 0, len(tags)) + if false { + _, _, repoConfig := SetupRepositoryTestWithDB(t) + localDir := repoConfig.Path + //dir := t.TempDir() + //remoteDir := path.Join(dir, "remote") + //localDir := path.Join(dir, "local") + //repoConfig := RepositoryConfig{ + // URL: "file://" + remoteDir, + // Path: localDir, + // Branch: "master", + // ArgoCdGenerateFiles: true, + //} + //cmd := exec.Command("git", "init", "--bare", remoteDir) + //cmd.Start() + //cmd.Wait() + //_, err := New( + // testutil.MakeTestContext(), + // repoConfig, + //) + + //if err != nil { + // t.Fatal(err) + //} + tags, err := GetTags( + *repoConfig, + localDir, + testutil.MakeTestContext(), + ) + if err != nil { + t.Fatalf("new: expected no error, got '%e'", err) + } + if len(tags) != 0 { + t.Fatalf("expected %v tags but got %v", 0, len(tags)) + } + } else { + dir := t.TempDir() + remoteDir := path.Join(dir, "remote") + localDir := path.Join(dir, "local") + repoConfig := RepositoryConfig{ + URL: "file://" + remoteDir, + Path: localDir, + Branch: "master", + ArgoCdGenerateFiles: true, + } + cmd := exec.Command("git", "init", "--bare", remoteDir) + cmd.Start() + cmd.Wait() + _, err := New( + testutil.MakeTestContext(), + repoConfig, + ) + if err != nil { + t.Fatal(err) + } + tags, err := GetTags( + repoConfig, + localDir, + testutil.MakeTestContext(), + ) + if err != nil { + t.Fatalf("new: expected no error, got '%e'", err) + } + if len(tags) != 0 { + t.Fatalf("expected %v tags but got %v", 0, len(tags)) + } } }) From 220aa5d91789c450b757ef43287312380ae9ce4f Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Wed, 6 Nov 2024 18:05:56 +0100 Subject: [PATCH 08/17] fix tests --- .../pkg/repository/repository_test.go | 96 ++++--------------- .../pkg/repository/transformer.go | 8 +- .../pkg/repository/transformer_test.go | 6 +- 3 files changed, 29 insertions(+), 81 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/repository_test.go b/services/manifest-repo-export-service/pkg/repository/repository_test.go index da7b5b4d7..f3c763e77 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository_test.go +++ b/services/manifest-repo-export-service/pkg/repository/repository_test.go @@ -248,6 +248,7 @@ func SetupRepositoryTestWithDB(t *testing.T) (Repository, *db.DBHandler, *Reposi CommitterName: "kuberpult", ArgoCdGenerateFiles: true, DBHandler: dbHandler, + Branch: "master", } repo, err := New( testutil.MakeTestContext(), @@ -265,71 +266,18 @@ func TestGetTagsNoTags(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - if false { - _, _, repoConfig := SetupRepositoryTestWithDB(t) - localDir := repoConfig.Path - //dir := t.TempDir() - //remoteDir := path.Join(dir, "remote") - //localDir := path.Join(dir, "local") - //repoConfig := RepositoryConfig{ - // URL: "file://" + remoteDir, - // Path: localDir, - // Branch: "master", - // ArgoCdGenerateFiles: true, - //} - //cmd := exec.Command("git", "init", "--bare", remoteDir) - //cmd.Start() - //cmd.Wait() - //_, err := New( - // testutil.MakeTestContext(), - // repoConfig, - //) - - //if err != nil { - // t.Fatal(err) - //} - tags, err := GetTags( - *repoConfig, - localDir, - testutil.MakeTestContext(), - ) - if err != nil { - t.Fatalf("new: expected no error, got '%e'", err) - } - if len(tags) != 0 { - t.Fatalf("expected %v tags but got %v", 0, len(tags)) - } - } else { - dir := t.TempDir() - remoteDir := path.Join(dir, "remote") - localDir := path.Join(dir, "local") - repoConfig := RepositoryConfig{ - URL: "file://" + remoteDir, - Path: localDir, - Branch: "master", - ArgoCdGenerateFiles: true, - } - cmd := exec.Command("git", "init", "--bare", remoteDir) - cmd.Start() - cmd.Wait() - _, err := New( - testutil.MakeTestContext(), - repoConfig, - ) - if err != nil { - t.Fatal(err) - } - tags, err := GetTags( - repoConfig, - localDir, - testutil.MakeTestContext(), - ) - if err != nil { - t.Fatalf("new: expected no error, got '%e'", err) - } - if len(tags) != 0 { - t.Fatalf("expected %v tags but got %v", 0, len(tags)) - } + _, _, repoConfig := SetupRepositoryTestWithDB(t) + localDir := repoConfig.Path + tags, err := GetTags( + *repoConfig, + localDir, + testutil.MakeTestContext(), + ) + if err != nil { + t.Fatalf("new: expected no error, got '%e'", err) + } + if len(tags) != 0 { + t.Fatalf("expected %v tags but got %v", 0, len(tags)) } }) @@ -356,21 +304,11 @@ func TestGetTags(t *testing.T) { tc := tc t.Run(tc.Name, func(t *testing.T) { t.Parallel() - dir := t.TempDir() - remoteDir := path.Join(dir, "remote") - localDir := path.Join(dir, "local") - repoConfig := RepositoryConfig{ - URL: "file://" + remoteDir, - Path: localDir, - Branch: "master", - ArgoCdGenerateFiles: true, - } - cmd := exec.Command("git", "init", "--bare", remoteDir) - cmd.Start() - cmd.Wait() + _, _, repoConfig := SetupRepositoryTestWithDB(t) + localDir := repoConfig.Path _, err := New( testutil.MakeTestContext(), - repoConfig, + *repoConfig, ) if err != nil { t.Fatal(err) @@ -409,7 +347,7 @@ func TestGetTags(t *testing.T) { } } tags, err := GetTags( - repoConfig, + *repoConfig, localDir, testutil.MakeTestContext(), ) diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 29ca9bd40..fcb6e574c 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -1445,8 +1445,11 @@ func (c *CleanupOldApplicationVersions) Transform( var err error var oldVersions []uint64 if tCtx.ShouldMinimizeGitData() { - oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) + //oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) + var versionLimit = int(state.ReleaseVersionsLimit) + oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) } else { + //oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) var versionLimit = int(state.ReleaseVersionsLimit) oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) } @@ -1559,6 +1562,9 @@ func (u *ReleaseTrain) Transform( var envGroupConfigs, isEnvGroup = getEnvironmentGroupsEnvironmentsOrEnvironment(configs, targetGroupName, u.TargetType) for _, currentDeployment := range deployments { envConfig := envGroupConfigs[currentDeployment.Env] + if envConfig.Upstream == nil || envConfig.Upstream.Environment == "" { + return "", fmt.Errorf("could not find upstream config for env '%s'", currentDeployment.Env) + } upstreamEnvName := envConfig.Upstream.Environment var trainGroup *string if isEnvGroup { diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 512336834..32f417c7a 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -758,7 +758,7 @@ func TestReleaseTrain(t *testing.T) { tc := tc t.Run(tc.Name, func(t *testing.T) { t.Parallel() - repo, _ := setupRepositoryTestWithPath(t) + repo, _, _ := SetupRepositoryTestWithDB(t) ctx := AddGeneratorToContext(testutil.MakeTestContext(), testutil.NewIncrementalUUIDGenerator()) dbHandler := repo.State().DBHandler @@ -838,6 +838,10 @@ func TestReleaseTrain(t *testing.T) { if err != nil { return err } + err = dbHandler.DBWriteAllEnvironments(ctx, transaction, []string{"staging", "production"}) + if err != nil { + return err + } var v int64 v = 1 err = dbHandler.DBWriteDeployment(ctx, transaction, db.Deployment{ From fc7191704d841fa29623d3e16fdcfcdf78268fba Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Wed, 6 Nov 2024 18:17:12 +0100 Subject: [PATCH 09/17] fixed TestCreateUndeployApplicationVersion --- .../pkg/repository/transformer_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 32f417c7a..921c6cf85 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -95,7 +95,7 @@ func setupRepositoryTestWithPath(t *testing.T) (Repository, string) { CommitterEmail: "kuberpult@freiheit.com", CommitterName: "kuberpult", ArgoCdGenerateFiles: true, - ReleaseVersionLimit: 0, + ReleaseVersionLimit: 2, } if dbConfig != nil { @@ -1714,6 +1714,20 @@ func TestCreateUndeployApplicationVersion(t *testing.T) { } } + envConfig := config.EnvironmentConfig{ + Upstream: nil, + ArgoCd: nil, + EnvironmentGroup: nil, + } + err = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, envConfig, []string{}) + if err != nil { + return err + } + err = dbHandler.DBWriteAllEnvironments(ctx, transaction, []string{envAcceptance}) + if err != nil { + return err + } + err = dbHandler.DBWriteNewReleaseEvent(ctx, transaction, 2, 1, "00000000-0000-0000-0000-000000000001", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", &event.NewRelease{}) if err != nil { return err From 1bf5e0d9bb57110fff4783d7440aa1015a3e693c Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 7 Nov 2024 10:34:32 +0100 Subject: [PATCH 10/17] continue --- .../pkg/repository/transformer.go | 2 +- .../pkg/repository/transformer_test.go | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index fcb6e574c..26b952263 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -1825,7 +1825,7 @@ func (u *UndeployApplication) Transform( } isUndeploy := lastRelease.Metadata.UndeployVersion if !isUndeploy { - return "", fmt.Errorf("UndeployApplication: last release is not un-deployed application version of '%v'", u.Application) + return "", fmt.Errorf("UndeployApplication: error last release is not un-deployed application version of '%v'", u.Application) } appDir := applicationDirectory(fs, u.Application) configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) // we use ALL envs, to be sure diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 921c6cf85..ba43f1613 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -2996,7 +2996,23 @@ func TestUndeployLogic(t *testing.T) { err2 := dbHandler.DBWriteMigrationsTransformer(ctx, transaction) if err2 != nil { t.Fatal(err2) - + } + err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, config.EnvironmentConfig{ + Upstream: &config.EnvironmentConfigUpstream{ + Environment: "staging", + }, + }, []string{appName}) + err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, config.EnvironmentConfig{ + Upstream: &config.EnvironmentConfigUpstream{ + Environment: "staging", + }, + }, []string{appName}) + if err2 != nil { + return err2 + } + err2 = dbHandler.DBWriteAllEnvironments(ctx, transaction, []string{envAcceptance, envAcceptance2}) + if err2 != nil { + return err2 } //populate the database for _, tr := range tc.Transformers { From 1c77b597216fc516e4d862198c4a1b446a1d8bd2 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 7 Nov 2024 13:52:45 +0100 Subject: [PATCH 11/17] fix-one-test --- .../pkg/repository/transformer_test.go | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index ba43f1613..cc5104cf1 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -2405,7 +2405,7 @@ func TestCreateUndeployLogic(t *testing.T) { }{ { - Name: "Create undeploy applicatiom version and deploy it", + Name: "Create undeploy application version and deploy it", Transformers: []Transformer{ &CreateEnvironment{ Environment: envAcceptance, @@ -2704,6 +2704,8 @@ func TestUndeployLogic(t *testing.T) { const appName = "app1" const authorName = "testAuthorName" const authorEmail = "testAuthorEmail@example.com" + environmentConfigAcceptance := config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}} + environmentConfigAcceptance2 := config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance2, Latest: true}} tcs := []struct { Name string Transformers []Transformer @@ -2717,7 +2719,7 @@ func TestUndeployLogic(t *testing.T) { Transformers: []Transformer{ &CreateEnvironment{ Environment: envAcceptance, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}}, + Config: environmentConfigAcceptance, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2726,7 +2728,7 @@ func TestUndeployLogic(t *testing.T) { }, &CreateEnvironment{ Environment: envAcceptance2, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance2, Latest: true}}, + Config: environmentConfigAcceptance2, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2828,7 +2830,7 @@ func TestUndeployLogic(t *testing.T) { Transformers: []Transformer{ &CreateEnvironment{ Environment: envAcceptance, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}}, + Config: environmentConfigAcceptance, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2837,7 +2839,7 @@ func TestUndeployLogic(t *testing.T) { }, &CreateEnvironment{ Environment: envAcceptance2, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance2, Latest: true}}, + Config: environmentConfigAcceptance2, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2891,7 +2893,7 @@ func TestUndeployLogic(t *testing.T) { Transformers: []Transformer{ &CreateEnvironment{ Environment: envAcceptance, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}}, + Config: environmentConfigAcceptance, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2900,7 +2902,7 @@ func TestUndeployLogic(t *testing.T) { }, &CreateEnvironment{ Environment: envAcceptance2, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance2, Latest: true}}, + Config: environmentConfigAcceptance2, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2960,25 +2962,24 @@ func TestUndeployLogic(t *testing.T) { }, expectedError: errMatcher{msg: "error within transaction: first apply failed, aborting: error at index 0 of transformer batch: UndeployApplication(repo):" + " error cannot un-deploy application '" + appName + "' the release on 'acceptance2' is not un-deployed: 'environments/acceptance2/applications/" + appName + "/version/undeploy'"}, - expectedData: []*FilenameAndData{ - { //There is an undeploy version - path: "/applications/app1/releases/2/undeploy", - fileData: []byte(""), - }, + expectedData: []*FilenameAndData{}, + expectedMissing: []*FilenameAndData{ { //The first env has the undeploy version deployed path: "environments/acceptance/applications/app1/version/undeploy", fileData: []byte(""), }, - { //The second env has the undeploy version *queued* - path: "environments/acceptance2/applications/app1/queued_version/undeploy", - fileData: []byte(""), - }, - }, - expectedMissing: []*FilenameAndData{ { //The second env does NOT have the undeploy version path: "environments/acceptance2/applications/app1/version/undeploy", fileData: []byte(""), }, + { //There is no undeploy version, because all releases have been deleted + path: "/applications/app1/releases/2/undeploy", + fileData: []byte(""), + }, + { //The second env has the undeploy version *queued* + path: "environments/acceptance2/applications/app1/queued_version", + fileData: []byte(""), + }, }, }, } @@ -2997,16 +2998,11 @@ func TestUndeployLogic(t *testing.T) { if err2 != nil { t.Fatal(err2) } - err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, config.EnvironmentConfig{ - Upstream: &config.EnvironmentConfigUpstream{ - Environment: "staging", - }, - }, []string{appName}) - err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, config.EnvironmentConfig{ - Upstream: &config.EnvironmentConfigUpstream{ - Environment: "staging", - }, - }, []string{appName}) + err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, environmentConfigAcceptance, []string{appName}) + if err2 != nil { + return err2 + } + err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, environmentConfigAcceptance2, []string{appName}) if err2 != nil { return err2 } @@ -3104,9 +3100,15 @@ func TestUndeployLogic(t *testing.T) { } } var commitMsg []string - err := repo.Apply(ctx, transaction, tc.Transformers...) - if err != nil { - return err + for _, t := range tc.Transformers { + err := repo.Apply(ctx, transaction, t) + if err != nil { + return err + } + err = repo.PushRepo(ctx) // just for testing! + if err != nil { + return err + } } actualMsg := "" // note that we only check the LAST error here: From 708653c2ebcd7600fdfd93b4d14c97718653630d Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 7 Nov 2024 13:58:21 +0100 Subject: [PATCH 12/17] fix test TestCreateUndeployLogic --- .../pkg/repository/transformer_test.go | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index cc5104cf1..5bcd1532b 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -2397,6 +2397,8 @@ func TestCreateUndeployLogic(t *testing.T) { const appName = "app1" const authorName = "testAuthorName" const authorEmail = "testAuthorEmail@example.com" + envAcceptanceConfig := config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}} + envAcceptance2Config := config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance2, Latest: true}} tcs := []struct { Name string Transformers []Transformer @@ -2409,7 +2411,7 @@ func TestCreateUndeployLogic(t *testing.T) { Transformers: []Transformer{ &CreateEnvironment{ Environment: envAcceptance, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}}, + Config: envAcceptanceConfig, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2463,7 +2465,7 @@ func TestCreateUndeployLogic(t *testing.T) { Transformers: []Transformer{ &CreateEnvironment{ Environment: envAcceptance, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance, Latest: true}}, + Config: envAcceptanceConfig, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2472,7 +2474,7 @@ func TestCreateUndeployLogic(t *testing.T) { }, &CreateEnvironment{ Environment: envAcceptance2, - Config: config.EnvironmentConfig{Upstream: &config.EnvironmentConfigUpstream{Environment: envAcceptance2, Latest: true}}, + Config: envAcceptance2Config, TransformerMetadata: TransformerMetadata{ AuthorName: authorName, AuthorEmail: authorEmail, @@ -2586,8 +2588,21 @@ func TestCreateUndeployLogic(t *testing.T) { err2 := dbHandler.DBWriteMigrationsTransformer(ctx, transaction) if err2 != nil { t.Fatal(err2) + } + err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance, envAcceptanceConfig, []string{appName}) + if err2 != nil { + return err2 } + err2 = dbHandler.DBWriteEnvironment(ctx, transaction, envAcceptance2, envAcceptance2Config, []string{appName}) + if err2 != nil { + return err2 + } + err2 = dbHandler.DBWriteAllEnvironments(ctx, transaction, []string{envAcceptance, envAcceptance2}) + if err2 != nil { + return err2 + } + //populate the database for _, tr := range tc.Transformers { err2 := dbHandler.DBWriteEslEventInternal(ctx, tr.GetDBEventType(), transaction, t, db.ESLMetadata{AuthorName: tr.GetMetadata().AuthorName, AuthorEmail: tr.GetMetadata().AuthorEmail}) From b395a00b23ddbf07138e86e2b363815c5ac7b5ac Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 7 Nov 2024 14:26:14 +0100 Subject: [PATCH 13/17] fix TestOverviewAndAppDetails --- services/cd-service/pkg/service/overview_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/services/cd-service/pkg/service/overview_test.go b/services/cd-service/pkg/service/overview_test.go index 29ebe0315..1f89d5b78 100644 --- a/services/cd-service/pkg/service/overview_test.go +++ b/services/cd-service/pkg/service/overview_test.go @@ -286,14 +286,6 @@ func TestOverviewAndAppDetails(t *testing.T) { DeployAuthor: "test tester", }, }, - prod: { - Version: 3, - QueuedVersion: 0, - UndeployVersion: false, - DeploymentMetaData: &api.Deployment_DeploymentMetaData{ - DeployAuthor: "test tester", - }, - }, }, AppLocks: map[string]*api.Locks{}, TeamLocks: map[string]*api.Locks{}, From 5f0a0b8b26af2c9bcef07dd16b1f6c32e4a9e409 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 7 Nov 2024 14:39:58 +0100 Subject: [PATCH 14/17] simplifications --- pkg/db/db.go | 35 +++++-------------- .../cd-service/pkg/repository/transformer.go | 4 +-- .../pkg/repository/transformer.go | 23 ++++-------- .../pkg/repository/transformer_test.go | 5 ++- 4 files changed, 21 insertions(+), 46 deletions(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index f2b88f852..1721aaabe 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -864,33 +864,14 @@ func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, t return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true) } -func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) ([]*DBReleaseWithMetaData, error) { +func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByAppOrderedByEslVersion") defer span.Finish() selectQuery := h.AdaptQuery(fmt.Sprintf( "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + " FROM releases " + " WHERE appName=? AND deleted=?" + - " ORDER BY eslVersion DESC, releaseVersion DESC, created DESC;")) - span.SetTag("query", selectQuery) - rows, err := tx.QueryContext( - ctx, - selectQuery, - app, - deleted, - ) - - return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) -} - -func (h *DBHandler) DBSelectLastReleasesByApp(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { - span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByApp") - defer span.Finish() - selectQuery := h.AdaptQuery(fmt.Sprintf( - "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + - " FROM releases " + - " WHERE appName=? AND deleted=?" + - " ORDER BY eslVersion DESC, releaseVersion DESC" + + " ORDER BY eslVersion DESC, releaseVersion DESC, created DESC" + " LIMIT 1;")) span.SetTag("query", selectQuery) rows, err := tx.QueryContext( @@ -900,14 +881,14 @@ func (h *DBHandler) DBSelectLastReleasesByApp(ctx context.Context, tx *sql.Tx, a deleted, ) - releases, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) + result, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) if err != nil { return nil, err } - if len(releases) == 0 { + if result == nil { return nil, nil } - return releases[0], nil + return result[0], nil } func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) { @@ -2460,9 +2441,6 @@ func (h *DBHandler) DBSelectExistingApp(ctx context.Context, tx *sql.Tx, appName if app == nil { return nil, nil } - if app.StateChange == AppStateChangeDelete { - return nil, nil - } return app, nil } @@ -2490,6 +2468,9 @@ func processAppRow(ctx context.Context, rows *sql.Rows) (*DBAppWithMetaData, err return nil, fmt.Errorf("Error during json unmarshal of apps. Error: %w. Data: %s\n", err, metadataStr) } row.Metadata = metaData + if row.StateChange == AppStateChangeDelete { + return nil, nil + } } else { row = nil } diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index 3428def5e..4507b5429 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -677,8 +677,8 @@ func (c *CreateApplicationVersion) Transform( return "", err } var v = db.InitialEslVersion - 1 - if len(prevRelease) > 0 { - v = prevRelease[0].EslVersion + if prevRelease != nil { + v = prevRelease.EslVersion } isMinor, err := c.checkMinorFlags(ctx, transaction, state.DBHandler, version, state.MinorRegexes) if err != nil { diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 26b952263..31d4ff2da 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -879,7 +879,7 @@ func (c *CreateApplicationVersion) Transform( } if !isLatest { // check that we can actually backfill this version - oldVersions, err := findOldApplicationVersions(ctx, transaction, state, int(state.ReleaseVersionsLimit), c.Application) + oldVersions, err := findOldApplicationVersions(ctx, transaction, state, c.Application) if err != nil { return "", GetCreateReleaseGeneralFailure(err) } @@ -1059,7 +1059,7 @@ func isLatestVersion(ctx context.Context, state *State, transaction *sql.Tx, app // Finds old releases for an application: Checks for the oldest release that is currently deployed on any environment // Releases older that the oldest deployed release are eligible for deletion. releaseVersionsLimit -func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state *State, versionLimit int, appName string) ([]uint64, error) { +func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state *State, appName string) ([]uint64, error) { // 1) get release in each env: envConfigs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) //envConfigs, err := state.GetEnvironmentConfigs() @@ -1093,7 +1093,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state return versions[i] >= oldestDeployedVersion }) - if positionOfOldestVersion < (versionLimit - 1) { + if positionOfOldestVersion < (int(state.ReleaseVersionsLimit) - 1) { return nil, nil } indexToKeep := positionOfOldestVersion - 1 @@ -1109,7 +1109,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state } else if !release.Metadata.IsMinor && !release.Metadata.IsPrepublish { majorsCount += 1 } - if majorsCount >= versionLimit { + if majorsCount >= int(state.ReleaseVersionsLimit) { break } } @@ -1438,28 +1438,19 @@ func (c *CleanupOldApplicationVersions) GetDBEventType() db.EventType { func (c *CleanupOldApplicationVersions) Transform( ctx context.Context, state *State, - tCtx TransformerContext, + _ TransformerContext, transaction *sql.Tx, ) (string, error) { fs := state.Filesystem var err error var oldVersions []uint64 - if tCtx.ShouldMinimizeGitData() { - //oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) - var versionLimit = int(state.ReleaseVersionsLimit) - oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) - } else { - //oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) - var versionLimit = int(state.ReleaseVersionsLimit) - oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) - } + oldVersions, err = findOldApplicationVersions(ctx, transaction, state, c.Application) if err != nil { return "", fmt.Errorf("cleanup: could not get application releases for app '%s': %w", c.Application, err) } msg := "" for _, oldRelease := range oldVersions { - // delete oldRelease: releasesDir := releasesDirectoryWithVersion(fs, c.Application, oldRelease) _, err := fs.Stat(releasesDir) @@ -1816,7 +1807,7 @@ func (u *UndeployApplication) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - lastRelease, err := state.DBHandler.DBSelectLastReleasesByApp(ctx, transaction, u.Application, false, true) + lastRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, u.Application, false, true) if err != nil { return "", err } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 5bcd1532b..5a7be1850 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -3120,7 +3120,10 @@ func TestUndeployLogic(t *testing.T) { if err != nil { return err } - err = repo.PushRepo(ctx) // just for testing! + // just for testing, we push each transformer change separately. + // if you need to debug this test, you can git clone the repo + // and we will only see anything if we push. + err = repo.PushRepo(ctx) if err != nil { return err } From 50376daffa3dee4cf82b4d47317a6f05a14fbb15 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 7 Nov 2024 19:23:00 +0100 Subject: [PATCH 15/17] WIP: fixed query, need to test it --- pkg/db/db.go | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index 1721aaabe..090b0bf9b 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -864,25 +864,37 @@ func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, t return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true) } -func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { +func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByAppOrderedByEslVersion") defer span.Finish() - selectQuery := h.AdaptQuery(fmt.Sprintf( - "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + - " FROM releases " + - " WHERE appName=? AND deleted=?" + - " ORDER BY eslVersion DESC, releaseVersion DESC, created DESC" + - " LIMIT 1;")) + selectQuery := h.AdaptQuery(fmt.Sprintf(` +SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments +FROM ( + SELECT max(eslVersion) as latestEslVersion, appName, releaseVersion + FROM releases + WHERE appName=? + GROUP BY appName, releaseVersion + ) as c +JOIN releases + ON c.appName= releases.appName + AND c.latestEslVersion = releases.eslversion + AND c.releaseVersion = releases.releaseVersion +ORDER BY c.releaseVersion DESC +LIMIT 1; +`), + ) span.SetTag("query", selectQuery) + span.SetTag("appName", app) rows, err := tx.QueryContext( ctx, selectQuery, app, - deleted, ) result, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) + span.SetTag("resultCount", len(result)) if err != nil { + span.Finish(tracer.WithError(err)) return nil, err } if result == nil { From 829c61e7fe9424adabab94df0bd4109bbb14a86f Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Fri, 8 Nov 2024 13:48:01 +0100 Subject: [PATCH 16/17] removed undeploy and created_at files --- docker-compose.yml | 2 +- pkg/db/db.go | 9 +- .../cd-service/pkg/repository/transformer.go | 2 +- .../pkg/repository/repository.go | 108 ------------------ .../pkg/repository/transformer.go | 22 ++-- 5 files changed, 22 insertions(+), 121 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ea82915f9..e70786c85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,7 +74,7 @@ services: - KUBERPULT_DB_SSL_MODE=disable - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=5 - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=1 - - KUBERPULT_MINIMIZE_EXPORTED_DATA=false + - KUBERPULT_MINIMIZE_EXPORTED_DATA=true volumes: - ./services/cd-service:/kp/kuberpult - ./database:/kp/database diff --git a/pkg/db/db.go b/pkg/db/db.go index 090b0bf9b..dc3372774 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -868,7 +868,14 @@ func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByAppOrderedByEslVersion") defer span.Finish() selectQuery := h.AdaptQuery(fmt.Sprintf(` -SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments +SELECT + releases.eslVersion, + releases.created, + releases.appName, + releases.metadata, + releases.releaseVersion, + releases.deleted, + releases.environments FROM ( SELECT max(eslVersion) as latestEslVersion, appName, releaseVersion FROM releases diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index 4507b5429..e9aad28a2 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -672,7 +672,7 @@ func (c *CreateApplicationVersion) Transform( sortedKeys := sorting.SortKeys(c.Manifests) if state.DBHandler.ShouldUseOtherTables() { - prevRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, c.Application, false, false) + prevRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, c.Application, false) if err != nil { return "", err } diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 288eaadfe..fd4e345a1 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -880,55 +880,6 @@ type Lock struct { CreatedAt time.Time } -func readLock(fs billy.Filesystem, lockDir string) (*Lock, error) { - lock := &Lock{ - Message: "", - CreatedBy: Actor{ - Name: "", - Email: "", - }, - CreatedAt: time.Time{}, - } - - if cnt, err := readFile(fs, fs.Join(lockDir, "message")); err != nil { - if !os.IsNotExist(err) { - return nil, err - } - } else { - lock.Message = string(cnt) - } - - if cnt, err := readFile(fs, fs.Join(lockDir, "created_by_email")); err != nil { - if !os.IsNotExist(err) { - return nil, err - } - } else { - lock.CreatedBy.Email = string(cnt) - } - - if cnt, err := readFile(fs, fs.Join(lockDir, "created_by_name")); err != nil { - if !os.IsNotExist(err) { - return nil, err - } - } else { - lock.CreatedBy.Name = string(cnt) - } - - if cnt, err := readFile(fs, fs.Join(lockDir, "created_at")); err != nil { - if !os.IsNotExist(err) { - return nil, err - } - } else { - if createdAt, err := time.Parse(time.RFC3339, strings.TrimSpace(string(cnt))); err != nil { - return nil, err - } else { - lock.CreatedAt = createdAt - } - } - - return lock, nil -} - func (s *State) GetEnvLocksDir(environment string) string { return s.Filesystem.Join("environments", environment, "locks") } @@ -997,65 +948,6 @@ func (s *State) GetLastRelease(ctx context.Context, fs billy.Filesystem, applica } } -func (s *State) GetEnvironmentLocks(environment string) (map[string]Lock, error) { - base := s.GetEnvLocksDir(environment) - if entries, err := s.Filesystem.ReadDir(base); err != nil { - return nil, err - } else { - result := make(map[string]Lock, len(entries)) - for _, e := range entries { - if !e.IsDir() { - return nil, fmt.Errorf("error getting environment locks: found file in the locks directory. run migration script to generate correct metadata") - } - if lock, err := readLock(s.Filesystem, s.Filesystem.Join(base, e.Name())); err != nil { - return nil, err - } else { - result[e.Name()] = *lock - } - } - return result, nil - } -} - -func (s *State) GetEnvironmentApplicationLocks(environment, application string) (map[string]Lock, error) { - base := s.GetAppLocksDir(environment, application) - if entries, err := s.Filesystem.ReadDir(base); err != nil { - return nil, err - } else { - result := make(map[string]Lock, len(entries)) - for _, e := range entries { - if !e.IsDir() { - return nil, fmt.Errorf("error getting application locks: found file in the locks directory. run migration script to generate correct metadata") - } - if lock, err := readLock(s.Filesystem, s.Filesystem.Join(base, e.Name())); err != nil { - return nil, err - } else { - result[e.Name()] = *lock - } - } - return result, nil - } -} -func (s *State) GetEnvironmentTeamLocks(environment, team string) (map[string]Lock, error) { - base := s.GetTeamLocksDir(environment, team) - if entries, err := s.Filesystem.ReadDir(base); err != nil { - return nil, err - } else { - result := make(map[string]Lock, len(entries)) - for _, e := range entries { - if !e.IsDir() { - return nil, fmt.Errorf("error getting team locks: found file in the locks directory. run migration script to generate correct metadata") - } - if lock, err := readLock(s.Filesystem, s.Filesystem.Join(base, e.Name())); err != nil { - return nil, err - } else { - result[e.Name()] = *lock - } - } - return result, nil - } -} - func (s *State) GetEnvironmentApplicationLocksFromDB(ctx context.Context, transaction *sql.Tx, environment, application string) (map[string]Lock, error) { if transaction == nil { return nil, fmt.Errorf("GetEnvironmentApplicationLocksFromDB: No transaction provided") diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 31d4ff2da..49fe02d2f 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -1696,7 +1696,7 @@ func (c *CreateUndeployApplicationVersion) GetDBEventType() db.EventType { func (c *CreateUndeployApplicationVersion) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { fs := state.Filesystem @@ -1719,12 +1719,14 @@ func (c *CreateUndeployApplicationVersion) Transform( if err != nil { return "", err } - // this is a flag to indicate that this is the special "undeploy" version - if err := util.WriteFile(fs, fs.Join(releaseDir, "undeploy"), []byte(""), 0666); err != nil { - return "", err - } - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil { - return "", err + if tCtx.ShouldMaximizeGitData() { + // this is a flag to indicate that this is the special "undeploy" version + if err := util.WriteFile(fs, fs.Join(releaseDir, "undeploy"), []byte(""), 0666); err != nil { + return "", err + } + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil { + return "", err + } } for env := range configs { envDir := fs.Join(releaseDir, "environments", env) @@ -1749,7 +1751,7 @@ func (c *CreateUndeployApplicationVersion) Transform( if err != nil { return "", err } - t.AddAppEnv(c.Application, env, teamOwner) + tCtx.AddAppEnv(c.Application, env, teamOwner) if hasUpstream && config.Upstream.Latest { d := &DeployApplicationVersion{ SourceTrain: nil, @@ -1767,7 +1769,7 @@ func (c *CreateUndeployApplicationVersion) Transform( AuthorEmail: "", }, } - err := t.Execute(d, transaction) + err := tCtx.Execute(d, transaction) if err != nil { _, ok := err.(*LockedError) if ok { @@ -1807,7 +1809,7 @@ func (u *UndeployApplication) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - lastRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, u.Application, false, true) + lastRelease, err := state.DBHandler.DBSelectReleasesByAppOrderedByEslVersion(ctx, transaction, u.Application, true) if err != nil { return "", err } From be9bd859773e8d9e10acbace8fdc3ff559da1d5d Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Fri, 8 Nov 2024 16:22:18 +0100 Subject: [PATCH 17/17] fix linter --- pkg/db/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index dc3372774..5e5af902c 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -867,7 +867,7 @@ func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, t func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context, tx *sql.Tx, app string, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByAppOrderedByEslVersion") defer span.Finish() - selectQuery := h.AdaptQuery(fmt.Sprintf(` + selectQuery := h.AdaptQuery(` SELECT releases.eslVersion, releases.created, @@ -888,7 +888,7 @@ JOIN releases AND c.releaseVersion = releases.releaseVersion ORDER BY c.releaseVersion DESC LIMIT 1; -`), +`, ) span.SetTag("query", selectQuery) span.SetTag("appName", app)