From 6febaf4b79d29f44c5f1e5e996a729f8636b5a26 Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Wed, 16 Oct 2024 11:32:13 +0200 Subject: [PATCH] feat(git): Allow exporting only strictly required data for Argo CD Ref: SRX-2CSAEK --- .../manifest-repo-export-service.yaml | 2 + charts/kuberpult/tests/charts_test.go | 41 ++ charts/kuberpult/values.yaml | 6 + docker-compose.yml | 31 +- pkg/db/db.go | 52 ++- pkg/db/overview.go | 59 ++- pkg/valid/validations.go | 8 + .../cd-service/pkg/repository/repository.go | 30 +- .../cd-service/pkg/repository/transformer.go | 7 +- services/cd-service/pkg/service/overview.go | 79 ++-- .../pkg/cmd/server.go | 17 +- .../pkg/repository/repository.go | 149 +++++-- .../pkg/repository/transformer.go | 367 +++++++++++------- .../pkg/repository/transformer_test.go | 2 +- .../pkg/service/git_test.go | 16 +- .../pkg/service/version_test.go | 13 +- 16 files changed, 648 insertions(+), 231 deletions(-) diff --git a/charts/kuberpult/templates/manifest-repo-export-service.yaml b/charts/kuberpult/templates/manifest-repo-export-service.yaml index 20af96bb5..df793a1a4 100644 --- a/charts/kuberpult/templates/manifest-repo-export-service.yaml +++ b/charts/kuberpult/templates/manifest-repo-export-service.yaml @@ -231,6 +231,8 @@ spec: value: "{{ .Values.datadogProfiling.enabled }}" - name: KUBERPULT_RELEASE_VERSIONS_LIMIT value: "{{ .Values.git.releaseVersionsLimit }}" + - name: KUBERPULT_MINIMIZE_EXPORTED_DATA + value: "{{ .Values.git.minimizeExportedData }}" volumeMounts: - name: repository # The repository volume, an emptyDir, is mounted to the kp directory. diff --git a/charts/kuberpult/tests/charts_test.go b/charts/kuberpult/tests/charts_test.go index 8c27edb6b..d5ab7c17d 100644 --- a/charts/kuberpult/tests/charts_test.go +++ b/charts/kuberpult/tests/charts_test.go @@ -1051,6 +1051,47 @@ manifestRepoExport: }, ExpectedMissing: []core.EnvVar{}, }, + { + Name: "Git MinimizeExportData mode explicitely set", + Values: ` +git: + url: "testURL" + minimizeExportedData: true +ingress: + domainName: "kuberpult-example.com" +db: + dbOption: "postgreSQL" + writeEslTableOnly: false + sslMode: disable +`, + ExpectedEnvs: []core.EnvVar{ + { + Name: "KUBERPULT_MINIMIZE_EXPORTED_DATA", + Value: "true", + }, + }, + ExpectedMissing: []core.EnvVar{}, + }, + { + Name: "Git MinimizeExportData mode default value", + Values: ` +git: + url: "testURL" +ingress: + domainName: "kuberpult-example.com" +db: + dbOption: "postgreSQL" + writeEslTableOnly: false + sslMode: disable +`, + ExpectedEnvs: []core.EnvVar{ + { + Name: "KUBERPULT_MINIMIZE_EXPORTED_DATA", + Value: "false", + }, + }, + ExpectedMissing: []core.EnvVar{}, + }, { Name: "DB ssl mode", Values: ` diff --git a/charts/kuberpult/values.yaml b/charts/kuberpult/values.yaml index e0c2638a6..845e9913d 100644 --- a/charts/kuberpult/values.yaml +++ b/charts/kuberpult/values.yaml @@ -67,6 +67,12 @@ git: # Values outside of the range will fail the service during startup. releaseVersionsLimit: 20 + # When minimizeExportedData==false, all data will be written to git, including meta information that is not relevant for argoCd, like locks. + # When true, only the files needed for Argo CD to work are written. + # Note that when this option is true, in case of data loss in the database, kuberpult can *not* recover all information from the git repo. + # Recommendation: Only activate this option, if frequent database backups are made and if the git repo takes up too many resources. + minimizeExportedData: false + hub: europe-west3-docker.pkg.dev/fdc-public-docker-registry/kuberpult kubernetesEngine: "GKE" diff --git a/docker-compose.yml b/docker-compose.yml index 1f8abb0f7..7b943a530 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,20 @@ services: + datadog-agent: + image: datadog/agent:latest + env_file: + - docker.env + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc/:/host/proc/:ro + - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro + ports: + - 127.0.0.1:8126:8126/tcp + environment: + - DD_APM_ENABLED=true + - DD_APM_NON_LOCAL_TRAFFIC=true + - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true + - DD_AC_EXCLUDE=name:datadog-agent + stop_grace_period: 2.0s postgres: image: postgres:13.15 restart: unless-stopped @@ -32,7 +48,7 @@ services: - KUBERPULT_DEX_RBAC_POLICY_PATH=/etc/policy.csv - KUBERPULT_DEX_RBAC_TEAM_PATH=/etc/team.csv - KUBERPULT_DEX_MOCK_ROLE=developer - - KUBERPULT_GIT_WRITE_COMMIT_DATA=true + - KUBERPULT_GIT_WRITE_COMMIT_DATA=false - KUBERPULT_MAXIMUM_QUEUE_SIZE=2 - KUBERPULT_ALLOW_LONG_APP_NAMES=true - KUBERPULT_ARGO_CD_GENERATE_FILES=true @@ -40,6 +56,12 @@ services: - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=10 - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=5 - KUBERPULT_ALLOWED_DOMAINS=localhost + - KUBERPULT_ENABLE_METRICS=true + - DD_AGENT_HOST=datadog-agent + - KUBERPULT_ENABLE_TRACING=true + - DD_SERVICE=kuberpult-cd-service + - DD_VERSION=0.0.0 + - DD_ENV=example-su-oct-30-b ports: - "8080:8080" - "8443:8443" @@ -48,6 +70,8 @@ services: - ./database:/kp/database stop_grace_period: 0.5s depends_on: + datadog-agent: + condition: service_healthy postgres: condition: service_healthy manifest-repo-export-service: @@ -74,6 +98,7 @@ services: - KUBERPULT_DB_SSL_MODE=disable - KUBERPULT_DB_MAX_OPEN_CONNECTIONS=5 - KUBERPULT_DB_MAX_IDLE_CONNECTIONS=1 + - KUBERPULT_MINIMIZE_EXPORTED_DATA=false volumes: - ./services/cd-service:/kp/kuberpult - ./database:/kp/database @@ -119,8 +144,8 @@ services: container_name: kuberpult-ui ports: - "3000:3000" - depends_on: - - frontend-service +# depends_on: +# - frontend-service volumes: - ./services/frontend-service/src:/kp/src - ./services/frontend-service/public:/kp/public diff --git a/pkg/db/db.go b/pkg/db/db.go index 362e75274..f8c9f8790 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -883,6 +883,33 @@ func (h *DBHandler) DBSelectReleasesByAppOrderedByEslVersion(ctx context.Context return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) } +func (h *DBHandler) DBSelectLastReleasesByApp(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { + span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByApp") + defer span.Finish() + selectQuery := h.AdaptQuery(fmt.Sprintf( + "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + + " FROM releases " + + " WHERE appName=? AND deleted=?" + + " ORDER BY eslVersion DESC, releaseVersion DESC" + + " LIMIT 1;")) + span.SetTag("query", selectQuery) + rows, err := tx.QueryContext( + ctx, + selectQuery, + app, + deleted, + ) + + releases, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) + if err != nil { + return nil, err + } + if len(releases) == 0 { + return nil, nil + } + return releases[0], nil +} + func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectAllReleasesOfApp") defer span.Finish() @@ -2286,6 +2313,9 @@ type DBAppWithMetaData struct { func (h *DBHandler) DBInsertApplication(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion EslVersion, stateChange AppStateChange, metaData DBAppMetaData) error { span, ctx := tracer.StartSpanFromContext(ctx, "DBInsertApplication") defer span.Finish() + log := logger.FromContext(ctx).Sugar() + log.Warnf("plain dbinsert app: %s/%v", appName, stateChange) + jsonToInsert, err := json.Marshal(metaData) if err != nil { return fmt.Errorf("could not marshal json data: %w", err) @@ -2418,6 +2448,25 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro if err != nil { return nil, fmt.Errorf("could not query apps table from DB. Error: %w\n", err) } + return processAppRow(ctx, rows) +} + +func (h *DBHandler) DBSelectExistingApp(ctx context.Context, tx *sql.Tx, appName string) (*DBAppWithMetaData, error) { + app, err := h.DBSelectApp(ctx, tx, appName) + if err != nil { + return nil, err + } + if app == nil { + return nil, nil + } + if app.StateChange == AppStateChangeDelete { + return nil, nil + } + return app, nil +} + +func processAppRow(ctx context.Context, rows *sql.Rows) (*DBAppWithMetaData, error) { + //exhaustruct:ignore defer func(rows *sql.Rows) { err := rows.Close() if err != nil { @@ -2425,7 +2474,6 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro } }(rows) - //exhaustruct:ignore var row = &DBAppWithMetaData{} if rows.Next() { var metadataStr string @@ -2445,7 +2493,7 @@ func (h *DBHandler) processAppsRow(ctx context.Context, rows *sql.Rows, err erro } else { row = nil } - err = closeRows(rows) + err := closeRows(rows) if err != nil { return nil, err } diff --git a/pkg/db/overview.go b/pkg/db/overview.go index 3d4224e92..8b50e07f9 100644 --- a/pkg/db/overview.go +++ b/pkg/db/overview.go @@ -114,6 +114,63 @@ func (h *DBHandler) UpdateOverviewEnvironmentLock(ctx context.Context, transacti return nil } +func (h *DBHandler) UpdateOverviewDeployment(ctx context.Context, transaction *sql.Tx, deployment Deployment, createdTime time.Time) error { + latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return err + } + if h.IsOverviewEmpty(latestOverview) { + return nil + } + env := getEnvironmentByName(latestOverview.EnvironmentGroups, deployment.Env) + if env == nil { + return fmt.Errorf("could not find environment %s in overview", deployment.Env) + } + + selectApp, err := h.DBSelectExistingApp(ctx, transaction, deployment.App) + if err != nil { + return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", deployment.App, err) + } + if selectApp == nil { + return fmt.Errorf("could not find application '%s' in apps table: got no result", deployment.App) + } + + err = h.WriteOverviewCache(ctx, transaction, latestOverview) + if err != nil { + return err + } + return nil +} + +func (h *DBHandler) UpdateOverviewDeploymentAttempt(ctx context.Context, transaction *sql.Tx, queuedDeployment *QueuedDeployment) error { + latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) + if err != nil { + return err + } + if h.IsOverviewEmpty(latestOverview) { + return nil + } + if queuedDeployment == nil { + return nil + } + env := getEnvironmentByName(latestOverview.EnvironmentGroups, queuedDeployment.Env) + if env == nil { + return fmt.Errorf("could not find environment %s in overview", queuedDeployment.Env) + } + selectApp, err := h.DBSelectExistingApp(ctx, transaction, queuedDeployment.App) + if err != nil { + return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", queuedDeployment.App, err) + } + if selectApp == nil { + return fmt.Errorf("could not find application '%s' in apps table: got no result", queuedDeployment.App) + } + err = h.WriteOverviewCache(ctx, transaction, latestOverview) + if err != nil { + return err + } + return nil +} + func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transaction *sql.Tx, applicationLock ApplicationLock, createdTime time.Time) error { latestOverview, err := h.ReadLatestOverviewCache(ctx, transaction) if err != nil { @@ -126,7 +183,7 @@ func (h *DBHandler) UpdateOverviewApplicationLock(ctx context.Context, transacti if env == nil { return fmt.Errorf("could not find environment %s in overview", applicationLock.Env) } - selectApp, err := h.DBSelectApp(ctx, transaction, applicationLock.App) + selectApp, err := h.DBSelectExistingApp(ctx, transaction, applicationLock.App) if err != nil { return fmt.Errorf("could not find application '%s' in apps table, got an error: %w", applicationLock.App, err) } diff --git a/pkg/valid/validations.go b/pkg/valid/validations.go index 1dcfec142..9ac76a12b 100644 --- a/pkg/valid/validations.go +++ b/pkg/valid/validations.go @@ -105,3 +105,11 @@ func ReadEnvVarUInt(envName string) (uint, error) { } return uint(i), nil } + +func ReadEnvVarBool(envName string) (bool, error) { + envValue, err := ReadEnvVar(envName) + if err != nil { + return false, err + } + return envValue == "true", nil +} diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index b62ee9588..fc399446b 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -2462,6 +2462,9 @@ func (s *State) WriteAllCommitEvents(ctx context.Context, transaction *sql.Tx, d } func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction *sql.Tx, appName string, previousEslVersion db.EslVersion, stateChange db.AppStateChange, metaData db.DBAppMetaData) error { + log := logger.FromContext(ctx).Sugar() + log.Warnf("dbinsert app with overview: %s/%v", appName, stateChange) + h := s.DBHandler err := h.DBInsertApplication(ctx, transaction, appName, previousEslVersion, stateChange, metaData) if err != nil { @@ -2478,12 +2481,11 @@ func (s *State) DBInsertApplicationWithOverview(ctx context.Context, transaction } shouldDelete := stateChange == db.AppStateChangeDelete - if shouldDelete { lApps := make([]*api.OverviewApplication, 0) for _, curr := range cache.LightweightApps { - if curr.Name != appName { + if curr != nil && curr.Name != appName { lApps = append(lApps, curr) } } @@ -2533,6 +2535,30 @@ func (s *State) DBInsertEnvironmentWithOverview(ctx context.Context, tx *sql.Tx, return nil } +func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sql.Tx, appName string, result *api.GetOverviewResponse, deleteApp bool, allReleasesOfAllApps map[string][]int64) error { + if deleteApp { + removeOverviewAppFromLightweightApps(result, appName) + return nil + } + team, err := s.GetApplicationTeamOwner(ctx, transaction, appName) + if err != nil { + return fmt.Errorf("could not obtain application team owner to update top level app in overview: %w", err) + } + result.LightweightApps = append(result.LightweightApps, &api.OverviewApplication{Name: appName, Team: team}) + return nil +} + +func removeOverviewAppFromLightweightApps(result *api.GetOverviewResponse, appName string) { + lApps := make([]*api.OverviewApplication, max(len(result.LightweightApps)-1, 0)) + + for _, curr := range result.LightweightApps { + if curr != nil && curr.Name != appName { + lApps = append(lApps, curr) + } + } + result.LightweightApps = lApps +} + func getEnvironmentInGroup(groups []*api.EnvironmentGroup, groupNameToReturn string, envNameToReturn string) *api.Environment { for _, currentGroup := range groups { if currentGroup.EnvironmentGroupName == groupNameToReturn { diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index cd91e7c61..0e79671d9 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -1493,8 +1493,8 @@ func (u *UndeployApplication) GetDBEventType() db.EventType { return db.EvtUndeployApplication } -func (c *UndeployApplication) SetEslVersion(id db.TransformerID) { - c.TransformerEslVersion = id +func (u *UndeployApplication) SetEslVersion(id db.TransformerID) { + u.TransformerEslVersion = id } func (u *UndeployApplication) Transform( @@ -1616,10 +1616,11 @@ func (u *UndeployApplication) Transform( if err != nil { return "", fmt.Errorf("UndeployApplication: could not write all apps '%v': '%w'", u.Application, err) } - dbApp, err := state.DBHandler.DBSelectApp(ctx, transaction, u.Application) + dbApp, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, u.Application) if err != nil { return "", fmt.Errorf("UndeployApplication: could not select app '%s': %v", u.Application, err) } + logger.FromContext(ctx).Sugar().Warnf("before insert app fun") err = state.DBHandler.InsertAppFun(ctx, transaction, dbApp.App, dbApp.EslVersion, db.AppStateChangeDelete, db.DBAppMetaData{Team: dbApp.Metadata.Team}) if err != nil { return "", fmt.Errorf("UndeployApplication: could not insert app '%s': %v", u.Application, err) diff --git a/services/cd-service/pkg/service/overview.go b/services/cd-service/pkg/service/overview.go index b06de293c..ccc590098 100644 --- a/services/cd-service/pkg/service/overview.go +++ b/services/cd-service/pkg/service/overview.go @@ -126,7 +126,7 @@ func (o *OverviewServiceServer) GetAppDetails( sort.Slice(result.Releases, func(i, j int) bool { return result.Releases[j].Version < result.Releases[i].Version }) - if app, err := o.DBHandler.DBSelectApp(ctx, transaction, appName); err != nil { + if app, err := o.DBHandler.DBSelectExistingApp(ctx, transaction, appName); err != nil { return nil, err } else { if app == nil { @@ -190,31 +190,47 @@ func (o *OverviewServiceServer) GetAppDetails( return nil, fmt.Errorf("could not obtain deployments for app %s: %w", appName, err) } for envName, currentDeployment := range deployments { - deployment := &api.Deployment{ - Version: uint64(*currentDeployment.Version), - QueuedVersion: 0, - UndeployVersion: false, - DeploymentMetaData: &api.Deployment_DeploymentMetaData{ - CiLink: currentDeployment.Metadata.CiLink, - DeployAuthor: currentDeployment.Metadata.DeployedByName, - DeployTime: currentDeployment.Created.String(), - }, + environment, err := o.DBHandler.DBSelectEnvironment(ctx, transaction, envName) + if err != nil { + return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err) } - if queuedVersion, err := o.Repository.State().GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else { - if queuedVersion == nil { - deployment.QueuedVersion = 0 - } else { - deployment.QueuedVersion = *queuedVersion + if environment == nil { + return nil, fmt.Errorf("could not obtain environment %s for app %s: %w", envName, appName, err) + } + foundApp := false + for _, appInEnv := range environment.Applications { + if appInEnv == appName { + foundApp = true + break } } + if foundApp { + deployment := &api.Deployment{ + Version: uint64(*currentDeployment.Version), + QueuedVersion: 0, + UndeployVersion: false, + DeploymentMetaData: &api.Deployment_DeploymentMetaData{ + CiLink: currentDeployment.Metadata.CiLink, + DeployAuthor: currentDeployment.Metadata.DeployedByName, + DeployTime: currentDeployment.Created.String(), + }, + } + if queuedVersion, err := o.Repository.State().GetQueuedVersion(ctx, transaction, envName, appName); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else { + if queuedVersion == nil { + deployment.QueuedVersion = 0 + } else { + deployment.QueuedVersion = *queuedVersion + } + } - rel := getReleaseFromVersion(releases, uint64(*currentDeployment.Version)) - if rel != nil { - deployment.UndeployVersion = rel.Metadata.UndeployVersion + rel := getReleaseFromVersion(releases, uint64(*currentDeployment.Version)) + if rel != nil { + deployment.UndeployVersion = rel.Metadata.UndeployVersion + } + response.Deployments[envName] = deployment } - response.Deployments[envName] = deployment } result.UndeploySummary = deriveUndeploySummary(appName, response.Deployments) result.Warnings = CalculateWarnings(deployments, appLocks, envGroups) @@ -224,6 +240,11 @@ func (o *OverviewServiceServer) GetAppDetails( return nil, err } response.Application = resultApp + + //span2, ctx := tracer.StartSpanFromContext(ctx, "Delay") + //time.Sleep(1000 * time.Millisecond) + //defer span2.Finish() + return response, nil } @@ -352,7 +373,17 @@ func (o *OverviewServiceServer) StreamOverview(in *api.GetOverviewRequest, case <-o.Shutdown: return nil case <-ch: - ov := o.response.Load().(*api.GetOverviewResponse) + loaded := o.response.Load() + var ov *api.GetOverviewResponse = nil + if loaded == nil { + ov, err := o.getOverviewDB(stream.Context(), o.Repository.State()) + if err != nil { + return fmt.Errorf("could not load overview") + } + o.response.Store(ov) + } else { + ov = loaded.(*api.GetOverviewResponse) + } if err := stream.Send(ov); err != nil { // if we don't log this here, the details will be lost - so this is an exception to the rule "either return an error or log it". @@ -467,6 +498,10 @@ func (o *OverviewServiceServer) update(s *repository.State) { logger.FromContext(o.Context).Error("error getting overview:", zap.Error(err)) return } + if r == nil { + logger.FromContext(o.Context).Error("overview is nil") + return + } o.response.Store(r) o.notify.Notify() } diff --git a/services/manifest-repo-export-service/pkg/cmd/server.go b/services/manifest-repo-export-service/pkg/cmd/server.go index 97242c622..fa1b5010a 100755 --- a/services/manifest-repo-export-service/pkg/cmd/server.go +++ b/services/manifest-repo-export-service/pkg/cmd/server.go @@ -154,6 +154,11 @@ func Run(ctx context.Context) error { return fmt.Errorf("error parsing KUBERPULT_RELEASE_VERSIONS_LIMIT, error: %w", err) } + minimizeExportedData, err := valid.ReadEnvVarBool("KUBERPULT_MINIMIZE_EXPORTED_DATA") + if err != nil { + return err + } + var eslProcessingIdleTimeSeconds uint64 if val, exists := os.LookupEnv("KUBERPULT_ESL_PROCESSING_BACKOFF"); !exists { log.Infof("environment variable KUBERPULT_ESL_PROCESSING_BACKOFF is not set, using default backoff of 10 seconds") @@ -237,11 +242,13 @@ func Run(ctx context.Context) error { Certificates: repository.Certificates{ KnownHostsFile: gitSshKnownHosts, }, - Branch: gitBranch, - NetworkTimeout: time.Duration(networkTimeoutSeconds) * time.Second, - ReleaseVersionLimit: uint(releaseVersionLimit), - ArgoCdGenerateFiles: argoCdGenerateFiles, - DBHandler: dbHandler, + Branch: gitBranch, + NetworkTimeout: time.Duration(networkTimeoutSeconds) * time.Second, + ReleaseVersionLimit: uint(releaseVersionLimit), + ArgoCdGenerateFiles: argoCdGenerateFiles, + MinimizeExportedData: minimizeExportedData, + + DBHandler: dbHandler, } repo, err := repository.New(ctx, cfg) diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 783336058..86b83f5f0 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -131,9 +131,12 @@ type RepositoryConfig struct { // network timeout NetworkTimeout time.Duration + DBHandler *db.DBHandler + ArgoCdGenerateFiles bool ReleaseVersionLimit uint - DBHandler *db.DBHandler + + MinimizeExportedData bool } func openOrCreate(path string) (*git.Repository, error) { @@ -275,7 +278,10 @@ func New(ctx context.Context, cfg RepositoryConfig) (Repository, error) { } // Check configuration for errors and abort early if any: - _, err = state.GetEnvironmentConfigsAndValidate(ctx) + err = state.DBHandler.WithTransaction(ctx, true, func(ctx context.Context, transaction *sql.Tx) error { + _, err = state.GetEnvironmentConfigsAndValidate(ctx, transaction) + return err + }) if err != nil { return nil, err } @@ -443,7 +449,7 @@ func (r *repository) ApplyTransformersInternal(ctx context.Context, transaction } return nil, nil, nil, &applyErr } - if msg, subChanges, err := RunTransformer(ctxWithTime, transformer, state, transaction); err != nil { + if msg, subChanges, err := RunTransformer(ctxWithTime, transformer, state, transaction, r.config.MinimizeExportedData); err != nil { applyErr := TransformerBatchApplyError{ TransformerError: err, Index: 0, @@ -697,7 +703,7 @@ func (r *repository) afterTransform(ctx context.Context, transaction *sql.Tx, st span, ctx := tracer.StartSpanFromContext(ctx, "afterTransform") defer span.Finish() - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return err } @@ -727,11 +733,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, transaction *sql.Tx, appData := []argocd.AppData{} sort.Strings(apps) for _, appName := range apps { - if err != nil { - return err - } - //team, err := state.DBHandler.DBSelectApp().GetApplicationTeamOwner(ctx, transaction, appName) - oneAppData, err := state.DBHandler.DBSelectApp(ctx, transaction, appName) + oneAppData, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, appName) if err != nil { return fmt.Errorf("updateArgoCdApps: could not select app '%s' in db %v", appName, err) } @@ -1051,6 +1053,76 @@ func (s *State) GetEnvironmentTeamLocks(environment, team string) (map[string]Lo return result, nil } } + +func (s *State) GetEnvironmentApplicationLocksFromDB(ctx context.Context, transaction *sql.Tx, environment, application string) (map[string]Lock, error) { + if transaction == nil { + return nil, fmt.Errorf("GetEnvironmentApplicationLocksFromDB: No transaction provided") + } + activeLockIds, err := s.DBHandler.DBSelectAllAppLocks(ctx, transaction, environment, application) + if err != nil { + return nil, err + } + var lockIds []string + if activeLockIds != nil { + lockIds = activeLockIds.AppLocks + } + locks, err := s.DBHandler.DBSelectAppLockSet(ctx, transaction, environment, application, lockIds) + + if err != nil { + return nil, err + } + result := make(map[string]Lock, len(locks)) + for _, lock := range locks { + genericLock := Lock{ + Message: lock.Metadata.Message, + CreatedBy: Actor{ + Name: lock.Metadata.CreatedByName, + Email: lock.Metadata.CreatedByEmail, + }, + CreatedAt: lock.Created, + } + result[lock.LockID] = genericLock + } + return result, nil +} + +func (s *State) GetEnvironmentTeamLocksFromDB(ctx context.Context, transaction *sql.Tx, environment, team string) (map[string]Lock, error) { + if team == "" { + return map[string]Lock{}, nil + } + + if transaction == nil { + return nil, fmt.Errorf("GetEnvironmentTeamLocksFromDB: No transaction provided") + } + activeLockIDs, err := s.DBHandler.DBSelectAllTeamLocks(ctx, transaction, environment, team) + if err != nil { + return nil, err + } + + var lockIds []string + if activeLockIDs != nil { + lockIds = activeLockIDs.TeamLocks + } + locks, err := s.DBHandler.DBSelectTeamLockSet(ctx, transaction, environment, team, lockIds) + + if err != nil { + return nil, err + } + result := make(map[string]Lock, len(locks)) + for _, lock := range locks { + genericLock := Lock{ + Message: lock.Metadata.Message, + CreatedBy: Actor{ + Name: lock.Metadata.CreatedByName, + Email: lock.Metadata.CreatedByEmail, + }, + CreatedAt: lock.Created, + } + result[lock.LockID] = genericLock + } + return result, nil +} + func (s *State) GetDeploymentMetaData(environment, application string) (string, time.Time, error) { base := s.Filesystem.Join("environments", environment, "applications", application) author, err := readFile(s.Filesystem, s.Filesystem.Join(base, "deployed_by")) @@ -1203,9 +1275,28 @@ func envExists(envConfigs map[string]config.EnvironmentConfig, envNameToSearchFo return false } -func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context) (map[string]config.EnvironmentConfig, error) { +func (s *State) GetAllEnvironmentConfigsFromDB(ctx context.Context, transaction *sql.Tx) (map[string]config.EnvironmentConfig, error) { + dbAllEnvs, err := s.DBHandler.DBSelectAllEnvironments(ctx, transaction) + if err != nil { + return nil, fmt.Errorf("unable to retrieve all environments, error: %w", err) + } + if dbAllEnvs == nil { + return nil, nil + } + envs, err := s.DBHandler.DBSelectEnvironmentsBatch(ctx, transaction, dbAllEnvs.Environments) + if err != nil { + return nil, fmt.Errorf("unable to retrieve manifests for environments %v from the database, error: %w", dbAllEnvs.Environments, err) + } + ret := make(map[string]config.EnvironmentConfig) + for _, env := range *envs { + ret[env.Name] = env.Config + } + return ret, nil +} + +func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context, transaction *sql.Tx) (map[string]config.EnvironmentConfig, error) { logger := logger.FromContext(ctx) - envConfigs, err := s.GetEnvironmentConfigs() + envConfigs, err := s.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return nil, err } @@ -1233,22 +1324,22 @@ func (s *State) GetEnvironmentConfigsAndValidate(ctx context.Context) (map[strin return envConfigs, err } -func (s *State) GetEnvironmentConfigs() (map[string]config.EnvironmentConfig, error) { - envs, err := s.Filesystem.ReadDir("environments") - if err != nil { - return nil, err - } - result := map[string]config.EnvironmentConfig{} - for _, env := range envs { - c, err := s.GetEnvironmentConfig(env.Name()) - if err != nil { - return nil, err - - } - result[env.Name()] = *c - } - return result, nil -} +//func (s *State) GetEnvironmentConfigs() (map[string]config.EnvironmentConfig, error) { +//envs, err := s.Filesystem.ReadDir("environments") +//if err != nil { +// return nil, err +//} +//result := map[string]config.EnvironmentConfig{} +//for _, env := range envs { +// c, err := s.GetEnvironmentConfig(env.Name()) +// if err != nil { +// return nil, err +// +// } +// result[env.Name()] = *c +//} +//return result, nil +//} func (s *State) GetEnvironmentConfig(environmentName string) (*config.EnvironmentConfig, error) { fileName := s.Filesystem.Join("environments", environmentName, "config.json") @@ -1261,8 +1352,8 @@ func (s *State) GetEnvironmentConfig(environmentName string) (*config.Environmen return &config, nil } -func (s *State) GetEnvironmentConfigsForGroup(envGroup string) ([]string, error) { - allEnvConfigs, err := s.GetEnvironmentConfigs() +func (s *State) GetEnvironmentConfigsForGroup(ctx context.Context, transaction *sql.Tx, envGroup string) ([]string, error) { + allEnvConfigs, err := s.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return nil, err } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 9583893f4..cc8e63a1c 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -81,22 +81,27 @@ func versionToString(Version uint64) string { return strconv.FormatUint(Version, 10) } +// releasesDirectory returns applications//releases/ func releasesDirectory(fs billy.Filesystem, application string) string { return fs.Join("applications", application, "releases") } +// applicationDirectory returns applications// func applicationDirectory(fs billy.Filesystem, application string) string { return fs.Join("applications", application) } +// environmentApplicationDirectory returns environments//applications// func environmentApplicationDirectory(fs billy.Filesystem, environment, application string) string { return fs.Join("environments", environment, "applications", application) } +// releasesDirectoryWithVersion returns applications//releases/ func releasesDirectoryWithVersion(fs billy.Filesystem, application string, version uint64) string { return fs.Join(releasesDirectory(fs, application), versionToString(version)) } +// environmentApplicationDirectory returns applications//releases//environments/ func manifestDirectoryWithReleasesVersion(fs billy.Filesystem, application string, version uint64) string { return fs.Join(releasesDirectoryWithVersion(fs, application, version), "environments") } @@ -122,6 +127,8 @@ type TransformerContext interface { Execute(t Transformer, transaction *sql.Tx) error AddAppEnv(app string, env string, team string) DeleteEnvFromApp(app string, env string) + ShouldMinimizeGitData() bool + ShouldMaximizeGitData() bool } type TransformerMetadata struct { @@ -133,7 +140,12 @@ func (t *TransformerMetadata) GetMetadata() *TransformerMetadata { return t } -func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *sql.Tx) (string, *TransformerResult, error) { +func GetNoOpMessage(t Transformer) (string, error) { + evt := t.GetDBEventType() + return fmt.Sprintf("Empty Commit\nNo files changed in %s", evt), nil +} + +func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *sql.Tx, minimizeExportedData bool) (string, *TransformerResult, error) { runner := transformerRunner{ ChangedApps: nil, DeletedRootApps: nil, @@ -141,6 +153,8 @@ func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *s Context: ctx, State: s, Stack: [][]string{nil}, + + MinimizeGitData: minimizeExportedData, } if err := runner.Execute(t, transaction); err != nil { return "", nil, err @@ -193,6 +207,8 @@ type transformerRunner struct { ChangedApps []AppEnv DeletedRootApps []RootApp Commits *CommitIds + + MinimizeGitData bool } func (r *transformerRunner) Execute(t Transformer, transaction *sql.Tx) error { @@ -235,6 +251,14 @@ func (r *transformerRunner) DeleteEnvFromApp(app string, env string) { }) } +func (r *transformerRunner) ShouldMinimizeGitData() bool { + return r.MinimizeGitData +} + +func (r *transformerRunner) ShouldMaximizeGitData() bool { + return !r.MinimizeGitData +} + type RawNode struct{ *yaml3.Node } func (n *RawNode) UnmarshalYAML(node *yaml3.Node) error { @@ -322,7 +346,7 @@ type DeployApplicationVersionSource struct { func (c *DeployApplicationVersion) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { @@ -340,30 +364,31 @@ func (c *DeployApplicationVersion) Transform( if c.LockBehaviour != api.LockBehavior_IGNORE { // Check that the environment is not locked var ( - envLocks, appLocks, teamLocks map[string]Lock + appLocks, envLocks, teamLocks map[string]Lock err error ) - envLocks, err = state.GetEnvironmentLocks(c.Environment) + envLocks, err = state.GetEnvironmentLocksFromDB(ctx, transaction, c.Environment) if err != nil { return "", err } - appLocks, err = state.GetEnvironmentApplicationLocks(c.Environment, c.Application) + appLocks, err = state.GetEnvironmentApplicationLocksFromDB(ctx, transaction, c.Environment, c.Application) if err != nil { return "", err } - teamName, err := state.GetTeamName(c.Application) + app, err := state.DBHandler.DBSelectExistingApp(ctx, transaction, c.Application) if err != nil { return "", err } + var teamName = "" + if app != nil && app.Metadata.Team != "" { + teamName = app.Metadata.Team + } - if errors.Is(err, os.ErrNotExist) { - teamLocks = map[string]Lock{} //If we dont find the team file, there is no team for application, meaning there can't be any team locks - } else { - teamLocks, err = state.GetEnvironmentTeamLocks(c.Environment, string(teamName)) - if err != nil { - return "", err - } + teamLocks, err = state.GetEnvironmentTeamLocksFromDB(ctx, transaction, c.Environment, teamName) + //teamLocks, err = state.GetEnvironmentTeamLocks(c.Environment, teamName) + if err != nil { + return "", err } if len(envLocks) > 0 || len(appLocks) > 0 || len(teamLocks) > 0 { @@ -375,7 +400,7 @@ func (c *DeployApplicationVersion) Transform( Version: c.Version, TransformerEslVersion: c.TransformerEslVersion, } - return q.Transform(ctx, state, t, nil) + return q.Transform(ctx, state, tCtx, transaction) case api.LockBehavior_FAIL: return "", &LockedError{ EnvironmentApplicationLocks: appLocks, @@ -386,18 +411,21 @@ func (c *DeployApplicationVersion) Transform( } } applicationDir := fsys.Join("environments", c.Environment, "applications", c.Application) - versionFile := fsys.Join(applicationDir, "version") - // Create a symlink to the release if err := fsys.MkdirAll(applicationDir, 0777); err != nil { return "", err } + versionFile := fsys.Join(applicationDir, "version") if err := fsys.Remove(versionFile); err != nil && !errors.Is(err, os.ErrNotExist) { return "", err } - if err := fsys.Symlink(fsys.Join("..", "..", "..", "..", releaseDir), versionFile); err != nil { - return "", err + + if tCtx.ShouldMaximizeGitData() { + if err := fsys.Symlink(fsys.Join("..", "..", "..", "..", releaseDir), versionFile); err != nil { + return "", err + } } + // Copy the manifest for argocd manifestsDir := fsys.Join(applicationDir, "manifests") if err := fsys.MkdirAll(manifestsDir, 0777); err != nil { @@ -414,30 +442,33 @@ func (c *DeployApplicationVersion) Transform( if err := util.WriteFile(fsys, manifestFilename, manifestContent, 0666); err != nil { return "", err } + teamOwner, err := state.GetApplicationTeamOwner(ctx, transaction, c.Application) if err != nil { return "", err } - t.AddAppEnv(c.Application, c.Environment, teamOwner) + tCtx.AddAppEnv(c.Application, c.Environment, teamOwner) existingDeployment, err := state.DBHandler.DBSelectLatestDeployment(ctx, transaction, c.Application, c.Environment) if err != nil { return "", fmt.Errorf("error while retrieving deployment: %v", err) } - logger.FromContext(ctx).Sugar().Warnf("writing deployed name...") - if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by"), []byte(existingDeployment.Metadata.DeployedByName), 0666); err != nil { - return "", err - } + if tCtx.ShouldMaximizeGitData() { + logger.FromContext(ctx).Sugar().Warnf("writing deployed name...") + if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by"), []byte(existingDeployment.Metadata.DeployedByName), 0666); err != nil { + return "", err + } - logger.FromContext(ctx).Sugar().Warnf("writing deployed email...") - if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by_email"), []byte(existingDeployment.Metadata.DeployedByEmail), 0666); err != nil { - return "", err - } + logger.FromContext(ctx).Sugar().Warnf("writing deployed email...") + if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_by_email"), []byte(existingDeployment.Metadata.DeployedByEmail), 0666); err != nil { + return "", err + } - logger.FromContext(ctx).Sugar().Warnf("writing deployed at...") - if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_at_utc"), []byte(existingDeployment.Created.UTC().String()), 0666); err != nil { - return "", err + logger.FromContext(ctx).Sugar().Warnf("writing deployed at...") + if err := util.WriteFile(fsys, fsys.Join(applicationDir, "deployed_at_utc"), []byte(existingDeployment.Created.UTC().String()), 0666); err != nil { + return "", err + } } err = state.DeleteQueuedVersionIfExists(c.Environment, c.Application) @@ -453,7 +484,7 @@ func (c *DeployApplicationVersion) Transform( }, TransformerEslVersion: c.TransformerEslVersion, } - if err := t.Execute(d, transaction); err != nil { + if err := tCtx.Execute(d, transaction); err != nil { return "", err } return fmt.Sprintf("deployed version %d of %q to %q", c.Version, c.Application, c.Environment), nil @@ -505,9 +536,12 @@ func (c *CreateEnvironmentLock) GetDBEventType() db.EventType { func (c *CreateEnvironmentLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem envDir := fs.Join("environments", c.Environment) if _, err := fs.Stat(envDir); err != nil { @@ -591,9 +625,12 @@ func (c *DeleteEnvironmentLock) GetDBEventType() db.EventType { func (c *DeleteEnvironmentLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, _ *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem s := State{ Commit: nil, @@ -644,9 +681,12 @@ func (c *CreateEnvironmentApplicationLock) GetDBEventType() db.EventType { func (c *CreateEnvironmentApplicationLock) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem envDir := fs.Join("environments", c.Environment) if _, err := fs.Stat(envDir); err != nil { @@ -706,10 +746,12 @@ func (c *DeleteEnvironmentApplicationLock) SetEslVersion(eslVersion db.Transform func (c *DeleteEnvironmentApplicationLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { - + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem queueMessage := "" lockDir := fs.Join("environments", c.Environment, "applications", c.Application, "locks", c.LockId) @@ -763,7 +805,7 @@ func (c *CreateApplicationVersion) GetDBEventType() db.EventType { func (c *CreateApplicationVersion) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { version := c.Version @@ -774,8 +816,11 @@ func (c *CreateApplicationVersion) Transform( releaseDir := releasesDirectoryWithVersion(fs, c.Application, version) appDir := applicationDirectory(fs, c.Application) - if err := fs.MkdirAll(releaseDir, 0777); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + + if tCtx.ShouldMaximizeGitData() { + if err := fs.MkdirAll(releaseDir, 0777); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } var checkForInvalidCommitId = func(commitId, helperText string) { @@ -789,54 +834,56 @@ func (c *CreateApplicationVersion) Transform( checkForInvalidCommitId(c.SourceCommitId, "Source") checkForInvalidCommitId(c.PreviousCommit, "Previous") - if c.SourceCommitId != "" { - c.SourceCommitId = strings.ToLower(c.SourceCommitId) - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceCommitId), []byte(c.SourceCommitId), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if tCtx.ShouldMaximizeGitData() { + if c.SourceCommitId != "" { + c.SourceCommitId = strings.ToLower(c.SourceCommitId) + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceCommitId), []byte(c.SourceCommitId), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } - } - if c.SourceAuthor != "" { - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceAuthor), []byte(c.SourceAuthor), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if c.SourceAuthor != "" { + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceAuthor), []byte(c.SourceAuthor), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } - } - if c.SourceMessage != "" { - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceMessage), []byte(c.SourceMessage), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if c.SourceMessage != "" { + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldSourceMessage), []byte(c.SourceMessage), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } - } - if c.DisplayVersion != "" { - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldDisplayVersion), []byte(c.DisplayVersion), 0666); err != nil { + if c.DisplayVersion != "" { + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldDisplayVersion), []byte(c.DisplayVersion), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } + } + if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil { return "", GetCreateReleaseGeneralFailure(err) } - } - if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } - if c.Team != "" { - //util.WriteFile has a bug where it does not truncate the old file content. If two application versions with the same - //team are deployed, team names simply get concatenated. Just remove the file beforehand. - //This bug can't be fixed because it is part of the util library - teamFileLoc := fs.Join(appDir, fieldTeam) - if _, err := fs.Stat(teamFileLoc); err == nil { //If path to file exists - err := fs.Remove(teamFileLoc) - if err != nil { + if c.Team != "" { + //util.WriteFile has a bug where it does not truncate the old file content. If two application versions with the same + //team are deployed, team names simply get concatenated. Just remove the file beforehand. + //This bug can'tCtx be fixed because it is part of the util library + teamFileLoc := fs.Join(appDir, fieldTeam) + if _, err := fs.Stat(teamFileLoc); err == nil { //If path to file exists + err := fs.Remove(teamFileLoc) + if err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } + } + if err := util.WriteFile(fs, teamFileLoc, []byte(c.Team), 0666); err != nil { return "", GetCreateReleaseGeneralFailure(err) } } - if err := util.WriteFile(fs, teamFileLoc, []byte(c.Team), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } } - isLatest, err := isLatestVersion(state, c.Application, version) + isLatest, err := isLatestVersion(ctx, state, transaction, c.Application, version) if err != nil { return "", GetCreateReleaseGeneralFailure(err) } if !isLatest { // check that we can actually backfill this version - oldVersions, err := findOldApplicationVersions(ctx, transaction, state, c.Application) + oldVersions, err := findOldApplicationVersions(ctx, transaction, state, int(state.ReleaseVersionsLimit), c.Application) if err != nil { return "", GetCreateReleaseGeneralFailure(err) } @@ -869,7 +916,7 @@ func (c *CreateApplicationVersion) Transform( } } - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return "", err } @@ -878,9 +925,6 @@ func (c *CreateApplicationVersion) Transform( env := sortedKeys[i] man := c.Manifests[env] - if err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } envDir := fs.Join(releaseDir, "environments", env) config, found := configs[env] @@ -889,18 +933,20 @@ func (c *CreateApplicationVersion) Transform( hasUpstream = config.Upstream != nil } - if err = fs.MkdirAll(envDir, 0777); err != nil { - return "", GetCreateReleaseGeneralFailure(err) - } - if err := util.WriteFile(fs, fs.Join(envDir, "manifests.yaml"), []byte(man), 0666); err != nil { - return "", GetCreateReleaseGeneralFailure(err) + if tCtx.ShouldMaximizeGitData() { + if err = fs.MkdirAll(envDir, 0777); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } + if err := util.WriteFile(fs, fs.Join(envDir, "manifests.yaml"), []byte(man), 0666); err != nil { + return "", GetCreateReleaseGeneralFailure(err) + } } teamOwner, err := state.GetApplicationTeamOwner(ctx, transaction, c.Application) if err != nil { return "", err } - t.AddAppEnv(c.Application, env, teamOwner) + tCtx.AddAppEnv(c.Application, env, teamOwner) if hasUpstream && config.Upstream.Latest && isLatest { d := &DeployApplicationVersion{ SourceTrain: nil, @@ -917,7 +963,7 @@ func (c *CreateApplicationVersion) Transform( AuthorEmail: "", }, } - err := t.Execute(d, transaction) + err := tCtx.Execute(d, transaction) if err != nil { _, ok := err.(*LockedError) if ok { @@ -1002,13 +1048,13 @@ func writeNextPrevInfo(ctx context.Context, sourceCommitId string, otherCommitId return nil } -func isLatestVersion(state *State, application string, version uint64) (bool, error) { - rels, err := state.GetApplicationReleasesFromFile(application) +func isLatestVersion(ctx context.Context, state *State, transaction *sql.Tx, application string, version uint64) (bool, error) { + rels, err := state.DBHandler.DBSelectReleasesByAppLatestEslVersion(ctx, transaction, application, true) if err != nil { return false, err } for _, r := range rels { - if r > version { + if r.ReleaseNumber > version { return false, nil } } @@ -1017,13 +1063,14 @@ func isLatestVersion(state *State, application string, version uint64) (bool, er // Finds old releases for an application: Checks for the oldest release that is currently deployed on any environment // Releases older that the oldest deployed release are eligible for deletion. releaseVersionsLimit -func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state *State, name string) ([]uint64, error) { +func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state *State, versionLimit int, appName string) ([]uint64, error) { // 1) get release in each env: - envConfigs, err := state.GetEnvironmentConfigs() + envConfigs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) + //envConfigs, err := state.GetEnvironmentConfigs() if err != nil { return nil, err } - versions, err := state.GetApplicationReleasesFromFile(name) + versions, err := state.GetApplicationReleasesFromFile(appName) if err != nil { return nil, err } @@ -1036,7 +1083,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state // Use the latest version as oldest deployed version oldestDeployedVersion := versions[len(versions)-1] for env := range envConfigs { - version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, name) + version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, appName) if err != nil { return nil, err } @@ -1050,13 +1097,13 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state return versions[i] >= oldestDeployedVersion }) - if positionOfOldestVersion < (int(state.ReleaseVersionsLimit) - 1) { + if positionOfOldestVersion < (versionLimit - 1) { return nil, nil } indexToKeep := positionOfOldestVersion - 1 majorsCount := 0 for ; indexToKeep >= 0; indexToKeep-- { - release, err := state.DBHandler.DBSelectReleaseByVersion(ctx, transaction, name, versions[indexToKeep], false) + release, err := state.DBHandler.DBSelectReleaseByVersion(ctx, transaction, appName, versions[indexToKeep], false) if err != nil { return nil, err } @@ -1066,7 +1113,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state } else if !release.Metadata.IsMinor && !release.Metadata.IsPrepublish { majorsCount += 1 } - if majorsCount >= int(state.ReleaseVersionsLimit) { + if majorsCount >= versionLimit { break } } @@ -1126,9 +1173,12 @@ func (c *CreateEnvironmentTeamLock) GetDBEventType() db.EventType { func (c *CreateEnvironmentTeamLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, tx *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } if !valid.EnvironmentName(c.Environment) { return "", status.Error(codes.InvalidArgument, fmt.Sprintf("cannot create environment team lock: invalid environment: '%s'", c.Environment)) @@ -1216,9 +1266,13 @@ func (c *DeleteEnvironmentTeamLock) GetDBEventType() db.EventType { func (c *DeleteEnvironmentTeamLock) Transform( ctx context.Context, state *State, - _ TransformerContext, + tCtx TransformerContext, _ *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } + if !valid.EnvironmentName(c.Environment) { return "", status.Error(codes.InvalidArgument, fmt.Sprintf("cannot delete environment team lock: invalid environment: '%s'", c.Environment)) } @@ -1270,11 +1324,14 @@ func (c *CreateEnvironment) GetDBEventType() db.EventType { } func (c *CreateEnvironment) Transform( - ctx context.Context, + _ context.Context, state *State, - t TransformerContext, - transaction *sql.Tx, + tCtx TransformerContext, + _ *sql.Tx, ) (string, error) { + if tCtx.ShouldMinimizeGitData() { + return GetNoOpMessage(c) + } fs := state.Filesystem envDir := fs.Join("environments", c.Environment) if err := fs.MkdirAll(envDir, 0777); err != nil { @@ -1385,11 +1442,20 @@ func (c *CleanupOldApplicationVersions) GetDBEventType() db.EventType { func (c *CleanupOldApplicationVersions) Transform( ctx context.Context, state *State, - t TransformerContext, + tCtx TransformerContext, transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - oldVersions, err := findOldApplicationVersions(ctx, transaction, state, c.Application) + var versionLimit = int(state.ReleaseVersionsLimit) + var err error = nil + var oldVersions []uint64 + if tCtx.ShouldMinimizeGitData() { + versionLimit = 0 + //oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) + oldVersions, err = state.GetApplicationReleasesFromFile(c.Application) + } else { + oldVersions, err = findOldApplicationVersions(ctx, transaction, state, versionLimit, c.Application) + } if err != nil { return "", fmt.Errorf("cleanup: could not get application releases for app '%s': %w", c.Application, err) } @@ -1479,20 +1545,23 @@ func (u *ReleaseTrain) Transform( ctx context.Context, state *State, t TransformerContext, - tx *sql.Tx, + transaction *sql.Tx, ) (string, error) { //Gets deployments generated by the releasetrain with elsVersion u.TransformerEslVersion from the database and simply deploys them - deployments, err := state.DBHandler.DBSelectDeploymentsByTransformerID(ctx, tx, u.TransformerEslVersion, 100) + deployments, err := state.DBHandler.DBSelectDeploymentsByTransformerID(ctx, transaction, u.TransformerEslVersion, 100) if err != nil { return "", err } - skippedDeployments, err := state.DBHandler.DBSelectAllLockPreventedEventsForTransformerID(ctx, tx, u.TransformerEslVersion) + skippedDeployments, err := state.DBHandler.DBSelectAllLockPreventedEventsForTransformerID(ctx, transaction, u.TransformerEslVersion) if err != nil { return "", err } var targetGroupName = u.Target - configs, _ := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) + if err != nil { + return "", err + } var envGroupConfigs, isEnvGroup = getEnvironmentGroupsEnvironmentsOrEnvironment(configs, targetGroupName, u.TargetType) for _, currentDeployment := range deployments { envConfig := envGroupConfigs[currentDeployment.Env] @@ -1515,7 +1584,7 @@ func (u *ReleaseTrain) Transform( }, TransformerEslVersion: u.TransformerEslVersion, Author: "", - }, tx); err != nil { + }, transaction); err != nil { return "", err } } @@ -1559,47 +1628,49 @@ type DeleteEnvFromApp struct { TransformerEslVersion db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslVersion } -func (u *DeleteEnvFromApp) GetEslVersion() db.TransformerID { - return u.TransformerEslVersion +func (c *DeleteEnvFromApp) GetEslVersion() db.TransformerID { + return c.TransformerEslVersion } func (c *DeleteEnvFromApp) SetEslVersion(eslVersion db.TransformerID) { c.TransformerEslVersion = eslVersion } -func (u *DeleteEnvFromApp) GetDBEventType() db.EventType { +func (c *DeleteEnvFromApp) GetDBEventType() db.EventType { return db.EvtDeleteEnvFromApp } -func (u *DeleteEnvFromApp) Transform( +func (c *DeleteEnvFromApp) Transform( ctx context.Context, state *State, - t TransformerContext, - transsaction *sql.Tx, + tCtx TransformerContext, + _ *sql.Tx, ) (string, error) { fs := state.Filesystem thisSprintf := func(format string, a ...any) string { - return fmt.Sprintf("DeleteEnvFromApp app '%s' on env '%s': %s", u.Application, u.Environment, fmt.Sprintf(format, a...)) + return fmt.Sprintf("DeleteEnvFromApp app '%s' on env '%s': %s", c.Application, c.Environment, fmt.Sprintf(format, a...)) } - if u.Application == "" { + if c.Application == "" { return "", fmt.Errorf(thisSprintf("Need to provide the application")) } - if u.Environment == "" { + if c.Environment == "" { return "", fmt.Errorf(thisSprintf("Need to provide the environment")) } - envAppDir := environmentApplicationDirectory(fs, u.Environment, u.Application) + envAppDir := environmentApplicationDirectory(fs, c.Environment, c.Application) entries, err := fs.ReadDir(envAppDir) + msg := fmt.Sprintf("Attempted to remove environment '%v' from application '%v' but it did not exist", c.Environment, c.Application) if err != nil { - return "", wrapFileError(err, envAppDir, thisSprintf("Could not open application directory. Does the app exist?")) + return "", wrapFileError(err, envAppDir, thisSprintf("Could not open application directory")) } if entries == nil { // app was never deployed on this env, so that's unusual - but for idempotency we treat it just like a success case: - return fmt.Sprintf("Attempted to remove environment '%v' from application '%v' but it did not exist.", u.Environment, u.Application), nil + logger.FromContext(ctx).Warn(msg) + return msg, nil } err = fs.Remove(envAppDir) @@ -1607,8 +1678,8 @@ func (u *DeleteEnvFromApp) Transform( return "", wrapFileError(err, envAppDir, thisSprintf("Cannot delete app.'")) } - t.DeleteEnvFromApp(u.Application, u.Environment) - return fmt.Sprintf("Environment '%v' was removed from application '%v' successfully.", u.Environment, u.Application), nil + tCtx.DeleteEnvFromApp(c.Application, c.Environment) + return fmt.Sprintf("Environment '%v' was removed from application '%v' successfully.", c.Environment, c.Application), nil } type CreateUndeployApplicationVersion struct { @@ -1619,15 +1690,15 @@ type CreateUndeployApplicationVersion struct { TransformerEslVersion db.TransformerID `json:"-"` // Tags the transformer with EventSourcingLight eslVersion } -func (u *CreateUndeployApplicationVersion) GetEslVersion() db.TransformerID { - return u.TransformerEslVersion +func (c *CreateUndeployApplicationVersion) GetEslVersion() db.TransformerID { + return c.TransformerEslVersion } func (c *CreateUndeployApplicationVersion) SetEslVersion(eslVersion db.TransformerID) { c.TransformerEslVersion = eslVersion } -func (u *CreateUndeployApplicationVersion) GetDBEventType() db.EventType { +func (c *CreateUndeployApplicationVersion) GetDBEventType() db.EventType { return db.EvtCreateUndeployApplicationVersion } @@ -1638,20 +1709,23 @@ func (c *CreateUndeployApplicationVersion) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - lastRelease, err := state.GetLastRelease(ctx, fs, c.Application) + lastRelease, err := state.DBHandler.DBSelectReleasesByAppLatestEslVersion(ctx, transaction, c.Application, false) + //lastRelease, err := state.GetLastRelease(ctx, fs, c.Application) if err != nil { - return "", fmt.Errorf("Could not get last reelase for app '%v': %v\n", c.Application, err) + return "", fmt.Errorf("Could not get last relase for app '%v': %v\n", c.Application, err) } - if lastRelease == 0 { - return "", fmt.Errorf("cannot undeploy non-existing application '%v'", c.Application) + var nextReleaseNumber uint64 = 0 + if len(lastRelease) == 0 { + return "", fmt.Errorf("cannot undeploy application '%v'", c.Application) } + nextReleaseNumber = lastRelease[0].ReleaseNumber - releaseDir := releasesDirectoryWithVersion(fs, c.Application, lastRelease+1) + releaseDir := releasesDirectoryWithVersion(fs, c.Application, nextReleaseNumber) if err = fs.MkdirAll(releaseDir, 0777); err != nil { return "", err } - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) if err != nil { return "", err } @@ -1663,9 +1737,6 @@ func (c *CreateUndeployApplicationVersion) Transform( return "", err } for env := range configs { - if err != nil { - return "", err - } envDir := fs.Join(releaseDir, "environments", env) config, found := configs[env] @@ -1694,7 +1765,7 @@ func (c *CreateUndeployApplicationVersion) Transform( SourceTrain: nil, Environment: env, Application: c.Application, - Version: lastRelease + 1, + Version: nextReleaseNumber, // the train should queue deployments, instead of giving up: LockBehaviour: api.LockBehavior_RECORD, Authentication: c.Authentication, @@ -1717,7 +1788,7 @@ func (c *CreateUndeployApplicationVersion) Transform( } } } - return fmt.Sprintf("created undeploy-version %d of '%v'", lastRelease+1, c.Application), nil + return fmt.Sprintf("created undeploy-version %d of '%v'", nextReleaseNumber, c.Application), nil } type UndeployApplication struct { @@ -1735,8 +1806,8 @@ func (u *UndeployApplication) GetDBEventType() db.EventType { return db.EvtUndeployApplication } -func (c *UndeployApplication) SetEslVersion(id db.TransformerID) { - c.TransformerEslVersion = id +func (u *UndeployApplication) SetEslVersion(id db.TransformerID) { + u.TransformerEslVersion = id } func (u *UndeployApplication) Transform( @@ -1746,29 +1817,23 @@ func (u *UndeployApplication) Transform( transaction *sql.Tx, ) (string, error) { fs := state.Filesystem - lastRelease, err := state.GetLastRelease(ctx, fs, u.Application) + lastRelease, err := state.DBHandler.DBSelectLastReleasesByApp(ctx, transaction, u.Application, false, true) if err != nil { return "", err } - if lastRelease == 0 { - return "", fmt.Errorf("UndeployApplication: error cannot undeploy non-existing application '%v'", u.Application) - } - isUndeploy, err := state.IsUndeployVersion(u.Application, lastRelease) - if err != nil { - return "", err + if lastRelease == nil { + return "", fmt.Errorf("cannot undeploy application without releases '%v'", u.Application) } + isUndeploy := lastRelease.Metadata.UndeployVersion if !isUndeploy { - return "", fmt.Errorf("UndeployApplication: error last release is not un-deployed application version of '%v'", u.Application) + return "", fmt.Errorf("UndeployApplication: last release is not un-deployed application version of '%v'", u.Application) } appDir := applicationDirectory(fs, u.Application) - configs, err := state.GetEnvironmentConfigs() + configs, err := state.GetAllEnvironmentConfigsFromDB(ctx, transaction) // we use ALL envs, to be sure if err != nil { - return "", err + return "", fmt.Errorf("could not get environment configs: %w", err) } for env := range configs { - if err != nil { - return "", err - } envAppDir := environmentApplicationDirectory(fs, env, u.Application) entries, err := fs.ReadDir(envAppDir) if err != nil { @@ -1824,14 +1889,15 @@ func (u *UndeployApplication) Transform( } } } - if err = fs.Remove(appDir); err != nil { - return "", err + + if err = fs.Remove(appDir); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", wrapFileError(err, appDir, "UndeployApplication: could not remove application directory") } for env := range configs { appDir := environmentApplicationDirectory(fs, env, u.Application) teamOwner, err := state.GetApplicationTeamOwner(ctx, transaction, u.Application) if err != nil { - return "", err + return "", fmt.Errorf("could not find team for app %s: %w", u.Application, err) } t.AddAppEnv(u.Application, env, teamOwner) // remove environment application @@ -1839,6 +1905,7 @@ func (u *UndeployApplication) Transform( return "", fmt.Errorf("UndeployApplication: unexpected error application '%v' environment '%v': '%w'", u.Application, env, err) } } + return fmt.Sprintf("application '%v' was deleted successfully", u.Application), nil } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 8b752281a..512336834 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -95,7 +95,7 @@ func setupRepositoryTestWithPath(t *testing.T) (Repository, string) { CommitterEmail: "kuberpult@freiheit.com", CommitterName: "kuberpult", ArgoCdGenerateFiles: true, - ReleaseVersionLimit: 2, + ReleaseVersionLimit: 0, } if dbConfig != nil { diff --git a/services/manifest-repo-export-service/pkg/service/git_test.go b/services/manifest-repo-export-service/pkg/service/git_test.go index f4fd50b88..c2a8e30fe 100644 --- a/services/manifest-repo-export-service/pkg/service/git_test.go +++ b/services/manifest-repo-export-service/pkg/service/git_test.go @@ -54,11 +54,12 @@ func setupRepositoryTestWithoutDB(t *testing.T) (rp.Repository, error) { t.Logf("test created dir: %s", localDir) repoCfg := rp.RepositoryConfig{ - URL: remoteDir, - Path: localDir, - CommitterEmail: "kuberpult@freiheit.com", - CommitterName: "kuberpult", - ArgoCdGenerateFiles: true, + URL: remoteDir, + Path: localDir, + CommitterEmail: "kuberpult@freiheit.com", + CommitterName: "kuberpult", + ArgoCdGenerateFiles: true, + MinimizeExportedData: false, } repoCfg.DBHandler = nil @@ -1031,8 +1032,9 @@ func TestGetCommitInfo(t *testing.T) { } config := rp.RepositoryConfig{ - ArgoCdGenerateFiles: true, - DBHandler: repo.State().DBHandler, + ArgoCdGenerateFiles: true, + DBHandler: repo.State().DBHandler, + MinimizeExportedData: false, } sv := &GitServer{ Repository: repo, diff --git a/services/manifest-repo-export-service/pkg/service/version_test.go b/services/manifest-repo-export-service/pkg/service/version_test.go index 4b318bbfe..99e6ca3ee 100644 --- a/services/manifest-repo-export-service/pkg/service/version_test.go +++ b/services/manifest-repo-export-service/pkg/service/version_test.go @@ -68,12 +68,13 @@ func setupRepositoryTestWithPath(t *testing.T) (repository.Repository, string) { } repoCfg := repository.RepositoryConfig{ - URL: remoteDir, - Path: localDir, - CommitterEmail: "kuberpult@freiheit.com", - CommitterName: "kuberpult", - ArgoCdGenerateFiles: true, - ReleaseVersionLimit: 2, + URL: remoteDir, + Path: localDir, + CommitterEmail: "kuberpult@freiheit.com", + CommitterName: "kuberpult", + ArgoCdGenerateFiles: true, + ReleaseVersionLimit: 2, + MinimizeExportedData: false, } if dbConfig != nil {