From 13537245efb076a2eeafafcbc95797e96278bbac Mon Sep 17 00:00:00 2001 From: Sven Urbanski Date: Thu, 20 Jun 2024 14:51:59 +0200 Subject: [PATCH] fix(db): render argo manifests correctly (#1681) The manifest-export service accidentally used the manifest repo to gather data. Now it uses the database. --- .../cd-service/pkg/repository/repository.go | 6 +-- .../cd-service/pkg/repository/transformer.go | 10 ++-- .../pkg/repository/transformer_test.go | 46 +++++++++---------- services/cd-service/pkg/service/batch_test.go | 4 +- services/cd-service/pkg/service/overview.go | 2 +- services/cd-service/pkg/service/version.go | 2 +- .../pkg/repository/repository.go | 32 +++++++++---- .../pkg/repository/transformer.go | 6 +-- 8 files changed, 61 insertions(+), 47 deletions(-) diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index 80f87d679..936cb8562 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -1307,7 +1307,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, state *State, env str if err != nil { return err } - version, err := state.GetEnvironmentApplicationVersion(ctx, env, appName, transaction) + version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, appName) if err != nil { if errors.Is(err, os.ErrNotExist) { // if the app does not exist, we skip it @@ -1921,7 +1921,7 @@ func (s *State) DeleteQueuedVersionIfExists(ctx context.Context, transaction *sq return s.DeleteQueuedVersion(ctx, transaction, environment, application) } -func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, environment, application string, transaction *sql.Tx) (*uint64, error) { +func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, transaction *sql.Tx, environment string, application string) (*uint64, error) { if s.DBHandler.ShouldUseOtherTables() && transaction != nil { depl, err := s.DBHandler.DBSelectDeployment(ctx, transaction, application, environment) if err != nil { @@ -2737,7 +2737,7 @@ func (s *State) ProcessQueue(ctx context.Context, transaction *sql.Tx, fs billy. return "", nil } - currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(ctx, environment, application, transaction) + currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(ctx, transaction, environment, application) if err != nil { return "", err } diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index cd1d9a7a4..79a95b57f 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -1403,7 +1403,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state // Use the latest version as oldest deployed version oldestDeployedVersion := versions[len(versions)-1] for env := range envConfigs { - version, err := state.GetEnvironmentApplicationVersion(ctx, env, name, transaction) + version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, name) if err != nil { return nil, err } @@ -2658,7 +2658,7 @@ func getOverrideVersions(ctx context.Context, transaction *sql.Tx, commitHash, u TeamLocks: nil, Team: "", } - version, err := s.GetEnvironmentApplicationVersion(ctx, envName, appName, transaction) + version, err := s.GetEnvironmentApplicationVersion(ctx, transaction, envName, appName) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("unable to get EnvironmentApplicationVersion for %s: %w", appName, err) } @@ -3010,7 +3010,7 @@ func (c *envReleaseTrain) prognosis( } } - currentlyDeployedVersion, err := state.GetEnvironmentApplicationVersion(ctx, c.Env, appName, transaction) + currentlyDeployedVersion, err := state.GetEnvironmentApplicationVersion(ctx, transaction, c.Env, appName) if err != nil { return ReleaseTrainEnvironmentPrognosis{ SkipCause: nil, @@ -3038,7 +3038,7 @@ func (c *envReleaseTrain) prognosis( } } } else { - upstreamVersion, err := state.GetEnvironmentApplicationVersion(ctx, upstreamEnvName, appName, transaction) + upstreamVersion, err := state.GetEnvironmentApplicationVersion(ctx, transaction, upstreamEnvName, appName) if err != nil { return ReleaseTrainEnvironmentPrognosis{ SkipCause: nil, @@ -3195,7 +3195,7 @@ func (c *envReleaseTrain) Transform( renderApplicationSkipCause := func(SkipCause *api.ReleaseTrainAppPrognosis_SkipCause, appName string) string { envConfig := c.EnvGroupConfigs[c.Env] upstreamEnvName := envConfig.Upstream.Environment - currentlyDeployedVersion, _ := state.GetEnvironmentApplicationVersion(ctx, c.Env, appName, transaction) + currentlyDeployedVersion, _ := state.GetEnvironmentApplicationVersion(ctx, transaction, c.Env, appName) teamName, _ := state.GetTeamName(ctx, transaction, appName) switch SkipCause.SkipCause { case api.ReleaseTrainAppSkipCause_APP_HAS_NO_VERSION_IN_UPSTREAM_ENV: diff --git a/services/cd-service/pkg/repository/transformer_test.go b/services/cd-service/pkg/repository/transformer_test.go index 8bb8d3d84..65ce3db75 100644 --- a/services/cd-service/pkg/repository/transformer_test.go +++ b/services/cd-service/pkg/repository/transformer_test.go @@ -4573,7 +4573,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersForDelete(3), Test: func(t *testing.T, s *State) { { - prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil) + prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test") if err != nil { t.Fatal(err) } @@ -4599,7 +4599,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersForDelete(5), Test: func(t *testing.T, s *State) { { - prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil) + prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test") if err != nil { t.Fatal(err) } @@ -4625,7 +4625,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersForDelete(18), Test: func(t *testing.T, s *State) { { - prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil) + prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test") if err != nil { t.Fatal(err) } @@ -4716,11 +4716,11 @@ func TestTransformer(t *testing.T) { }, Test: func(t *testing.T, s *State) { { - prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil) + prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test") if err != nil { t.Fatal(err) } - acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envAcceptance, "test", nil) + acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envAcceptance, "test") if err != nil { t.Fatal(err) } @@ -4764,7 +4764,7 @@ func TestTransformer(t *testing.T) { }, Test: func(t *testing.T, s *State) { { - acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envAcceptance, "test", nil) + acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envAcceptance, "test") if err != nil { t.Fatal(err) } @@ -4833,11 +4833,11 @@ func TestTransformer(t *testing.T) { }, Test: func(t *testing.T, s *State) { { - prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil) + prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test") if err != nil { t.Fatal(err) } - acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envAcceptance, "test", nil) + acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envAcceptance, "test") if err != nil { t.Fatal(err) } @@ -5058,7 +5058,7 @@ func TestTransformer(t *testing.T) { Test: func(t *testing.T, s *State) { // check that the state reads the correct versions { - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5239,7 +5239,7 @@ func TestTransformer(t *testing.T) { }, }, Test: func(t *testing.T, s *State) { - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5292,7 +5292,7 @@ func TestTransformer(t *testing.T) { Test: func(t *testing.T, s *State) { // check that the state reads the correct versions { - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "one", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "one", "test") if err != nil { t.Fatal(err) } @@ -5339,14 +5339,14 @@ func TestTransformer(t *testing.T) { // check that the state reads the correct versions { // version should only exist for "two" - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "two", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "two", "test") if err != nil { t.Fatal(err) } if *i != 1 { t.Errorf("unexpected version: expected 1, actual %d", i) } - i, err = s.GetEnvironmentApplicationVersion(context.Background(), "one", "test", nil) + i, err = s.GetEnvironmentApplicationVersion(context.Background(), nil, "one", "test") if i != nil || err != nil { t.Fatalf("expect file to not exist, because the env is locked.") } @@ -5391,7 +5391,7 @@ func TestTransformer(t *testing.T) { Test: func(t *testing.T, s *State) { // check that the state reads the correct versions { - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "one", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "one", "test") if err != nil { t.Fatal(err) } @@ -5433,7 +5433,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersDeployTestEnvLock(api.LockBehavior_IGNORE), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5447,7 +5447,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersDeployTestEnvLock(api.LockBehavior_RECORD), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5461,7 +5461,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersDeployTestAppLock(api.LockBehavior_IGNORE), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5475,7 +5475,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersDeployTestAppLock(api.LockBehavior_RECORD), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil && err.Error() != "file does not exist" { t.Fatalf("unexpected error: %v", err.Error()) } @@ -5520,7 +5520,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersTwoDeploymentsWriteToQueue(api.LockBehavior_RECORD, api.LockBehavior_RECORD), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5546,7 +5546,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersTwoDeploymentsWriteToQueue(api.LockBehavior_RECORD, api.LockBehavior_IGNORE), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5572,7 +5572,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersTwoDeploymentsWriteToQueue(api.LockBehavior_IGNORE, api.LockBehavior_RECORD), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5602,7 +5602,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersDoubleLock(api.LockBehavior_RECORD, false), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } @@ -5628,7 +5628,7 @@ func TestTransformer(t *testing.T) { Transformers: makeTransformersDoubleLock(api.LockBehavior_RECORD, true), Test: func(t *testing.T, s *State) { // check that the state reads the correct versions - i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } diff --git a/services/cd-service/pkg/service/batch_test.go b/services/cd-service/pkg/service/batch_test.go index 45e649143..02fe4e874 100644 --- a/services/cd-service/pkg/service/batch_test.go +++ b/services/cd-service/pkg/service/batch_test.go @@ -263,7 +263,7 @@ func TestBatchServiceWorks(t *testing.T) { } // check deployment version { - version, err := tc.svc.Repository.State().GetEnvironmentApplicationVersion(tc.context, "production", "test", nil) + version, err := tc.svc.Repository.State().GetEnvironmentApplicationVersion(tc.context, nil, "production", "test") if err != nil { t.Fatal(err) } @@ -592,7 +592,7 @@ func TestBatchServiceLimit(t *testing.T) { if err != nil { t.Fatal(err) } - version, err := svc.Repository.State().GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil) + version, err := svc.Repository.State().GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test") if err != nil { t.Fatal(err) } diff --git a/services/cd-service/pkg/service/overview.go b/services/cd-service/pkg/service/overview.go index 3dc502222..2d4c07a5c 100644 --- a/services/cd-service/pkg/service/overview.go +++ b/services/cd-service/pkg/service/overview.go @@ -192,7 +192,7 @@ func (o *OverviewServiceServer) getOverview( } // Err != nil means no team name was found so no need to parse team locks var version *uint64 - version, err = s.GetEnvironmentApplicationVersion(ctx, envName, appName, transaction) + version, err = s.GetEnvironmentApplicationVersion(ctx, transaction, envName, appName) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } else { diff --git a/services/cd-service/pkg/service/version.go b/services/cd-service/pkg/service/version.go index 9db5a269a..7a2237c83 100644 --- a/services/cd-service/pkg/service/version.go +++ b/services/cd-service/pkg/service/version.go @@ -64,7 +64,7 @@ func (o *VersionServiceServer) GetVersion( if state.DBHandler.ShouldUseOtherTables() { return nil, grpc.PublicError(ctx, fmt.Errorf("getVersion: not supported yet for Database mode")) } - version, err := state.GetEnvironmentApplicationVersion(ctx, in.Environment, in.Application, nil) + version, err := state.GetEnvironmentApplicationVersion(ctx, nil, in.Environment, in.Application) if err != nil { return nil, err } diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 31adbf1f0..be81d6c7f 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -730,7 +730,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, transaction *sql.Tx, span, ctx := tracer.StartSpanFromContext(ctx, "updateArgoCdApps") defer span.Finish() fs := state.Filesystem - if apps, err := state.GetEnvironmentApplications(env); err != nil { + if apps, err := state.GetEnvironmentApplications(ctx, transaction, env); err != nil { return err } else { spanCollectData, _ := tracer.StartSpanFromContext(ctx, "collectData") @@ -746,7 +746,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, transaction *sql.Tx, if err != nil { return fmt.Errorf("updateArgoCdApps: could not select app '%s' in db %v", appName, err) } - version, err := state.GetEnvironmentApplicationVersion(env, appName) + version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, appName) if err != nil { if errors.Is(err, os.ErrNotExist) { // if the app does not exist, we skip it @@ -1148,8 +1148,16 @@ func (s *State) DeleteQueuedVersionIfExists(environment string, application stri return s.DeleteQueuedVersion(environment, application) } -func (s *State) GetEnvironmentApplicationVersion(environment, application string) (*uint64, error) { - return s.readSymlink(environment, application, "version") +func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, transaction *sql.Tx, environment, application string) (*uint64, error) { + depl, err := s.DBHandler.DBSelectDeployment(ctx, transaction, application, environment) + if err != nil { + return nil, err + } + if depl == nil || depl.Version == nil { + return nil, nil + } + var v = uint64(*depl.Version) + return &v, nil } // returns nil if there is no file @@ -1288,9 +1296,15 @@ func (s *State) GetEnvironmentConfigsForGroup(envGroup string) ([]string, error) return groupEnvNames, nil } -func (s *State) GetEnvironmentApplications(environment string) ([]string, error) { - appDir := s.Filesystem.Join("environments", environment, "applications") - return names(s.Filesystem, appDir) +func (s *State) GetEnvironmentApplications(ctx context.Context, transaction *sql.Tx, environment string) ([]string, error) { + applications, err := s.DBHandler.DBSelectAllApplications(ctx, transaction) + if err != nil { + return nil, err + } + if applications == nil { + return make([]string, 0), nil + } + return applications.Apps, nil } // GetApplicationsFromFile returns apps from the filesystem @@ -1543,7 +1557,7 @@ func readFile(fs billy.Filesystem, path string) ([]byte, error) { // ProcessQueue checks if there is something in the queue // deploys if necessary // deletes the queue -func (s *State) ProcessQueue(ctx context.Context, fs billy.Filesystem, environment string, application string) (string, error) { +func (s *State) ProcessQueue(ctx context.Context, transaction *sql.Tx, fs billy.Filesystem, environment string, application string) (string, error) { queuedVersion, err := s.GetQueuedVersion(environment, application) queueDeploymentMessage := "" if err != nil { @@ -1555,7 +1569,7 @@ func (s *State) ProcessQueue(ctx context.Context, fs billy.Filesystem, environme return "", nil } - currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(environment, application) + currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(ctx, transaction, environment, application) if err != nil { return "", err } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 0c0689491..555f5d405 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -555,7 +555,7 @@ func (c *DeleteEnvironmentApplicationLock) Transform( ctx context.Context, state *State, _ TransformerContext, - _ *sql.Tx, + transaction *sql.Tx, ) (string, error) { fs := state.Filesystem @@ -569,7 +569,7 @@ func (c *DeleteEnvironmentApplicationLock) Transform( return "", fmt.Errorf("failed to delete directory %q: %w.", lockDir, err) } - queueMessage, err = state.ProcessQueue(ctx, fs, c.Environment, c.Application) + queueMessage, err = state.ProcessQueue(ctx, transaction, fs, c.Environment, c.Application) if err != nil { return "", err } @@ -734,7 +734,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state // Use the latest version as oldest deployed version oldestDeployedVersion := versions[len(versions)-1] for env := range envConfigs { - version, err := state.GetEnvironmentApplicationVersion(env, name) + version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, name) if err != nil { return nil, err }