Skip to content

Commit

Permalink
fix(db): render argo manifests correctly (#1681)
Browse files Browse the repository at this point in the history 
The manifest-export service accidentally used the manifest repo to
gather data. Now it uses the database.
  • Loading branch information
@sven-urbanski-freiheit-com
sven-urbanski-freiheit-com authored Jun 20, 2024
1 parent 09fbfdb commit 1353724
Show file tree
Hide file tree
Showing 8 changed files with 61 additions and 47 deletions.
6 changes: 3 additions & 3 deletions services/cd-service/pkg/repository/repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1307,7 +1307,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, state *State, env str
if err != nil {
return err
}
version, err := state.GetEnvironmentApplicationVersion(ctx, env, appName, transaction)
version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, appName)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
// if the app does not exist, we skip it
Expand Down Expand Up @@ -1921,7 +1921,7 @@ func (s *State) DeleteQueuedVersionIfExists(ctx context.Context, transaction *sq
return s.DeleteQueuedVersion(ctx, transaction, environment, application)
}

func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, environment, application string, transaction *sql.Tx) (*uint64, error) {
func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, transaction *sql.Tx, environment string, application string) (*uint64, error) {
if s.DBHandler.ShouldUseOtherTables() && transaction != nil {
depl, err := s.DBHandler.DBSelectDeployment(ctx, transaction, application, environment)
if err != nil {
Expand Down Expand Up @@ -2737,7 +2737,7 @@ func (s *State) ProcessQueue(ctx context.Context, transaction *sql.Tx, fs billy.
return "", nil
}

currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(ctx, environment, application, transaction)
currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(ctx, transaction, environment, application)
if err != nil {
return "", err
}
Expand Down
10 changes: 5 additions & 5 deletions services/cd-service/pkg/repository/transformer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1403,7 +1403,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state
// Use the latest version as oldest deployed version
oldestDeployedVersion := versions[len(versions)-1]
for env := range envConfigs {
version, err := state.GetEnvironmentApplicationVersion(ctx, env, name, transaction)
version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, name)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -2658,7 +2658,7 @@ func getOverrideVersions(ctx context.Context, transaction *sql.Tx, commitHash, u
TeamLocks: nil,
Team: "",
}
version, err := s.GetEnvironmentApplicationVersion(ctx, envName, appName, transaction)
version, err := s.GetEnvironmentApplicationVersion(ctx, transaction, envName, appName)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("unable to get EnvironmentApplicationVersion for %s: %w", appName, err)
}
Expand Down Expand Up @@ -3010,7 +3010,7 @@ func (c *envReleaseTrain) prognosis(
}
}

currentlyDeployedVersion, err := state.GetEnvironmentApplicationVersion(ctx, c.Env, appName, transaction)
currentlyDeployedVersion, err := state.GetEnvironmentApplicationVersion(ctx, transaction, c.Env, appName)
if err != nil {
return ReleaseTrainEnvironmentPrognosis{
SkipCause: nil,
Expand Down Expand Up @@ -3038,7 +3038,7 @@ func (c *envReleaseTrain) prognosis(
}
}
} else {
upstreamVersion, err := state.GetEnvironmentApplicationVersion(ctx, upstreamEnvName, appName, transaction)
upstreamVersion, err := state.GetEnvironmentApplicationVersion(ctx, transaction, upstreamEnvName, appName)
if err != nil {
return ReleaseTrainEnvironmentPrognosis{
SkipCause: nil,
Expand Down Expand Up @@ -3195,7 +3195,7 @@ func (c *envReleaseTrain) Transform(
renderApplicationSkipCause := func(SkipCause *api.ReleaseTrainAppPrognosis_SkipCause, appName string) string {
envConfig := c.EnvGroupConfigs[c.Env]
upstreamEnvName := envConfig.Upstream.Environment
currentlyDeployedVersion, _ := state.GetEnvironmentApplicationVersion(ctx, c.Env, appName, transaction)
currentlyDeployedVersion, _ := state.GetEnvironmentApplicationVersion(ctx, transaction, c.Env, appName)
teamName, _ := state.GetTeamName(ctx, transaction, appName)
switch SkipCause.SkipCause {
case api.ReleaseTrainAppSkipCause_APP_HAS_NO_VERSION_IN_UPSTREAM_ENV:
Expand Down
46 changes: 23 additions & 23 deletions services/cd-service/pkg/repository/transformer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4573,7 +4573,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersForDelete(3),
Test: func(t *testing.T, s *State) {
{
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil)
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -4599,7 +4599,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersForDelete(5),
Test: func(t *testing.T, s *State) {
{
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil)
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -4625,7 +4625,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersForDelete(18),
Test: func(t *testing.T, s *State) {
{
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil)
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -4716,11 +4716,11 @@ func TestTransformer(t *testing.T) {
},
Test: func(t *testing.T, s *State) {
{
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil)
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test")
if err != nil {
t.Fatal(err)
}
acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envAcceptance, "test", nil)
acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envAcceptance, "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -4764,7 +4764,7 @@ func TestTransformer(t *testing.T) {
},
Test: func(t *testing.T, s *State) {
{
acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envAcceptance, "test", nil)
acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envAcceptance, "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -4833,11 +4833,11 @@ func TestTransformer(t *testing.T) {
},
Test: func(t *testing.T, s *State) {
{
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envProduction, "test", nil)
prodVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envProduction, "test")
if err != nil {
t.Fatal(err)
}
acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), envAcceptance, "test", nil)
acceptanceVersion, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, envAcceptance, "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -5058,7 +5058,7 @@ func TestTransformer(t *testing.T) {
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
{
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -5239,7 +5239,7 @@ func TestTransformer(t *testing.T) {
},
},
Test: func(t *testing.T, s *State) {
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -5292,7 +5292,7 @@ func TestTransformer(t *testing.T) {
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
{
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "one", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "one", "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -5339,14 +5339,14 @@ func TestTransformer(t *testing.T) {
// check that the state reads the correct versions
{
// version should only exist for "two"
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "two", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "two", "test")
if err != nil {
t.Fatal(err)
}
if *i != 1 {
t.Errorf("unexpected version: expected 1, actual %d", i)
}
i, err = s.GetEnvironmentApplicationVersion(context.Background(), "one", "test", nil)
i, err = s.GetEnvironmentApplicationVersion(context.Background(), nil, "one", "test")
if i != nil || err != nil {
t.Fatalf("expect file to not exist, because the env is locked.")
}
Expand Down Expand Up @@ -5391,7 +5391,7 @@ func TestTransformer(t *testing.T) {
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
{
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "one", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "one", "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -5433,7 +5433,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersDeployTestEnvLock(api.LockBehavior_IGNORE),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -5447,7 +5447,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersDeployTestEnvLock(api.LockBehavior_RECORD),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -5461,7 +5461,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersDeployTestAppLock(api.LockBehavior_IGNORE),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -5475,7 +5475,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersDeployTestAppLock(api.LockBehavior_RECORD),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil && err.Error() != "file does not exist" {
t.Fatalf("unexpected error: %v", err.Error())
}
Expand Down Expand Up @@ -5520,7 +5520,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersTwoDeploymentsWriteToQueue(api.LockBehavior_RECORD, api.LockBehavior_RECORD),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -5546,7 +5546,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersTwoDeploymentsWriteToQueue(api.LockBehavior_RECORD, api.LockBehavior_IGNORE),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -5572,7 +5572,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersTwoDeploymentsWriteToQueue(api.LockBehavior_IGNORE, api.LockBehavior_RECORD),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -5602,7 +5602,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersDoubleLock(api.LockBehavior_RECORD, false),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand All @@ -5628,7 +5628,7 @@ func TestTransformer(t *testing.T) {
Transformers: makeTransformersDoubleLock(api.LockBehavior_RECORD, true),
Test: func(t *testing.T, s *State) {
// check that the state reads the correct versions
i, err := s.GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
i, err := s.GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand Down
4 changes: 2 additions & 2 deletions services/cd-service/pkg/service/batch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ func TestBatchServiceWorks(t *testing.T) {
}
// check deployment version
{
version, err := tc.svc.Repository.State().GetEnvironmentApplicationVersion(tc.context, "production", "test", nil)
version, err := tc.svc.Repository.State().GetEnvironmentApplicationVersion(tc.context, nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -592,7 +592,7 @@ func TestBatchServiceLimit(t *testing.T) {
if err != nil {
t.Fatal(err)
}
version, err := svc.Repository.State().GetEnvironmentApplicationVersion(context.Background(), "production", "test", nil)
version, err := svc.Repository.State().GetEnvironmentApplicationVersion(context.Background(), nil, "production", "test")
if err != nil {
t.Fatal(err)
}
Expand Down
2 changes: 1 addition & 1 deletion services/cd-service/pkg/service/overview.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func (o *OverviewServiceServer) getOverview(
} // Err != nil means no team name was found so no need to parse team locks

var version *uint64
version, err = s.GetEnvironmentApplicationVersion(ctx, envName, appName, transaction)
version, err = s.GetEnvironmentApplicationVersion(ctx, transaction, envName, appName)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
} else {
Expand Down
2 changes: 1 addition & 1 deletion services/cd-service/pkg/service/version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func (o *VersionServiceServer) GetVersion(
if state.DBHandler.ShouldUseOtherTables() {
return nil, grpc.PublicError(ctx, fmt.Errorf("getVersion: not supported yet for Database mode"))
}
version, err := state.GetEnvironmentApplicationVersion(ctx, in.Environment, in.Application, nil)
version, err := state.GetEnvironmentApplicationVersion(ctx, nil, in.Environment, in.Application)
if err != nil {
return nil, err
}
Expand Down
32 changes: 23 additions & 9 deletions services/manifest-repo-export-service/pkg/repository/repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -730,7 +730,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, transaction *sql.Tx,
span, ctx := tracer.StartSpanFromContext(ctx, "updateArgoCdApps")
defer span.Finish()
fs := state.Filesystem
if apps, err := state.GetEnvironmentApplications(env); err != nil {
if apps, err := state.GetEnvironmentApplications(ctx, transaction, env); err != nil {
return err
} else {
spanCollectData, _ := tracer.StartSpanFromContext(ctx, "collectData")
Expand All @@ -746,7 +746,7 @@ func (r *repository) updateArgoCdApps(ctx context.Context, transaction *sql.Tx,
if err != nil {
return fmt.Errorf("updateArgoCdApps: could not select app '%s' in db %v", appName, err)
}
version, err := state.GetEnvironmentApplicationVersion(env, appName)
version, err := state.GetEnvironmentApplicationVersion(ctx, transaction, env, appName)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
// if the app does not exist, we skip it
Expand Down Expand Up @@ -1148,8 +1148,16 @@ func (s *State) DeleteQueuedVersionIfExists(environment string, application stri
return s.DeleteQueuedVersion(environment, application)
}

func (s *State) GetEnvironmentApplicationVersion(environment, application string) (*uint64, error) {
return s.readSymlink(environment, application, "version")
func (s *State) GetEnvironmentApplicationVersion(ctx context.Context, transaction *sql.Tx, environment, application string) (*uint64, error) {
depl, err := s.DBHandler.DBSelectDeployment(ctx, transaction, application, environment)
if err != nil {
return nil, err
}
if depl == nil || depl.Version == nil {
return nil, nil
}
var v = uint64(*depl.Version)
return &v, nil
}

// returns nil if there is no file
Expand Down Expand Up @@ -1288,9 +1296,15 @@ func (s *State) GetEnvironmentConfigsForGroup(envGroup string) ([]string, error)
return groupEnvNames, nil
}

func (s *State) GetEnvironmentApplications(environment string) ([]string, error) {
appDir := s.Filesystem.Join("environments", environment, "applications")
return names(s.Filesystem, appDir)
func (s *State) GetEnvironmentApplications(ctx context.Context, transaction *sql.Tx, environment string) ([]string, error) {
applications, err := s.DBHandler.DBSelectAllApplications(ctx, transaction)
if err != nil {
return nil, err
}
if applications == nil {
return make([]string, 0), nil
}
return applications.Apps, nil
}

// GetApplicationsFromFile returns apps from the filesystem
Expand Down Expand Up @@ -1543,7 +1557,7 @@ func readFile(fs billy.Filesystem, path string) ([]byte, error) {
// ProcessQueue checks if there is something in the queue
// deploys if necessary
// deletes the queue
func (s *State) ProcessQueue(ctx context.Context, fs billy.Filesystem, environment string, application string) (string, error) {
func (s *State) ProcessQueue(ctx context.Context, transaction *sql.Tx, fs billy.Filesystem, environment string, application string) (string, error) {
queuedVersion, err := s.GetQueuedVersion(environment, application)
queueDeploymentMessage := ""
if err != nil {
Expand All @@ -1555,7 +1569,7 @@ func (s *State) ProcessQueue(ctx context.Context, fs billy.Filesystem, environme
return "", nil
}

currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(environment, application)
currentlyDeployedVersion, err := s.GetEnvironmentApplicationVersion(ctx, transaction, environment, application)
if err != nil {
return "", err
}
Expand Down
Loading

0 comments on commit 1353724

Please sign in to comment.