diff --git a/.github/workflows/execution-plan-snippet-earthly.yml b/.github/workflows/execution-plan-snippet-earthly.yml index edb2de73f..c9494cb81 100644 --- a/.github/workflows/execution-plan-snippet-earthly.yml +++ b/.github/workflows/execution-plan-snippet-earthly.yml @@ -20,6 +20,10 @@ jobs: name: "Authenticate to Google Cloud" with: credentials_json: ${{ secrets.FDC_CORE_CI_IMAGE_READER }} + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' - name: gcloud authorize run: | gcloud auth configure-docker diff --git a/pkg/db/db.go b/pkg/db/db.go index 2e5307628..341c85749 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -555,7 +555,7 @@ type DBAllReleasesWithMetaData struct { func (h *DBHandler) DBSelectAnyRelease(ctx context.Context, tx *sql.Tx, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { selectQuery := h.AdaptQuery(fmt.Sprintf( - "SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted, environments " + + "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + " FROM releases " + " LIMIT 1;")) span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectAnyRelease") @@ -566,7 +566,7 @@ func (h *DBHandler) DBSelectAnyRelease(ctx context.Context, tx *sql.Tx, ignorePr ctx, selectQuery, ) - processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes) + processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) if err != nil { return nil, err } @@ -608,13 +608,79 @@ func (h *DBHandler) DBSelectReleasesWithoutEnvironments(ctx context.Context, tx LIMIT 100; `) rows, err := tx.QueryContext(ctx, selectQuery) - processedRows, err := h.processReleaseRows(ctx, err, rows, true) + processedRows, err := h.processReleaseRows(ctx, err, rows, true, true) if err != nil { return nil, err } return processedRows, nil } +func (h *DBHandler) DBSelectReleasesByVersions(ctx context.Context, tx *sql.Tx, app string, releaseVersions []uint64, ignorePrePublishes bool) ([]*DBReleaseWithMetaData, error) { + span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByVersions") + defer span.Finish() + + if len(releaseVersions) == 0 { + return []*DBReleaseWithMetaData{}, nil + } + repeatedQuestionMarks := strings.Repeat(",?", len(releaseVersions)-1) + selectQuery := h.AdaptQuery(` + + SELECT DISTINCT + releases.eslVersion, + releases.created, + releases.appName, + releases.metadata, + releases.releaseVersion, + releases.deleted, + releases.environments + FROM + ( SELECT + MAX(eslVersion) AS latestEslVersion, + appname, + releaseversion + FROM + "releases" + WHERE + appname=? + AND + releaseversion IN (?` + repeatedQuestionMarks + `) + GROUP BY + appname, releaseversion + ) AS latest + JOIN + releases AS releases + ON + latest.latestEslVersion=releases.eslVersion + AND latest.releaseVersion=releases.releaseVersion + AND latest.appname=releases.appname + LIMIT ? + `) + + span.SetTag("query", selectQuery) + + args := []any{} + args = append(args, app) + for _, version := range releaseVersions { + args = append(args, version) + } + args = append(args, uint64(len(releaseVersions))) + + rows, err := tx.QueryContext( + ctx, + selectQuery, + args..., + ) + + processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrePublishes, false) + if err != nil { + return nil, err + } + if len(processedRows) == 0 { + return nil, nil + } + return processedRows, nil +} + func (h *DBHandler) DBSelectReleaseByVersion(ctx context.Context, tx *sql.Tx, app string, releaseVersion uint64, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) { selectQuery := h.AdaptQuery(fmt.Sprintf( "SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted, environments " + @@ -632,7 +698,7 @@ func (h *DBHandler) DBSelectReleaseByVersion(ctx context.Context, tx *sql.Tx, ap releaseVersion, ) - processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes) + processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true) if err != nil { return nil, err } @@ -737,14 +803,14 @@ func (h *DBHandler) DBSelectReleasesByApp(ctx context.Context, tx *sql.Tx, app s deleted, ) - return h.processReleaseRows(ctx, err, rows, ignorePrepublishes) + return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true) } func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) ([]*DBReleaseWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByApp") defer span.Finish() selectQuery := h.AdaptQuery(fmt.Sprintf( - "SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted, environments " + + "SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " + " FROM releases " + " WHERE appName=? AND deleted=?" + " ORDER BY eslVersion DESC, releaseVersion DESC, created DESC;")) @@ -756,7 +822,7 @@ func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, t deleted, ) - return h.processReleaseRows(ctx, err, rows, ignorePrepublishes) + return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false) } func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) { @@ -887,7 +953,8 @@ func (h *DBHandler) processAllReleasesRow(ctx context.Context, err error, rows * } return row, nil } -func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql.Rows, ignorePrepublishes bool) ([]*DBReleaseWithMetaData, error) { + +func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql.Rows, ignorePrepublishes bool, withManifests bool) ([]*DBReleaseWithMetaData, error) { span, ctx := tracer.StartSpanFromContext(ctx, "processReleaseRows") defer span.Finish() @@ -909,12 +976,17 @@ func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql var metadataStr string var manifestStr string var environmentsStr sql.NullString - err := rows.Scan(&row.EslVersion, &row.Created, &row.App, &metadataStr, &manifestStr, &row.ReleaseNumber, &row.Deleted, &environmentsStr) + var err error + if withManifests { + err = rows.Scan(&row.EslVersion, &row.Created, &row.App, &metadataStr, &manifestStr, &row.ReleaseNumber, &row.Deleted, &environmentsStr) + } else { + err = rows.Scan(&row.EslVersion, &row.Created, &row.App, &metadataStr /*manifests*/, &row.ReleaseNumber, &row.Deleted, &environmentsStr) + } if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } - return nil, fmt.Errorf("Error scanning releases row from DB. Error: %w\n", err) + return nil, fmt.Errorf("Error scanning releases row from DB withManifests=%v. Error: %w\n", withManifests, err) } if row.ReleaseNumber != lastSeenRelease { lastSeenRelease = row.ReleaseNumber @@ -942,9 +1014,11 @@ func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql var manifestData = DBReleaseManifests{ Manifests: map[string]string{}, } - err = json.Unmarshal(([]byte)(manifestStr), &manifestData) - if err != nil { - return nil, fmt.Errorf("Error during json unmarshal of manifests for releases. Error: %w. Data: %s\n", err, metadataStr) + if withManifests { + err = json.Unmarshal(([]byte)(manifestStr), &manifestData) + if err != nil { + return nil, fmt.Errorf("Error during json unmarshal of manifests for releases. Error: %w. Data: %s\n", err, metadataStr) + } } row.Manifests = manifestData environments := make([]string, 0) diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index 50712cd8e..99cf9cda5 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -2079,6 +2079,9 @@ func (s *State) GetEnvironmentApplicationsFromDB(ctx context.Context, transactio if err != nil { return nil, err } + if envInfo == nil { + return nil, fmt.Errorf("environment %s not found", environment) + } if envInfo.Applications == nil { return make([]string, 0), nil } @@ -2512,16 +2515,15 @@ func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sq } rels = retrievedReleasesOfApp } - for _, id := range rels { - if rel, err := s.GetApplicationRelease(ctx, transaction, appName, id); err != nil { - return err - } else { + + if releasesInDb, err := s.GetApplicationReleasesDB(ctx, transaction, appName, rels); err != nil { + return err + } else { + for _, rel := range releasesInDb { if rel == nil { // ignore } else { release := rel.ToProto() - release.Version = id - release.UndeployVersion = rel.UndeployVersion app.Releases = append(app.Releases, release) } } @@ -3119,6 +3121,42 @@ func (s *State) IsUndeployVersionFromManifest(application string, version uint64 return true, nil } +func (s *State) GetApplicationReleasesDB(ctx context.Context, transaction *sql.Tx, application string, versions []uint64) ([]*Release, error) { + var result []*Release + if s.DBHandler.ShouldUseOtherTables() { + rels, err := s.DBHandler.DBSelectReleasesByVersions(ctx, transaction, application, versions, true) + if err != nil { + return nil, fmt.Errorf("could not get release of app %s: %v", application, err) + } + if rels == nil { + return nil, nil + } + for _, rel := range rels { + r := &Release{ + Version: rel.ReleaseNumber, + UndeployVersion: rel.Metadata.UndeployVersion, + SourceAuthor: rel.Metadata.SourceAuthor, + SourceCommitId: rel.Metadata.SourceCommitId, + SourceMessage: rel.Metadata.SourceMessage, + CreatedAt: rel.Created, + DisplayVersion: rel.Metadata.DisplayVersion, + IsMinor: rel.Metadata.IsMinor, + IsPrepublish: rel.Metadata.IsPrepublish, + } + result = append(result, r) + } + } else { + for i, v := range versions { + rel, err := s.GetApplicationRelease(ctx, transaction, application, v) + if err != nil { + return nil, fmt.Errorf("could not get release of app %s at index %d for version %v: %v", application, i, v, err) + } + result = append(result, rel) + } + } + return result, nil +} + func (s *State) GetApplicationRelease(ctx context.Context, transaction *sql.Tx, application string, version uint64) (*Release, error) { if s.DBHandler.ShouldUseOtherTables() { env, err := s.DBHandler.DBSelectReleaseByVersion(ctx, transaction, application, version, true) diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go index e4d4a613d..9996e2722 100644 --- a/services/cd-service/pkg/repository/transformer.go +++ b/services/cd-service/pkg/repository/transformer.go @@ -3388,7 +3388,7 @@ func getOverrideVersions(ctx context.Context, transaction *sql.Tx, commitHash, u return resp, nil } -func (c *ReleaseTrain) getUpstreamLatestApp(ctx context.Context, transaction *sql.Tx, upstreamLatest bool, state *State, upstreamEnvName, source, commitHash string) (apps []string, appVersions []Overview, err error) { +func (c *ReleaseTrain) getUpstreamLatestApp(ctx context.Context, transaction *sql.Tx, upstreamLatest bool, state *State, upstreamEnvName, source, commitHash string, targetEnv string) (apps []string, appVersions []Overview, err error) { if commitHash != "" { appVersions, err := getOverrideVersions(ctx, transaction, c.CommitHash, upstreamEnvName, c.Repo) if err != nil { @@ -3409,7 +3409,9 @@ func (c *ReleaseTrain) getUpstreamLatestApp(ctx context.Context, transaction *sq return apps, appVersions, nil } if upstreamLatest { - apps, err = state.GetApplications(ctx, transaction) + // For "upstreamlatest" we cannot get the source environment, because it's not a real environment + // but since we only care about the names of the apps, we can just get the apps for the target env. + apps, err = state.GetEnvironmentApplications(ctx, transaction, targetEnv) if err != nil { return nil, nil, grpc.PublicError(ctx, fmt.Errorf("could not get all applications for %q: %w", source, err)) } @@ -3710,7 +3712,7 @@ func (c *envReleaseTrain) prognosis( source = "latest" } - apps, overrideVersions, err := c.Parent.getUpstreamLatestApp(ctx, transaction, upstreamLatest, state, upstreamEnvName, source, c.Parent.CommitHash) + apps, overrideVersions, err := c.Parent.getUpstreamLatestApp(ctx, transaction, upstreamLatest, state, upstreamEnvName, source, c.Parent.CommitHash, c.Env) if err != nil { return ReleaseTrainEnvironmentPrognosis{ SkipCause: nil, @@ -3820,6 +3822,10 @@ func (c *envReleaseTrain) prognosis( } } else if upstreamLatest { l := len(allLatestReleases[appName]) + if allLatestReleases == nil || allLatestReleases[appName] == nil || l == 0 { + logger.FromContext(ctx).Sugar().Warnf("app %s appears to have no releases on env=%s, so it is skipped", appName, c.Env) + continue + } versionToDeploy = uint64(allLatestReleases[appName][int(math.Max(0, float64(l-1)))]) } else { upstreamVersion := allLatestDeploymentsUpstreamEnv[appName] diff --git a/services/cd-service/pkg/service/git_test.go b/services/cd-service/pkg/service/git_test.go index bb195d688..564014a34 100644 --- a/services/cd-service/pkg/service/git_test.go +++ b/services/cd-service/pkg/service/git_test.go @@ -17,6 +17,7 @@ Copyright freiheit.com*/ package service import ( + "context" "fmt" "sort" "testing" @@ -237,7 +238,7 @@ func TestGetProductOverview(t *testing.T) { if err != nil { t.Fatalf("error setting up repository test: %v", err) } - sv := &GitServer{OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown}} + sv := &GitServer{OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown, Context: context.Background()}} for _, transformer := range tc.Setup { repo.Apply(testutil.MakeTestContext(), transformer) @@ -831,7 +832,7 @@ func TestGetCommitInfo(t *testing.T) { DBHandler: repo.State().DBHandler, } sv := &GitServer{ - OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown}, + OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown, Context: context.Background()}, Config: config, PageSize: uint64(pageSize), } diff --git a/services/cd-service/pkg/service/overview_test.go b/services/cd-service/pkg/service/overview_test.go index 56b602aa7..d949f1018 100644 --- a/services/cd-service/pkg/service/overview_test.go +++ b/services/cd-service/pkg/service/overview_test.go @@ -640,6 +640,7 @@ func TestOverviewService(t *testing.T) { svc := &OverviewServiceServer{ Repository: repo, Shutdown: shutdown, + Context: context.Background(), } tc.Test(t, svc) if tc.DB { @@ -972,6 +973,7 @@ func TestOverviewServiceFromCommit(t *testing.T) { svc := &OverviewServiceServer{ Repository: repo, Shutdown: shutdown, + Context: context.Background(), } ov, err := svc.GetOverview(testutil.MakeTestContext(), &api.GetOverviewRequest{}) diff --git a/services/frontend-service/package.json b/services/frontend-service/package.json index 4143a80f2..e72539359 100644 --- a/services/frontend-service/package.json +++ b/services/frontend-service/package.json @@ -37,8 +37,8 @@ "build": "GENERATE_SOURCEMAP=false react-scripts build", "test": "TC=Europe/Berlin react-scripts test --color", "start": "WATCHPACK_POLLING=true CHOKIDAR_USEPOLLING=true react-scripts start", - "eslint-check": "eslint --cache --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/", - "eslint-fix": "eslint --fix --cache --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/", + "eslint-check": "eslint --cache --max-warnings 0 --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/", + "eslint-fix": "eslint --fix --cache --max-warnings 0 --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/", "circular-check": "madge -c --extensions ts,tsx --ts-config tsconfig.json --no-spinner src/", "scss-check": "prettier -c --parser scss 'src/**/*.scss'", "scss-fix": "prettier --write --parser scss 'src/**/*.scss'" diff --git a/services/frontend-service/pkg/service/batch.go b/services/frontend-service/pkg/service/batch.go index e4de64fef..cb7a77b77 100644 --- a/services/frontend-service/pkg/service/batch.go +++ b/services/frontend-service/pkg/service/batch.go @@ -47,7 +47,7 @@ func (b *BatchServiceWithDefaultTimeout) ProcessBatch(ctx context.Context, req * if context.Cause(ctx) == kuberpultTimeoutError { logger.FromContext(ctx).Warn("Context cancelled due to kuberpult timeout") } else { - logger.FromContext(ctx).Warn("Context cancelled due %v", zap.Error(context.Cause(ctx))) + logger.FromContext(ctx).Warn("Context cancelled due", zap.Error(context.Cause(ctx))) } }