Merge branch 'main' into dn/manually_delete_envs

Merged with main to simplify pull request.

.github/workflows/execution-plan-snippet-earthly.yml

@@ -20,6 +20,10 @@ jobs:
         name: "Authenticate to Google Cloud"
         with:
           credentials_json: ${{ secrets.FDC_CORE_CI_IMAGE_READER }}
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v2'
+        with:
+          version: '>= 363.0.0'
       - name: gcloud authorize
         run: |
           gcloud auth configure-docker

pkg/db/db.go

@@ -555,7 +555,7 @@ type DBAllReleasesWithMetaData struct {
 
 func (h *DBHandler) DBSelectAnyRelease(ctx context.Context, tx *sql.Tx, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) {
 	selectQuery := h.AdaptQuery(fmt.Sprintf(
-		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted, environments " +
+		"SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " +
 			" FROM releases " +
 			" LIMIT 1;"))
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectAnyRelease")
@@ -566,7 +566,7 @@ func (h *DBHandler) DBSelectAnyRelease(ctx context.Context, tx *sql.Tx, ignorePr
 		ctx,
 		selectQuery,
 	)
-	processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes)
+	processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false)
 	if err != nil {
 		return nil, err
 	}
@@ -608,13 +608,79 @@ func (h *DBHandler) DBSelectReleasesWithoutEnvironments(ctx context.Context, tx
 	LIMIT 100;
 	`)
 	rows, err := tx.QueryContext(ctx, selectQuery)
-	processedRows, err := h.processReleaseRows(ctx, err, rows, true)
+	processedRows, err := h.processReleaseRows(ctx, err, rows, true, true)
 	if err != nil {
 		return nil, err
 	}
 	return processedRows, nil
 }
 
+func (h *DBHandler) DBSelectReleasesByVersions(ctx context.Context, tx *sql.Tx, app string, releaseVersions []uint64, ignorePrePublishes bool) ([]*DBReleaseWithMetaData, error) {
+	span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByVersions")
+	defer span.Finish()
+
+	if len(releaseVersions) == 0 {
+		return []*DBReleaseWithMetaData{}, nil
+	}
+	repeatedQuestionMarks := strings.Repeat(",?", len(releaseVersions)-1)
+	selectQuery := h.AdaptQuery(`
+
+	SELECT DISTINCT
+		releases.eslVersion,
+		releases.created,
+		releases.appName,
+		releases.metadata,
+		releases.releaseVersion,
+		releases.deleted,
+		releases.environments
+	FROM 
+		( 	SELECT
+				MAX(eslVersion) AS latestEslVersion,
+				appname,
+				releaseversion
+			FROM
+				"releases"
+    		WHERE
+				appname=?
+          	AND
+				releaseversion IN (?` + repeatedQuestionMarks + `)
+		GROUP BY
+			appname, releaseversion
+      ) AS latest
+	JOIN
+		releases AS releases
+	ON
+		latest.latestEslVersion=releases.eslVersion
+		AND latest.releaseVersion=releases.releaseVersion
+		AND latest.appname=releases.appname
+	LIMIT ?
+	`)
+
+	span.SetTag("query", selectQuery)
+
+	args := []any{}
+	args = append(args, app)
+	for _, version := range releaseVersions {
+		args = append(args, version)
+	}
+	args = append(args, uint64(len(releaseVersions)))
+
+	rows, err := tx.QueryContext(
+		ctx,
+		selectQuery,
+		args...,
+	)
+
+	processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrePublishes, false)
+	if err != nil {
+		return nil, err
+	}
+	if len(processedRows) == 0 {
+		return nil, nil
+	}
+	return processedRows, nil
+}
+
 func (h *DBHandler) DBSelectReleaseByVersion(ctx context.Context, tx *sql.Tx, app string, releaseVersion uint64, ignorePrepublishes bool) (*DBReleaseWithMetaData, error) {
 	selectQuery := h.AdaptQuery(fmt.Sprintf(
 		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted, environments " +
@@ -632,7 +698,7 @@ func (h *DBHandler) DBSelectReleaseByVersion(ctx context.Context, tx *sql.Tx, ap
 		releaseVersion,
 	)
 
-	processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes)
+	processedRows, err := h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true)
 	if err != nil {
 		return nil, err
 	}
@@ -737,14 +803,14 @@ func (h *DBHandler) DBSelectReleasesByApp(ctx context.Context, tx *sql.Tx, app s
 		deleted,
 	)
 
-	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes)
+	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, true)
 }
 
 func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, tx *sql.Tx, app string, deleted bool, ignorePrepublishes bool) ([]*DBReleaseWithMetaData, error) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "DBSelectReleasesByApp")
 	defer span.Finish()
 	selectQuery := h.AdaptQuery(fmt.Sprintf(
-		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted, environments " +
+		"SELECT eslVersion, created, appName, metadata, releaseVersion, deleted, environments " +
 			" FROM releases " +
 			" WHERE appName=? AND deleted=?" +
 			" ORDER BY eslVersion DESC, releaseVersion DESC, created DESC;"))
@@ -756,7 +822,7 @@ func (h *DBHandler) DBSelectReleasesByAppLatestEslVersion(ctx context.Context, t
 		deleted,
 	)
 
-	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes)
+	return h.processReleaseRows(ctx, err, rows, ignorePrepublishes, false)
 }
 
 func (h *DBHandler) DBSelectAllReleasesOfApp(ctx context.Context, tx *sql.Tx, app string) (*DBAllReleasesWithMetaData, error) {
@@ -887,7 +953,8 @@ func (h *DBHandler) processAllReleasesRow(ctx context.Context, err error, rows *
 	}
 	return row, nil
 }
-func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql.Rows, ignorePrepublishes bool) ([]*DBReleaseWithMetaData, error) {
+
+func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql.Rows, ignorePrepublishes bool, withManifests bool) ([]*DBReleaseWithMetaData, error) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "processReleaseRows")
 	defer span.Finish()
 
@@ -909,12 +976,17 @@ func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql
 		var metadataStr string
 		var manifestStr string
 		var environmentsStr sql.NullString
-		err := rows.Scan(&row.EslVersion, &row.Created, &row.App, &metadataStr, &manifestStr, &row.ReleaseNumber, &row.Deleted, &environmentsStr)
+		var err error
+		if withManifests {
+			err = rows.Scan(&row.EslVersion, &row.Created, &row.App, &metadataStr, &manifestStr, &row.ReleaseNumber, &row.Deleted, &environmentsStr)
+		} else {
+			err = rows.Scan(&row.EslVersion, &row.Created, &row.App, &metadataStr /*manifests*/, &row.ReleaseNumber, &row.Deleted, &environmentsStr)
+		}
 		if err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return nil, nil
 			}
-			return nil, fmt.Errorf("Error scanning releases row from DB. Error: %w\n", err)
+			return nil, fmt.Errorf("Error scanning releases row from DB withManifests=%v. Error: %w\n", withManifests, err)
 		}
 		if row.ReleaseNumber != lastSeenRelease {
 			lastSeenRelease = row.ReleaseNumber
@@ -942,9 +1014,11 @@ func (h *DBHandler) processReleaseRows(ctx context.Context, err error, rows *sql
 		var manifestData = DBReleaseManifests{
 			Manifests: map[string]string{},
 		}
-		err = json.Unmarshal(([]byte)(manifestStr), &manifestData)
-		if err != nil {
-			return nil, fmt.Errorf("Error during json unmarshal of manifests for releases. Error: %w. Data: %s\n", err, metadataStr)
+		if withManifests {
+			err = json.Unmarshal(([]byte)(manifestStr), &manifestData)
+			if err != nil {
+				return nil, fmt.Errorf("Error during json unmarshal of manifests for releases. Error: %w. Data: %s\n", err, metadataStr)
+			}
 		}
 		row.Manifests = manifestData
 		environments := make([]string, 0)

services/cd-service/pkg/repository/repository.go

@@ -2079,6 +2079,9 @@ func (s *State) GetEnvironmentApplicationsFromDB(ctx context.Context, transactio
 	if err != nil {
 		return nil, err
 	}
+	if envInfo == nil {
+		return nil, fmt.Errorf("environment %s not found", environment)
+	}
 	if envInfo.Applications == nil {
 		return make([]string, 0), nil
 	}
@@ -2512,16 +2515,15 @@ func (s *State) UpdateTopLevelAppInOverview(ctx context.Context, transaction *sq
 		}
 		rels = retrievedReleasesOfApp
 	}
-	for _, id := range rels {
-		if rel, err := s.GetApplicationRelease(ctx, transaction, appName, id); err != nil {
-			return err
-		} else {
+
+	if releasesInDb, err := s.GetApplicationReleasesDB(ctx, transaction, appName, rels); err != nil {
+		return err
+	} else {
+		for _, rel := range releasesInDb {
 			if rel == nil {
 				// ignore
 			} else {
 				release := rel.ToProto()
-				release.Version = id
-				release.UndeployVersion = rel.UndeployVersion
 				app.Releases = append(app.Releases, release)
 			}
 		}
@@ -3119,6 +3121,42 @@ func (s *State) IsUndeployVersionFromManifest(application string, version uint64
 	return true, nil
 }
 
+func (s *State) GetApplicationReleasesDB(ctx context.Context, transaction *sql.Tx, application string, versions []uint64) ([]*Release, error) {
+	var result []*Release
+	if s.DBHandler.ShouldUseOtherTables() {
+		rels, err := s.DBHandler.DBSelectReleasesByVersions(ctx, transaction, application, versions, true)
+		if err != nil {
+			return nil, fmt.Errorf("could not get release of app %s: %v", application, err)
+		}
+		if rels == nil {
+			return nil, nil
+		}
+		for _, rel := range rels {
+			r := &Release{
+				Version:         rel.ReleaseNumber,
+				UndeployVersion: rel.Metadata.UndeployVersion,
+				SourceAuthor:    rel.Metadata.SourceAuthor,
+				SourceCommitId:  rel.Metadata.SourceCommitId,
+				SourceMessage:   rel.Metadata.SourceMessage,
+				CreatedAt:       rel.Created,
+				DisplayVersion:  rel.Metadata.DisplayVersion,
+				IsMinor:         rel.Metadata.IsMinor,
+				IsPrepublish:    rel.Metadata.IsPrepublish,
+			}
+			result = append(result, r)
+		}
+	} else {
+		for i, v := range versions {
+			rel, err := s.GetApplicationRelease(ctx, transaction, application, v)
+			if err != nil {
+				return nil, fmt.Errorf("could not get release of app %s at index %d for version %v: %v", application, i, v, err)
+			}
+			result = append(result, rel)
+		}
+	}
+	return result, nil
+}
+
 func (s *State) GetApplicationRelease(ctx context.Context, transaction *sql.Tx, application string, version uint64) (*Release, error) {
 	if s.DBHandler.ShouldUseOtherTables() {
 		env, err := s.DBHandler.DBSelectReleaseByVersion(ctx, transaction, application, version, true)

services/cd-service/pkg/repository/transformer.go

@@ -3388,7 +3388,7 @@ func getOverrideVersions(ctx context.Context, transaction *sql.Tx, commitHash, u
 	return resp, nil
 }
 
-func (c *ReleaseTrain) getUpstreamLatestApp(ctx context.Context, transaction *sql.Tx, upstreamLatest bool, state *State, upstreamEnvName, source, commitHash string) (apps []string, appVersions []Overview, err error) {
+func (c *ReleaseTrain) getUpstreamLatestApp(ctx context.Context, transaction *sql.Tx, upstreamLatest bool, state *State, upstreamEnvName, source, commitHash string, targetEnv string) (apps []string, appVersions []Overview, err error) {
 	if commitHash != "" {
 		appVersions, err := getOverrideVersions(ctx, transaction, c.CommitHash, upstreamEnvName, c.Repo)
 		if err != nil {
@@ -3409,7 +3409,9 @@ func (c *ReleaseTrain) getUpstreamLatestApp(ctx context.Context, transaction *sq
 		return apps, appVersions, nil
 	}
 	if upstreamLatest {
-		apps, err = state.GetApplications(ctx, transaction)
+		// For "upstreamlatest" we cannot get the source environment, because it's not a real environment
+		// but since we only care about the names of the apps, we can just get the apps for the target env.
+		apps, err = state.GetEnvironmentApplications(ctx, transaction, targetEnv)
 		if err != nil {
 			return nil, nil, grpc.PublicError(ctx, fmt.Errorf("could not get all applications for %q: %w", source, err))
 		}
@@ -3710,7 +3712,7 @@ func (c *envReleaseTrain) prognosis(
 		source = "latest"
 	}
 
-	apps, overrideVersions, err := c.Parent.getUpstreamLatestApp(ctx, transaction, upstreamLatest, state, upstreamEnvName, source, c.Parent.CommitHash)
+	apps, overrideVersions, err := c.Parent.getUpstreamLatestApp(ctx, transaction, upstreamLatest, state, upstreamEnvName, source, c.Parent.CommitHash, c.Env)
 	if err != nil {
 		return ReleaseTrainEnvironmentPrognosis{
 			SkipCause:     nil,
@@ -3820,6 +3822,10 @@ func (c *envReleaseTrain) prognosis(
 			}
 		} else if upstreamLatest {
 			l := len(allLatestReleases[appName])
+			if allLatestReleases == nil || allLatestReleases[appName] == nil || l == 0 {
+				logger.FromContext(ctx).Sugar().Warnf("app %s appears to have no releases on env=%s, so it is skipped", appName, c.Env)
+				continue
+			}
 			versionToDeploy = uint64(allLatestReleases[appName][int(math.Max(0, float64(l-1)))])
 		} else {
 			upstreamVersion := allLatestDeploymentsUpstreamEnv[appName]

services/cd-service/pkg/service/git_test.go

@@ -17,6 +17,7 @@ Copyright freiheit.com*/
 package service
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"testing"
@@ -237,7 +238,7 @@ func TestGetProductOverview(t *testing.T) {
 			if err != nil {
 				t.Fatalf("error setting up repository test: %v", err)
 			}
-			sv := &GitServer{OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown}}
+			sv := &GitServer{OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown, Context: context.Background()}}
 
 			for _, transformer := range tc.Setup {
 				repo.Apply(testutil.MakeTestContext(), transformer)
@@ -831,7 +832,7 @@ func TestGetCommitInfo(t *testing.T) {
 				DBHandler:           repo.State().DBHandler,
 			}
 			sv := &GitServer{
-				OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown},
+				OverviewService: &OverviewServiceServer{Repository: repo, Shutdown: shutdown, Context: context.Background()},
 				Config:          config,
 				PageSize:        uint64(pageSize),
 			}

services/cd-service/pkg/service/overview_test.go

@@ -640,6 +640,7 @@ func TestOverviewService(t *testing.T) {
 			svc := &OverviewServiceServer{
 				Repository: repo,
 				Shutdown:   shutdown,
+				Context:    context.Background(),
 			}
 			tc.Test(t, svc)
 			if tc.DB {
@@ -972,6 +973,7 @@ func TestOverviewServiceFromCommit(t *testing.T) {
 			svc := &OverviewServiceServer{
 				Repository: repo,
 				Shutdown:   shutdown,
+				Context:    context.Background(),
 			}
 
 			ov, err := svc.GetOverview(testutil.MakeTestContext(), &api.GetOverviewRequest{})

services/frontend-service/package.json

@@ -37,8 +37,8 @@
     "build": "GENERATE_SOURCEMAP=false react-scripts build",
     "test": "TC=Europe/Berlin react-scripts test --color",
     "start": "WATCHPACK_POLLING=true CHOKIDAR_USEPOLLING=true react-scripts start",
-    "eslint-check": "eslint --cache --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/",
-    "eslint-fix": "eslint --fix --cache --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/",
+    "eslint-check": "eslint --cache --max-warnings 0 --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/",
+    "eslint-fix": "eslint --fix --cache --max-warnings 0 --cache-location 'misc/.eslintcache' --ext .ts,.tsx src/",
     "circular-check": "madge -c --extensions ts,tsx --ts-config tsconfig.json --no-spinner src/",
     "scss-check": "prettier -c --parser scss 'src/**/*.scss'",
     "scss-fix": "prettier --write --parser scss 'src/**/*.scss'"

services/frontend-service/pkg/service/batch.go

@@ -47,7 +47,7 @@ func (b *BatchServiceWithDefaultTimeout) ProcessBatch(ctx context.Context, req *
 		if context.Cause(ctx) == kuberpultTimeoutError {
 			logger.FromContext(ctx).Warn("Context cancelled due to kuberpult timeout")
 		} else {
-			logger.FromContext(ctx).Warn("Context cancelled due %v", zap.Error(context.Cause(ctx)))
+			logger.FromContext(ctx).Warn("Context cancelled due", zap.Error(context.Cause(ctx)))
 		}
 	}