feat: migration of clean old application version transformer to expor…

…t mainfest service (#1692) Migration of CleanOldApplicationVersion transformer to database and export manifest service --------- Co-authored-by: Sven Urbanski <[email protected]>
freiheit-com · Jun 25, 2024 · ea8f05b · ea8f05b
1 parent 84c5a7f
commit ea8f05b
Show file tree

Hide file tree

Showing 12 changed files with 602 additions and 58 deletions.
diff --git a/database/migrations/sqlite/1717749007994908_releases.up.sql b/database/migrations/sqlite/1717749007994908_releases.up.sql
@@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS releases
     appName VARCHAR,
     manifests VARCHAR, -- json blob, map where each key is an environment and each value is a manifest
     metadata VARCHAR,  -- json blob about the release, sourceAuthor, sourceCommitId, etc
+    deleted  BOOLEAN,
     PRIMARY KEY(eslVersion, releaseVersion, appName)
 );
 

diff --git a/docker-compose-earthly.yml b/docker-compose-earthly.yml
@@ -44,6 +44,7 @@ services:
       - KUBERPULT_ENABLE_METRICS=false
       - KUBERPULT_DOGSTATSD_ADDR=127.0.0.1:8125
       - KUBERPULT_ARGO_CD_GENERATE_FILES=true
+      - KUBERPULT_RELEASE_VERSIONS_LIMIT=20
     volumes:
       - ./services/cd-service:/kp/kuberpult
       - ./database:/kp/database

diff --git a/pkg/db/db.go b/pkg/db/db.go
@@ -453,6 +453,7 @@ type DBReleaseWithMetaData struct {
 	App           string
 	Manifests     DBReleaseManifests
 	Metadata      DBReleaseMetaData
+	Deleted       bool
 }
 
 type DBAllReleaseMetaData struct {
@@ -467,7 +468,7 @@ type DBAllReleasesWithMetaData struct {
 
 func (h *DBHandler) DBSelectAnyRelease(ctx context.Context, tx *sql.Tx) (*DBReleaseWithMetaData, error) {
 	selectQuery := h.AdaptQuery(fmt.Sprintf(
-		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion " +
+		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted " +
 			" FROM releases " +
 			" LIMIT 1;"))
 	rows, err := tx.QueryContext(
@@ -479,10 +480,10 @@ func (h *DBHandler) DBSelectAnyRelease(ctx context.Context, tx *sql.Tx) (*DBRele
 
 func (h *DBHandler) DBSelectReleaseByVersion(ctx context.Context, tx *sql.Tx, app string, releaseVersion uint64) (*DBReleaseWithMetaData, error) {
 	selectQuery := h.AdaptQuery(fmt.Sprintf(
-		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion " +
+		"SELECT eslVersion, created, appName, metadata, manifests, releaseVersion, deleted " +
 			" FROM releases " +
 			" WHERE appName=? AND releaseVersion=?" +
-			" ORDER BY eslVersion ASC " +
+			" ORDER BY eslVersion DESC " +
 			" LIMIT 1;"))
 	rows, err := tx.QueryContext(
 		ctx,
@@ -562,7 +563,7 @@ func (h *DBHandler) processReleaseRow(ctx context.Context, err error, rows *sql.
 	if rows.Next() {
 		var metadataStr string
 		var manifestStr string
-		err := rows.Scan(&row.EslId, &row.Created, &row.App, &metadataStr, &manifestStr, &row.ReleaseNumber)
+		err := rows.Scan(&row.EslId, &row.Created, &row.App, &metadataStr, &manifestStr, &row.ReleaseNumber, &row.Deleted)
 		if err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return nil, nil
@@ -610,7 +611,7 @@ func (h *DBHandler) DBInsertRelease(ctx context.Context, transaction *sql.Tx, re
 		return fmt.Errorf("insert release: could not marshal json data: %w", err)
 	}
 	insertQuery := h.AdaptQuery(
-		"INSERT INTO releases (eslVersion, created, releaseVersion, appName, manifests, metadata)  VALUES (?, ?, ?, ?, ?, ?);",
+		"INSERT INTO releases (eslVersion, created, releaseVersion, appName, manifests, metadata, deleted)  VALUES (?, ?, ?, ?, ?, ?, ?);",
 	)
 	span.SetTag("query", insertQuery)
 	manifestJson, err := json.Marshal(release.Manifests)
@@ -626,6 +627,7 @@ func (h *DBHandler) DBInsertRelease(ctx context.Context, transaction *sql.Tx, re
 		release.App,
 		manifestJson,
 		metadataJson,
+		release.Deleted,
 	)
 	if err != nil {
 		return fmt.Errorf(
@@ -643,6 +645,53 @@ func (h *DBHandler) DBInsertRelease(ctx context.Context, transaction *sql.Tx, re
 	return nil
 }
 
+func (h *DBHandler) DBDeleteReleaseFromAllReleases(ctx context.Context, transaction *sql.Tx, application string, releaseToDelete uint64) error {
+	span, _ := tracer.StartSpanFromContext(ctx, "DBDeleteReleaseFromAllReleases")
+	defer span.Finish()
+
+	allReleases, err := h.DBSelectAllReleasesOfApp(ctx, transaction, application)
+	if err != nil {
+		return err
+	}
+	if allReleases == nil {
+		logger.FromContext(ctx).Sugar().Infof("Could not find release '%d' for appliation '%s' to delete. No releases found.", releaseToDelete, application)
+		return nil
+	}
+	idxToDelete := slices.Index(allReleases.Metadata.Releases, int64(releaseToDelete))
+
+	if idxToDelete == -1 {
+		logger.FromContext(ctx).Sugar().Infof("Could not find release '%d' for appliation '%s' to delete.", releaseToDelete, application)
+		return nil //If we don't find it, not an error, but we do nothing
+	}
+	allReleases.Metadata.Releases = append(allReleases.Metadata.Releases[:idxToDelete], allReleases.Metadata.Releases[idxToDelete+1:]...)
+	if err := h.DBInsertAllReleases(ctx, transaction, application, allReleases.Metadata.Releases, allReleases.EslId); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (h *DBHandler) DBDeleteFromReleases(ctx context.Context, transaction *sql.Tx, application string, releaseToDelete uint64) error {
+	span, _ := tracer.StartSpanFromContext(ctx, "DBDeleteFromReleases")
+	defer span.Finish()
+
+	targetRelease, err := h.DBSelectReleaseByVersion(ctx, transaction, application, releaseToDelete)
+	if err != nil {
+		return err
+	}
+
+	if targetRelease.Deleted {
+		logger.FromContext(ctx).Sugar().Infof("Release '%d' for application '%s' has already been deleted.", releaseToDelete, application)
+		return nil
+	}
+
+	targetRelease.Deleted = true
+	if err := h.DBInsertRelease(ctx, transaction, *targetRelease, targetRelease.EslId); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (h *DBHandler) DBInsertAllReleases(ctx context.Context, transaction *sql.Tx, app string, allVersions []int64, previousEslVersion EslId) error {
 	span, _ := tracer.StartSpanFromContext(ctx, "DBInsertRelease")
 	defer span.Finish()
@@ -1300,6 +1349,7 @@ func (h *DBHandler) RunCustomMigrationReleases(ctx context.Context, getAllAppsFu
 						SourceMessage:  repoRelease.SourceMessage,
 						DisplayVersion: repoRelease.DisplayVersion,
 					},
+					Deleted: false,
 				}
 				err = h.DBInsertRelease(ctx, transaction, dbRelease, InitialEslId-1)
 				if err != nil {

diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go
@@ -1241,3 +1241,101 @@ func TestDeleteTeamLock(t *testing.T) {
 		})
 	}
 }
+
+func TestDeleteRelease(t *testing.T) {
+	tcs := []struct {
+		Name     string
+		toInsert DBReleaseWithMetaData
+		expected DBReleaseWithMetaData
+	}{
+		{
+			Name: "Delete Release from database",
+			toInsert: DBReleaseWithMetaData{
+				EslId:         InitialEslId,
+				Created:       time.Now(),
+				ReleaseNumber: 1,
+				App:           "app",
+				Manifests: DBReleaseManifests{
+					Manifests: map[string]string{"development": "development"},
+				},
+				Metadata: DBReleaseMetaData{
+					SourceAuthor:   "me",
+					SourceCommitId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+					SourceMessage:  "message",
+					DisplayVersion: "1.0.0",
+				},
+				Deleted: false,
+			},
+			expected: DBReleaseWithMetaData{
+				EslId:         InitialEslId + 1,
+				Created:       time.Now(),
+				ReleaseNumber: 1,
+				App:           "app",
+				Manifests: DBReleaseManifests{
+					Manifests: map[string]string{"development": "development"},
+				},
+				Metadata: DBReleaseMetaData{
+					SourceAuthor:   "me",
+					SourceCommitId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+					SourceMessage:  "message",
+					DisplayVersion: "1.0.0",
+				},
+				Deleted: true,
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		tc := tc
+		t.Run(tc.Name, func(t *testing.T) {
+			t.Parallel()
+			ctx := testutil.MakeTestContext()
+
+			dbHandler := setupDB(t)
+			err := dbHandler.WithTransaction(ctx, func(ctx context.Context, transaction *sql.Tx) error {
+				err2 := dbHandler.DBInsertRelease(ctx, transaction, tc.toInsert, tc.toInsert.EslId-1)
+				if err2 != nil {
+					return err2
+				}
+				err2 = dbHandler.DBInsertAllReleases(ctx, transaction, tc.toInsert.App, []int64{int64(tc.toInsert.ReleaseNumber)}, tc.toInsert.EslId-1)
+				if err2 != nil {
+					return err2
+				}
+
+				errDelete := dbHandler.DBDeleteFromReleases(ctx, transaction, tc.toInsert.App, tc.toInsert.ReleaseNumber)
+				if errDelete != nil {
+					t.Fatalf("error: %v\n", errDelete)
+				}
+
+				errDelete2 := dbHandler.DBDeleteReleaseFromAllReleases(ctx, transaction, tc.toInsert.App, tc.toInsert.ReleaseNumber)
+				if errDelete2 != nil {
+					return errDelete2
+				}
+
+				allReleases, err := dbHandler.DBSelectAllReleasesOfApp(ctx, transaction, tc.toInsert.App)
+				if err != nil {
+					return err
+				}
+				if len(allReleases.Metadata.Releases) != 0 {
+					t.Fatalf("number of team locks mismatch (-want, +got):\n%d", len(allReleases.Metadata.Releases))
+				}
+
+				latestRelease, err := dbHandler.DBSelectReleaseByVersion(ctx, transaction, tc.toInsert.App, tc.toInsert.ReleaseNumber)
+				if err != nil {
+					return err
+				}
+				if !latestRelease.Deleted {
+					t.Fatalf("Release has not beed deleted\n")
+				}
+
+				if diff := cmp.Diff(&tc.expected, latestRelease, cmpopts.IgnoreFields(DBReleaseWithMetaData{}, "Created")); diff != "" {
+					t.Fatalf("team locks mismatch (-want, +got):\n%s", diff)
+				}
+				return nil
+			})
+			if err != nil {
+				t.Fatalf("transaction error: %v", err)
+			}
+		})
+	}
+}
diff --git a/services/cd-service/pkg/repository/transformer.go b/services/cd-service/pkg/repository/transformer.go
@@ -549,8 +549,8 @@ func (c *CreateApplicationVersion) Transform(
 		if err := util.WriteFile(fs, fs.Join(releaseDir, fieldCreatedAt), []byte(time2.GetTimeNow(ctx).Format(time.RFC3339)), 0666); err != nil {
 			return "", GetCreateReleaseGeneralFailure(err)
 		}
-
 	}
+
 	if c.Team != "" && !state.DBHandler.ShouldUseOtherTables() {
 		//util.WriteFile has a bug where it does not truncate the old file content. If two application versions with the same
 		//team are deployed, team names simply get concatenated. Just remove the file beforehand.
@@ -625,6 +625,7 @@ func (c *CreateApplicationVersion) Transform(
 				DisplayVersion: c.DisplayVersion,
 			},
 			Created: time.Now(),
+			Deleted: false,
 		}
 		err = state.DBHandler.DBInsertRelease(ctx, transaction, release, v)
 		if err != nil {
@@ -1390,7 +1391,7 @@ func findOldApplicationVersions(ctx context.Context, transaction *sql.Tx, state
 	if err != nil {
 		return nil, err
 	}
-	versions, err := state.GetAllApplicationReleasesFromManifest(name)
+	versions, err := state.GetAllApplicationReleases(ctx, transaction, name)
 	if err != nil {
 		return nil, err
 	}
@@ -1437,32 +1438,44 @@ func (c *CleanupOldApplicationVersions) Transform(
 
 	msg := ""
 	for _, oldRelease := range oldVersions {
-		// delete oldRelease:
-		releasesDir := releasesDirectoryWithVersion(fs, c.Application, oldRelease)
-		_, err := fs.Stat(releasesDir)
-		if err != nil {
-			return "", wrapFileError(err, releasesDir, "CleanupOldApplicationVersions: could not stat")
-		}
-
-		{
-			commitIDFile := fs.Join(releasesDir, fieldSourceCommitId)
-			dat, err := util.ReadFile(fs, commitIDFile)
+		if state.DBHandler.ShouldUseOtherTables() {
+			// delete release from all_releases
+			if err := state.DBHandler.DBDeleteReleaseFromAllReleases(ctx, transaction, c.Application, oldRelease); err != nil {
+				return "", err
+			}
+			//'Delete' from releases table
+			if err := state.DBHandler.DBDeleteFromReleases(ctx, transaction, c.Application, oldRelease); err != nil {
+				return "", err
+			}
+		} else {
+			// delete oldRelease:
+			releasesDir := releasesDirectoryWithVersion(fs, c.Application, oldRelease)
+			_, err := fs.Stat(releasesDir)
 			if err != nil {
-				// not a problem, might be the undeploy commit or the commit has was not specified in CreateApplicationVersion
-			} else {
-				commitID := string(dat)
-				if valid.SHA1CommitID(commitID) {
-					if err := removeCommit(fs, commitID, c.Application); err != nil {
-						return "", wrapFileError(err, releasesDir, "CleanupOldApplicationVersions: could not remove commit path")
+				return "", wrapFileError(err, releasesDir, "CleanupOldApplicationVersions: could not stat")
+			}
+
+			{
+				commitIDFile := fs.Join(releasesDir, fieldSourceCommitId)
+				dat, err := util.ReadFile(fs, commitIDFile)
+				if err != nil {
+					// not a problem, might be the undeploy commit or the commit has was not specified in CreateApplicationVersion
+				} else {
+					commitID := string(dat)
+					if valid.SHA1CommitID(commitID) {
+						if err := removeCommit(fs, commitID, c.Application); err != nil {
+							return "", wrapFileError(err, releasesDir, "CleanupOldApplicationVersions: could not remove commit path")
+						}
 					}
 				}
 			}
-		}
 
-		err = fs.Remove(releasesDir)
-		if err != nil {
-			return "", fmt.Errorf("CleanupOldApplicationVersions: Unexpected error app %s: %w",
-				c.Application, err)
+			err = fs.Remove(releasesDir)
+			if err != nil {
+				return "", fmt.Errorf("CleanupOldApplicationVersions: Unexpected error app %s: %w",
+					c.Application, err)
+			}
+			msg = fmt.Sprintf("%sremoved version %d of app %v as cleanup\n", msg, oldRelease, c.Application)
 		}
 		msg = fmt.Sprintf("%sremoved version %d of app %v as cleanup\n", msg, oldRelease, c.Application)
 	}
@@ -2471,10 +2484,10 @@ func (c *DeployApplicationVersion) Transform(
 	d := &CleanupOldApplicationVersions{
 		Application: c.Application,
 	}
+
 	if err := t.Execute(d, transaction); err != nil {
 		return "", err
 	}
-
 	if c.WriteCommitData { // write the corresponding event
 		deploymentEvent := createDeploymentEvent(c.Application, c.Environment, c.SourceTrain)
 		if s.DBHandler.ShouldUseOtherTables() {