diff --git a/pkg/db/db.go b/pkg/db/db.go index 8b781f997..623a4f7a1 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -245,8 +245,13 @@ const ( // ESL EVENTS +type ESLMetadata struct { + AuthorName string `json:"authorName"` + AuthorEmail string `json:"authorEmail"` +} + // DBWriteEslEventInternal writes one event to the event-sourcing-light table, taking arbitrary data as input -func (h *DBHandler) DBWriteEslEventInternal(ctx context.Context, eventType EventType, tx *sql.Tx, data interface{}) error { +func (h *DBHandler) DBWriteEslEventInternal(ctx context.Context, eventType EventType, tx *sql.Tx, data interface{}, metadata ESLMetadata) error { if h == nil { return nil } @@ -256,9 +261,18 @@ func (h *DBHandler) DBWriteEslEventInternal(ctx context.Context, eventType Event span, _ := tracer.StartSpanFromContext(ctx, "DBWriteEslEventInternal") defer span.Finish() - jsonToInsert, err := json.Marshal(data) + dataMap, err := convertObjectToMap(data) if err != nil { - return fmt.Errorf("could not marshal json data: %w", err) + return fmt.Errorf("could not convert object to map: %w", err) + } + metadataMap, err := convertObjectToMap(metadata) + if err != nil { + return fmt.Errorf("could not convert object to map: %w", err) + } + dataMap["metadata"] = metadataMap + jsonToInsert, err := json.Marshal(dataMap) + if err != nil { + return fmt.Errorf("could not marshal combined json data: %w", err) } insertQuery := h.AdaptQuery("INSERT INTO event_sourcing_light (created, event_type , json) VALUES (?, ?, ?);") @@ -276,6 +290,22 @@ func (h *DBHandler) DBWriteEslEventInternal(ctx context.Context, eventType Event return nil } +func convertObjectToMap(obj interface{}) (map[string]interface{}, error) { + if obj == nil { + return map[string]interface{}{}, nil + } + data, err := json.Marshal(obj) + if err != nil { + return nil, err + } + var result = make(map[string]interface{}) + err = json.Unmarshal(data, &result) + if err != nil { + return nil, err + } + return result, nil +} + type EslEventRow struct { EslId EslId Created time.Time diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index 561343e12..4c44633b1 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -19,6 +19,7 @@ package db import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "os" @@ -1434,6 +1435,107 @@ func TestReadWriteEnvironment(t *testing.T) { }) } } +func TestReadWriteEslEvent(t *testing.T) { + const envName = "dev" + const appName = "my-app" + const lockId = "ui-v2-ke1up" + const message = "test" + const authorName = "testauthor" + const authorEmail = "testemail@example.com" + const evtType = EvtCreateApplicationVersion + expectedJson, _ := json.Marshal(map[string]interface{}{ + "env": envName, + "app": appName, + "lockId": lockId, + "message": message, + "metadata": map[string]string{ + "authorEmail": authorEmail, + "authorName": authorName, + }, + }) + expectedJsonWithoutMetadata, _ := json.Marshal(map[string]interface{}{ + "env": envName, + "app": appName, + "lockId": lockId, + "message": message, + "metadata": map[string]string{ + "authorEmail": "", + "authorName": "", + }, + }) + + tcs := []struct { + Name string + EventType EventType + EventData *map[string]string + EventMetadata ESLMetadata + ExpectedEsl EslEventRow + }{ + { + Name: "Write and read", + EventType: evtType, + EventData: &map[string]string{ + "env": envName, + "app": appName, + "lockId": lockId, + "message": message, + }, + EventMetadata: ESLMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, + ExpectedEsl: EslEventRow{ + EventType: evtType, + EventJson: string(expectedJson), + }, + }, + { + Name: "Write and read with nil metadata", + EventType: evtType, + EventData: &map[string]string{ + "env": envName, + "app": appName, + "lockId": lockId, + "message": message, + }, + ExpectedEsl: EslEventRow{ + EventType: evtType, + EventJson: string(expectedJsonWithoutMetadata), + }, + }, + } + + for _, tc := range tcs { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + ctx := testutil.MakeTestContext() + dbHandler := setupDB(t) + err := dbHandler.WithTransaction(ctx, func(ctx context.Context, transaction *sql.Tx) error { + err := dbHandler.DBWriteEslEventInternal(ctx, tc.EventType, transaction, tc.EventData, tc.EventMetadata) + if err != nil { + return err + } + + actual, err := dbHandler.DBReadEslEventInternal(ctx, transaction, true) + if err != nil { + return err + } + + if diff := cmp.Diff(tc.ExpectedEsl.EventType, actual.EventType); diff != "" { + t.Fatalf("event type mismatch (-want, +got):\n%s", diff) + } + if diff := cmp.Diff(tc.ExpectedEsl.EventJson, actual.EventJson); diff != "" { + t.Fatalf("event json mismatch (-want, +got):\n%s", diff) + } + return nil + }) + if err != nil { + t.Fatalf("transaction error: %v", err) + } + }) + } +} func TestReadWriteAllEnvironments(t *testing.T) { type TestCase struct { diff --git a/services/cd-service/pkg/repository/repository.go b/services/cd-service/pkg/repository/repository.go index f3cf144fd..562567f8b 100644 --- a/services/cd-service/pkg/repository/repository.go +++ b/services/cd-service/pkg/repository/repository.go @@ -1021,7 +1021,20 @@ func (r *repository) ApplyTransformersInternal(ctx context.Context, transaction return nil, nil, nil, &applyErr } logger.FromContext(ctx).Info("writing esl event...") - err = r.DB.DBWriteEslEventInternal(ctx, t.GetDBEventType(), transaction, t) + + user, readUserErr := auth.ReadUserFromContext(ctx) + + if readUserErr != nil { + return nil, nil, nil, &TransformerBatchApplyError{ + TransformerError: readUserErr, + Index: -1, + } + } + eventMetadata := db.ESLMetadata{ + AuthorName: user.Name, + AuthorEmail: user.Email, + } + err = r.DB.DBWriteEslEventInternal(ctx, t.GetDBEventType(), transaction, t, eventMetadata) if err != nil { return nil, nil, nil, &TransformerBatchApplyError{ TransformerError: err, diff --git a/services/manifest-repo-export-service/pkg/repository/repository.go b/services/manifest-repo-export-service/pkg/repository/repository.go index 767f1a75b..1ba1ce352 100644 --- a/services/manifest-repo-export-service/pkg/repository/repository.go +++ b/services/manifest-repo-export-service/pkg/repository/repository.go @@ -22,11 +22,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/freiheit-com/kuberpult/pkg/argocd" - "github.com/freiheit-com/kuberpult/pkg/config" - "github.com/freiheit-com/kuberpult/pkg/db" - "github.com/freiheit-com/kuberpult/pkg/mapper" - time2 "github.com/freiheit-com/kuberpult/pkg/time" "io" "os" "path/filepath" @@ -37,6 +32,12 @@ import ( "sync" "time" + "github.com/freiheit-com/kuberpult/pkg/argocd" + "github.com/freiheit-com/kuberpult/pkg/config" + "github.com/freiheit-com/kuberpult/pkg/db" + "github.com/freiheit-com/kuberpult/pkg/mapper" + time2 "github.com/freiheit-com/kuberpult/pkg/time" + "github.com/freiheit-com/kuberpult/pkg/grpc" "google.golang.org/protobuf/types/known/timestamppb" @@ -546,8 +547,8 @@ func CombineArray(others []*TransformerResult) *TransformerResult { return r } -func (r *repository) ApplyTransformers(ctx context.Context, transaction *sql.Tx, transformers ...Transformer) (*TransformerResult, *TransformerBatchApplyError) { - commitMsg, state, changes, applyErr := r.ApplyTransformersInternal(ctx, transaction, transformers...) +func (r *repository) ApplyTransformers(ctx context.Context, transaction *sql.Tx, transformer Transformer) (*TransformerResult, *TransformerBatchApplyError) { + commitMsg, state, changes, applyErr := r.ApplyTransformersInternal(ctx, transaction, transformer) if applyErr != nil { return nil, applyErr } @@ -565,10 +566,16 @@ func (r *repository) ApplyTransformers(ctx context.Context, transaction *sql.Tx, When: time.Now(), } - // TODO this will be handled in Ref SRX-PA568W + transformerMetadata := transformer.GetMetadata() + if transformerMetadata.AuthorEmail == "" || transformerMetadata.AuthorName == "" { + return nil, &TransformerBatchApplyError{ + TransformerError: fmt.Errorf("transformer metadata is empty"), + Index: -1, + } + } user := auth.User{ - Email: "invalid@example.com", - Name: "invalid", + Email: transformerMetadata.AuthorEmail, + Name: transformerMetadata.AuthorName, DexAuthContext: nil, } diff --git a/services/manifest-repo-export-service/pkg/repository/transformer.go b/services/manifest-repo-export-service/pkg/repository/transformer.go index 3bfe0385a..da213c0ac 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer.go @@ -88,6 +88,7 @@ func manifestDirectoryWithReleasesVersion(fs billy.Filesystem, application strin type Transformer interface { Transform(ctx context.Context, state *State, t TransformerContext, transaction *sql.Tx) (commitMsg string, e error) GetDBEventType() db.EventType + GetMetadata() *TransformerMetadata } type TransformerContext interface { @@ -96,6 +97,15 @@ type TransformerContext interface { DeleteEnvFromApp(app string, env string) } +type TransformerMetadata struct { + AuthorName string `json:"authorName,omitempty"` + AuthorEmail string `json:"authorEmail,omitempty"` +} + +func (t *TransformerMetadata) GetMetadata() *TransformerMetadata { + return t +} + func RunTransformer(ctx context.Context, t Transformer, s *State, transaction *sql.Tx) (string, *TransformerResult, error) { runner := transformerRunner{ ChangedApps: nil, @@ -218,14 +228,15 @@ func (c *QueueApplicationVersion) Transform( } type DeployApplicationVersion struct { - Authentication `json:"-"` - Environment string `json:"env"` - Application string `json:"app"` - Version uint64 `json:"version"` - LockBehaviour api.LockBehavior `json:"lockBehaviour"` - WriteCommitData bool `json:"writeCommitData"` - SourceTrain *DeployApplicationVersionSource `json:"sourceTrain"` - Author string `json:"author"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + Application string `json:"app"` + Version uint64 `json:"version"` + LockBehaviour api.LockBehavior `json:"lockBehaviour"` + WriteCommitData bool `json:"writeCommitData"` + SourceTrain *DeployApplicationVersionSource `json:"sourceTrain"` + Author string `json:"author"` } func (c *DeployApplicationVersion) GetDBEventType() db.EventType { @@ -380,6 +391,10 @@ func (c *DeployApplicationVersion) Transform( d := &CleanupOldApplicationVersions{ Application: c.Application, + TransformerMetadata: TransformerMetadata{ + AuthorName: existingDeployment.Metadata.DeployedByName, + AuthorEmail: existingDeployment.Metadata.DeployedByEmail, + }, } if err := t.Execute(d, transaction); err != nil { return "", err @@ -389,10 +404,11 @@ func (c *DeployApplicationVersion) Transform( } type CreateEnvironmentLock struct { - Authentication `json:"-"` - Environment string `json:"env"` - LockId string `json:"lockId"` - Message string `json:"message"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + LockId string `json:"lockId"` + Message string `json:"message"` } func (c *CreateEnvironmentLock) GetDBEventType() db.EventType { @@ -465,9 +481,10 @@ func createLock(ctx context.Context, fs billy.Filesystem, lockId, message, autho } type DeleteEnvironmentLock struct { - Authentication `json:"-"` - Environment string `json:"env"` - LockId string `json:"lockId"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + LockId string `json:"lockId"` } func (c *DeleteEnvironmentLock) GetDBEventType() db.EventType { @@ -507,11 +524,12 @@ func (c *DeleteEnvironmentLock) Transform( } type CreateEnvironmentApplicationLock struct { - Authentication `json:"-"` - Environment string `json:"env"` - Application string `json:"app"` - LockId string `json:"lockId"` - Message string `json:"message"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + Application string `json:"app"` + LockId string `json:"lockId"` + Message string `json:"message"` } func (c *CreateEnvironmentApplicationLock) GetDBEventType() db.EventType { @@ -559,10 +577,11 @@ func (c *CreateEnvironmentApplicationLock) Transform( } type DeleteEnvironmentApplicationLock struct { - Authentication `json:"-"` - Environment string `json:"env"` - Application string `json:"app"` - LockId string `json:"lockId"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + Application string `json:"app"` + LockId string `json:"lockId"` } func (c *DeleteEnvironmentApplicationLock) GetDBEventType() db.EventType { @@ -599,17 +618,18 @@ func (c *DeleteEnvironmentApplicationLock) Transform( } type CreateApplicationVersion struct { - Authentication `json:"-"` - Version uint64 `json:"version"` - Application string `json:"app"` - Manifests map[string]string `json:"manifests"` - SourceCommitId string `json:"sourceCommitId"` - SourceAuthor string `json:"sourceCommitAuthor"` - SourceMessage string `json:"sourceCommitMessage"` - Team string `json:"team"` - DisplayVersion string `json:"displayVersion"` - WriteCommitData bool `json:"writeCommitData"` - PreviousCommit string `json:"previousCommit"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Version uint64 `json:"version"` + Application string `json:"app"` + Manifests map[string]string `json:"manifests"` + SourceCommitId string `json:"sourceCommitId"` + SourceAuthor string `json:"sourceCommitAuthor"` + SourceMessage string `json:"sourceCommitMessage"` + Team string `json:"team"` + DisplayVersion string `json:"displayVersion"` + WriteCommitData bool `json:"writeCommitData"` + PreviousCommit string `json:"previousCommit"` } func (c *CreateApplicationVersion) GetDBEventType() db.EventType { @@ -725,6 +745,10 @@ func (c *CreateApplicationVersion) Transform( Authentication: c.Authentication, WriteCommitData: c.WriteCommitData, Author: c.SourceAuthor, + TransformerMetadata: TransformerMetadata{ + AuthorName: c.SourceAuthor, + AuthorEmail: "", + }, } err := t.Execute(d, transaction) if err != nil { @@ -820,11 +844,12 @@ func GetLastRelease(fs billy.Filesystem, application string) (uint64, error) { } type CreateEnvironmentTeamLock struct { - Authentication `json:"-"` - Environment string `json:"env"` - Team string `json:"team"` - LockId string `json:"lockId"` - Message string `json:"message"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + Team string `json:"team"` + LockId string `json:"lockId"` + Message string `json:"message"` } func (c *CreateEnvironmentTeamLock) GetDBEventType() db.EventType { @@ -900,10 +925,11 @@ func (c *CreateEnvironmentTeamLock) Transform( } type DeleteEnvironmentTeamLock struct { - Authentication `json:"-"` - Environment string `json:"env"` - Team string `json:"team"` - LockId string `json:"lockId"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + Team string `json:"team"` + LockId string `json:"lockId"` } func (c *DeleteEnvironmentTeamLock) GetDBEventType() db.EventType { @@ -946,9 +972,10 @@ func (c *DeleteEnvironmentTeamLock) Transform( } type CreateEnvironment struct { - Authentication `json:"-"` - Environment string `json:"env"` - Config config.EnvironmentConfig `json:"config"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Environment string `json:"env"` + Config config.EnvironmentConfig `json:"config"` } func (c *CreateEnvironment) GetDBEventType() db.EventType { @@ -1050,7 +1077,8 @@ func removeCommit(fs billy.Filesystem, commitID, application string) error { } type CleanupOldApplicationVersions struct { - Application string + Application string + TransformerMetadata `json:"metadata"` } func (c *CleanupOldApplicationVersions) GetDBEventType() db.EventType { @@ -1104,12 +1132,13 @@ func (c *CleanupOldApplicationVersions) Transform( } type ReleaseTrain struct { - Authentication `json:"-"` - Target string `json:"target"` - Team string `json:"team,omitempty"` - CommitHash string `json:"commitHash"` - WriteCommitData bool `json:"writeCommitData"` - Repo Repository `json:"-"` + Authentication `json:"-"` + TransformerMetadata `json:"metadata"` + Target string `json:"target"` + Team string `json:"team,omitempty"` + CommitHash string `json:"commitHash"` + WriteCommitData bool `json:"writeCommitData"` + Repo Repository `json:"-"` } func (c *ReleaseTrain) GetDBEventType() db.EventType { diff --git a/services/manifest-repo-export-service/pkg/repository/transformer_test.go b/services/manifest-repo-export-service/pkg/repository/transformer_test.go index 636755b0e..577c81358 100644 --- a/services/manifest-repo-export-service/pkg/repository/transformer_test.go +++ b/services/manifest-repo-export-service/pkg/repository/transformer_test.go @@ -23,13 +23,13 @@ import ( "path" "testing" - // "github.com/freiheit-com/kuberpult/pkg/config" + "time" + "github.com/freiheit-com/kuberpult/pkg/db" "github.com/freiheit-com/kuberpult/pkg/testutil" "github.com/go-git/go-billy/v5/util" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "time" ) const ( @@ -116,6 +116,8 @@ type FilenameAndData struct { func TestTransformerWorksWithDb(t *testing.T) { const appName = "myapp" + const authorName = "testAuthorName" + const authorEmail = "testAuthorEmail@example.com" tcs := []struct { Name string Transformers []Transformer @@ -123,6 +125,7 @@ func TestTransformerWorksWithDb(t *testing.T) { ExpectedApp *db.DBAppWithMetaData ExpectedAllReleases *db.DBReleaseWithMetaData ExpectedFile []*FilenameAndData + ExpectedAuthor *map[string]string }{ { // as of now we only have the DeployApplicationVersion transformer, @@ -170,6 +173,10 @@ func TestTransformerWorksWithDb(t *testing.T) { DisplayVersion: "", WriteCommitData: false, PreviousCommit: "", + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, }, ExpectedError: nil, @@ -186,6 +193,34 @@ func TestTransformerWorksWithDb(t *testing.T) { Team: "team-123", }, }, + ExpectedAuthor: &map[string]string{ + "Name": authorName, + "Email": authorEmail, + }, + }, + { + Name: "Should give an error when the metadata is nil", + Transformers: []Transformer{ + &CreateApplicationVersion{ + Authentication: Authentication{}, + Version: 7, + Application: appName, + Manifests: map[string]string{ + envAcceptance: "mani-1-acc", + envDev: "mani-1-dev", + }, + SourceCommitId: "", + SourceAuthor: "", + SourceMessage: "", + Team: "team-123", + DisplayVersion: "", + WriteCommitData: false, + PreviousCommit: "", + }, + }, + ExpectedError: errMatcher{"first apply failed, aborting: error not specific to one transformer of this batch: " + + "transformer metadata is empty", + }, }, { // as of now we only have the DeployApplicationVersion and CreateEnvironmentLock transformer, @@ -257,6 +292,10 @@ func TestTransformerWorksWithDb(t *testing.T) { DisplayVersion: "", WriteCommitData: false, PreviousCommit: "", + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, &CreateApplicationVersion{ Authentication: Authentication{}, @@ -273,6 +312,10 @@ func TestTransformerWorksWithDb(t *testing.T) { DisplayVersion: "", WriteCommitData: false, PreviousCommit: "", + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, &CreateApplicationVersion{ Authentication: Authentication{}, @@ -289,9 +332,17 @@ func TestTransformerWorksWithDb(t *testing.T) { DisplayVersion: "", WriteCommitData: false, PreviousCommit: "", + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, &CleanupOldApplicationVersions{ Application: appName, + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, }, ExpectedFile: []*FilenameAndData{ @@ -304,6 +355,7 @@ func TestTransformerWorksWithDb(t *testing.T) { fileData: []byte("abcdef"), }, }, + ExpectedAuthor: &map[string]string{"Name": authorName, "Email": authorEmail}, }, { Name: "Create a single environment", @@ -311,6 +363,10 @@ func TestTransformerWorksWithDb(t *testing.T) { &CreateEnvironment{ Environment: "development", Config: testutil.MakeEnvConfigLatest(nil), + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, }, ExpectedFile: []*FilenameAndData{ @@ -332,10 +388,18 @@ func TestTransformerWorksWithDb(t *testing.T) { &CreateEnvironment{ Environment: "staging", Config: testutil.MakeEnvConfigLatest(nil), + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, &CreateEnvironment{ Environment: "staging", Config: testutil.MakeEnvConfigUpstream("development", nil), + TransformerMetadata: TransformerMetadata{ + AuthorName: authorName, + AuthorEmail: authorEmail, + }, }, }, ExpectedFile: []*FilenameAndData{ @@ -412,6 +476,14 @@ func TestTransformerWorksWithDb(t *testing.T) { if !cmp.Equal(actualFileData, expectedFile.fileData) { t.Fatalf("Expected '%v', got '%v'", string(expectedFile.fileData), string(actualFileData)) } + if tc.ExpectedAuthor != nil { + if !cmp.Equal(updatedState.Commit.Author().Name, (*tc.ExpectedAuthor)["Name"]) { + t.Fatalf("Expected '%v', got '%v'", (*tc.ExpectedAuthor)["Name"], updatedState.Commit.Author().Name) + } + if !cmp.Equal(updatedState.Commit.Author().Email, (*tc.ExpectedAuthor)["Email"]) { + t.Fatalf("Expected '%v', got '%v'", (*tc.ExpectedAuthor)["Email"], updatedState.Commit.Author().Email) + } + } } } })