diff --git a/api/build/executable.go b/api/build/executable.go new file mode 100644 index 000000000..832e5abeb --- /dev/null +++ b/api/build/executable.go @@ -0,0 +1,93 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/executable builds GetBuildExecutable +// +// Get a build executable in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number to retrieve +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the build executable +// type: json +// schema: +// "$ref": "#/definitions/Build" +// '400': +// description: Bad request +// schema: +// "$ref": "#/definitions/Error" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Could not retrieve build executable +// schema: +// "$ref": "#/definitions/Error" + +// GetBuildExecutable represents the API handler to capture +// a build executable for a repo from the configured backend. +func GetBuildExecutable(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + cl := claims.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "subject": cl.Subject, + }).Infof("reading build executable %s/%d", r.GetFullName(), b.GetNumber()) + + bExecutable, err := database.FromContext(c).PopBuildExecutable(b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to pop build executable: %w", err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, bExecutable) +} diff --git a/api/build/publish.go b/api/build/publish.go index 7e1b3d5a3..f5e144232 100644 --- a/api/build/publish.go +++ b/api/build/publish.go @@ -20,7 +20,31 @@ import ( // PublishToQueue is a helper function that creates // a build item and publishes it to the queue. func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { - item := types.ToItem(p, b, r, u) + byteExecutable, err := json.Marshal(p) + if err != nil { + logrus.Errorf("Failed to marshal build executable %d for %s: %v", b.GetNumber(), r.GetFullName(), err) + + // error out the build + CleanBuild(ctx, db, b, nil, nil, err) + + return + } + + bExecutable := new(library.BuildExecutable) + bExecutable.SetBuildID(b.GetID()) + bExecutable.SetData(byteExecutable) + + err = db.CreateBuildExecutable(bExecutable) + if err != nil { + logrus.Errorf("Failed to publish build executable to database %d for %s: %v", b.GetNumber(), r.GetFullName(), err) + + // error out the build + CleanBuild(ctx, db, b, nil, nil, err) + + return + } + + item := types.ToItem(b, r, u) logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName()) diff --git a/database/database.go b/database/database.go index d0317e3ff..ea93bfaaf 100644 --- a/database/database.go +++ b/database/database.go @@ -10,6 +10,7 @@ import ( "time" "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -61,6 +62,7 @@ type ( logger *logrus.Entry build.BuildInterface + executable.BuildExecutableInterface hook.HookInterface log.LogInterface pipeline.PipelineInterface diff --git a/database/executable/create.go b/database/executable/create.go new file mode 100644 index 000000000..002ac03bc --- /dev/null +++ b/database/executable/create.go @@ -0,0 +1,56 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CreateBuildExecutable creates a new build executable in the database. +func (e *engine) CreateBuildExecutable(b *library.BuildExecutable) error { + e.logger.WithFields(logrus.Fields{ + "build": b.GetBuildID(), + }).Tracef("creating build executable for build %d in the database", b.GetBuildID()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutableFromLibrary + executable := database.BuildExecutableFromLibrary(b) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Validate + err := executable.Validate() + if err != nil { + return err + } + + // compress data for the build executable + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Compress + err = executable.Compress(e.config.CompressionLevel) + if err != nil { + return err + } + + // encrypt the data field for the build executable + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Encrypt + err = executable.Encrypt(e.config.EncryptionKey) + if err != nil { + return fmt.Errorf("unable to encrypt build executable for build %d: %w", b.GetBuildID(), err) + } + + // send query to the database + return e.client. + Table(constants.TableBuildExecutable). + Create(executable). + Error +} diff --git a/database/executable/create_test.go b/database/executable/create_test.go new file mode 100644 index 000000000..abf047f79 --- /dev/null +++ b/database/executable/create_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestExecutable_Engine_CreateBuildExecutable(t *testing.T) { + // setup types + _bExecutable := testBuildExecutable() + _bExecutable.SetID(1) + _bExecutable.SetBuildID(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "build_executables" +("build_id","data","id") +VALUES ($1,$2,$3) RETURNING "id"`). + WithArgs(1, AnyArgument{}, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateBuildExecutable(_bExecutable) + + if test.failure { + if err == nil { + t.Errorf("CreateBuildExecutable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateBuildExecutable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/executable/executable.go b/database/executable/executable.go new file mode 100644 index 000000000..c53759192 --- /dev/null +++ b/database/executable/executable.go @@ -0,0 +1,80 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // config represents the settings required to create the engine that implements the BuildExecutableService interface. + config struct { + // specifies the level of compression to use for the BuildExecutable engine + CompressionLevel int + // specifies the encryption key to use for the BuildExecutable engine + EncryptionKey string + // specifies to skip creating tables and indexes for the BuildExecutable engine + SkipCreation bool + // specifies the driver for proper popping query + Driver string + } + + // engine represents the build executable functionality that implements the BuildExecutableService interface. + engine struct { + // engine configuration settings used in build executable functions + config *config + + // gorm.io/gorm database client used in build executable functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in build executable functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with build executables in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new BuildExecutable engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating build executable database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of build executables table and indexes in the database") + + return e, nil + } + + // create the build executables table + err := e.CreateBuildExecutableTable(e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableBuildExecutable, err) + } + + return e, nil +} diff --git a/database/executable/executable_test.go b/database/executable/executable_test.go new file mode 100644 index 000000000..20ebd6b49 --- /dev/null +++ b/database/executable/executable_test.go @@ -0,0 +1,213 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "database/sql/driver" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestExecutable_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + level int + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + level: 1, + key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{ + CompressionLevel: 1, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + Driver: "postgres", + }, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + level: 1, + key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{ + CompressionLevel: 1, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + Driver: "sqlite3", + }, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithCompressionLevel(test.level), + WithEncryptionKey(test.key), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + WithDriver(test.name), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithCompressionLevel(0), + WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + WithDriver(constants.DriverPostgres), + ) + if err != nil { + t.Errorf("unable to create new postgres build_itnerary engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithCompressionLevel(0), + WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + WithDriver(constants.DriverSqlite), + ) + if err != nil { + t.Errorf("unable to create new sqlite build_itnerary engine: %v", err) + } + + return _engine +} + +// testBuildExecutable is a test helper function to create a library +// BuildExecutable type with all fields set to their zero values. +func testBuildExecutable() *library.BuildExecutable { + return &library.BuildExecutable{ + ID: new(int64), + BuildID: new(int64), + Data: new([]byte), + } +} + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type AnyArgument struct{} + +// Match satisfies sqlmock.Argument interface. +func (a AnyArgument) Match(v driver.Value) bool { + return v != nil +} diff --git a/database/executable/interface.go b/database/executable/interface.go new file mode 100644 index 000000000..852d77872 --- /dev/null +++ b/database/executable/interface.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import "github.com/go-vela/types/library" + +// BuildExecutableInterface represents the Vela interface for build executable +// functions with the supported Database backends. +type BuildExecutableInterface interface { + // BuildExecutable Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + CreateBuildExecutableTable(string) error + + // BuildExecutable Data Manipulation Language Functions + // + // https://en.wikipedia.org/wiki/Data_manipulation_language + + // CreateBuildExecutable defines a function that creates a build executable. + CreateBuildExecutable(*library.BuildExecutable) error + // PopBuildExecutable defines a function that gets and deletes a build executable. + PopBuildExecutable(int64) (*library.BuildExecutable, error) +} diff --git a/database/executable/opts.go b/database/executable/opts.go new file mode 100644 index 000000000..1d91a4a54 --- /dev/null +++ b/database/executable/opts.go @@ -0,0 +1,74 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for build executables. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for build executables. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the build executable engine + e.client = client + + return nil + } +} + +// WithCompressionLevel sets the compression level in the database engine for build executables. +func WithCompressionLevel(level int) EngineOpt { + return func(e *engine) error { + // set the compression level in the build executable engine + e.config.CompressionLevel = level + + return nil + } +} + +// WithEncryptionKey sets the encryption key in the database engine for build executables. +func WithEncryptionKey(key string) EngineOpt { + return func(e *engine) error { + // set the encryption key in the build executables engine + e.config.EncryptionKey = key + + return nil + } +} + +// WithDriver sets the driver type in the database engine for build executables. +func WithDriver(driver string) EngineOpt { + return func(e *engine) error { + // set the driver type in the build executable engine + e.config.Driver = driver + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for build executables. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the build executable engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for build executables. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the build executable engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/executable/opts_test.go b/database/executable/opts_test.go new file mode 100644 index 000000000..6bb89d0ae --- /dev/null +++ b/database/executable/opts_test.go @@ -0,0 +1,265 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "reflect" + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestExecutable_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestExecutable_EngineOpt_WithCompressionLevel(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + level int + want int + }{ + { + failure: false, + name: "compression level set to -1", + level: -1, + want: -1, + }, + { + failure: false, + name: "compression level set to 0", + level: 0, + want: 0, + }, + { + failure: false, + name: "compression level set to 1", + level: 1, + want: 1, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithCompressionLevel(test.level)(e) + + if test.failure { + if err == nil { + t.Errorf("WithCompressionLevel for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithCompressionLevel returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.CompressionLevel, test.want) { + t.Errorf("WithCompressionLevel is %v, want %v", e.config.CompressionLevel, test.want) + } + }) + } +} + +func TestExecutable_EngineOpt_WithEncryptionKey(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + key string + want string + }{ + { + failure: false, + name: "encryption key set", + key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + want: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + }, + { + failure: false, + name: "encryption key not set", + key: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithEncryptionKey(test.key)(e) + + if test.failure { + if err == nil { + t.Errorf("WithEncryptionKey for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithEncryptionKey returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.EncryptionKey, test.want) { + t.Errorf("WithEncryptionKey is %v, want %v", e.config.EncryptionKey, test.want) + } + }) + } +} + +func TestExecutable_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestExecutable_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/executable/pop.go b/database/executable/pop.go new file mode 100644 index 000000000..c38d472bc --- /dev/null +++ b/database/executable/pop.go @@ -0,0 +1,78 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "gorm.io/gorm/clause" +) + +// PopBuildExecutable pops a build executable by build_id from the database. +func (e *engine) PopBuildExecutable(id int64) (*library.BuildExecutable, error) { + e.logger.Tracef("popping build executable for build %d from the database", id) + + // variable to store query results + b := new(database.BuildExecutable) + + // at the time of coding, GORM does not implement a version of Sqlite3 that supports RETURNING. + // so we have to select and delete for the Sqlite driver. + switch e.config.Driver { + case constants.DriverPostgres: + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuildExecutable). + Clauses(clause.Returning{}). + Where("build_id = ?", id). + Delete(b). + Error + + if err != nil { + return nil, err + } + + case constants.DriverSqlite: + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuildExecutable). + Where("id = ?", id). + Take(b). + Error + if err != nil { + return nil, err + } + + // send query to the database to delete result just got + err = e.client. + Table(constants.TableBuildExecutable). + Delete(b). + Error + if err != nil { + return nil, err + } + } + + // decrypt the fields for the build executable + // + // https://pkg.go.dev/github.com/go-vela/types/database#Repo.Decrypt + err := b.Decrypt(e.config.EncryptionKey) + if err != nil { + return nil, err + } + + // decompress data for the build executable + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Decompress + err = b.Decompress() + if err != nil { + return nil, err + } + + // return the decompressed build executable + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.ToLibrary + return b.ToLibrary(), nil +} diff --git a/database/executable/pop_test.go b/database/executable/pop_test.go new file mode 100644 index 000000000..b89dcec9e --- /dev/null +++ b/database/executable/pop_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestExecutable_Engine_PopBuildExecutable(t *testing.T) { + // setup types + _bExecutable := testBuildExecutable() + _bExecutable.SetID(1) + _bExecutable.SetBuildID(1) + _bExecutable.SetData([]byte("foo")) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "build_id", "data"}). + AddRow(1, 1, "+//18dbf7mF+v7ZPK3Wo5h2TD6v4Zg95sCMUJYO2tpwY37DEgTxW5xdyt3Tey9w=") + + // ensure the mock expects the query + _mock.ExpectQuery(`DELETE FROM "build_executables" WHERE build_id = $1 RETURNING *`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuildExecutable(_bExecutable) + if err != nil { + t.Errorf("unable to create test build executable for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.BuildExecutable + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _bExecutable, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _bExecutable, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.PopBuildExecutable(1) + + if test.failure { + if err == nil { + t.Errorf("PopBuildExecutable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("PopBuildExecutable for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("PopBuildExecutable for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/executable/table.go b/database/executable/table.go new file mode 100644 index 000000000..c2b3323b2 --- /dev/null +++ b/database/executable/table.go @@ -0,0 +1,50 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import "github.com/go-vela/types/constants" + +const ( + // CreatePostgresTable represents a query to create the Postgres build_executables table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +build_executables ( + id SERIAL PRIMARY KEY, + build_id INTEGER, + data BYTEA, + UNIQUE(build_id) +); +` + + // CreateSqliteTable represents a query to create the Sqlite build_executables table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +build_executables ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + build_id INTEGER, + data BLOB, + UNIQUE(build_id) +); +` +) + +// CreateBuildExecutableTable creates the build executables table in the database. +func (e *engine) CreateBuildExecutableTable(driver string) error { + e.logger.Tracef("creating build_executables table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the build_executables table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the build_executables table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/executable/table_test.go b/database/executable/table_test.go new file mode 100644 index 000000000..ed203b113 --- /dev/null +++ b/database/executable/table_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package executable + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestExecutable_Engine_CreateBuildExecutableTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateBuildExecutableTable(test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateBuildExecutableTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateBuildExecutableTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/integration_test.go b/database/integration_test.go index cd4e3f9a7..c87dec0ae 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -32,6 +33,7 @@ import ( type Resources struct { Builds []*library.Build Deployments []*library.Deployment + Executables []*library.BuildExecutable Hooks []*library.Hook Logs []*library.Log Pipelines []*library.Pipeline @@ -117,6 +119,8 @@ func TestDatabase_Integration(t *testing.T) { t.Run("test_builds", func(t *testing.T) { testBuilds(t, db, resources) }) + t.Run("test_executables", func(t *testing.T) { testExecutables(t, db, resources) }) + t.Run("test_hooks", func(t *testing.T) { testHooks(t, db, resources) }) t.Run("test_logs", func(t *testing.T) { testLogs(t, db, resources) }) @@ -381,6 +385,53 @@ func testBuilds(t *testing.T, db Interface, resources *Resources) { } } +func testExecutables(t *testing.T, db Interface, resources *Resources) { + // create a variable to track the number of methods called for pipelines + methods := make(map[string]bool) + // capture the element type of the pipeline interface + element := reflect.TypeOf(new(executable.BuildExecutableInterface)).Elem() + // iterate through all methods found in the pipeline interface + for i := 0; i < element.NumMethod(); i++ { + // skip tracking the methods to create indexes and tables for pipelines + // since those are already called when the database engine starts + if strings.Contains(element.Method(i).Name, "Index") || + strings.Contains(element.Method(i).Name, "Table") { + continue + } + + // add the method name to the list of functions + methods[element.Method(i).Name] = false + } + + // create the pipelines + for _, executable := range resources.Executables { + err := db.CreateBuildExecutable(executable) + if err != nil { + t.Errorf("unable to create executable %d: %v", executable.GetID(), err) + } + } + methods["CreateBuildExecutable"] = true + + // pop executables for builds + for _, executable := range resources.Executables { + got, err := db.PopBuildExecutable(executable.GetBuildID()) + if err != nil { + t.Errorf("unable to get executable %d for build %d: %v", executable.GetID(), executable.GetBuildID(), err) + } + if !reflect.DeepEqual(got, executable) { + t.Errorf("PopBuildExecutable() is %v, want %v", got, executable) + } + } + methods["PopBuildExecutable"] = true + + // ensure we called all the methods we expected to + for method, called := range methods { + if !called { + t.Errorf("method %s was not called for pipelines", method) + } + } +} + func testHooks(t *testing.T, db Interface, resources *Resources) { // create a variable to track the number of methods called for hooks methods := make(map[string]bool) @@ -1862,6 +1913,16 @@ func newResources() *Resources { buildTwo.SetRuntime("docker") buildTwo.SetDistribution("linux") + executableOne := new(library.BuildExecutable) + executableOne.SetID(1) + executableOne.SetBuildID(1) + executableOne.SetData([]byte("foo")) + + executableTwo := new(library.BuildExecutable) + executableTwo.SetID(2) + executableTwo.SetBuildID(2) + executableTwo.SetData([]byte("foo")) + deploymentOne := new(library.Deployment) deploymentOne.SetID(1) deploymentOne.SetRepoID(1) @@ -2229,6 +2290,7 @@ func newResources() *Resources { return &Resources{ Builds: []*library.Build{buildOne, buildTwo}, Deployments: []*library.Deployment{deploymentOne, deploymentTwo}, + Executables: []*library.BuildExecutable{executableOne, executableTwo}, Hooks: []*library.Hook{hookOne, hookTwo}, Logs: []*library.Log{logServiceOne, logServiceTwo, logStepOne, logStepTwo}, Pipelines: []*library.Pipeline{pipelineOne, pipelineTwo}, diff --git a/database/interface.go b/database/interface.go index 3ed775707..cc7428378 100644 --- a/database/interface.go +++ b/database/interface.go @@ -6,6 +6,7 @@ package database import ( "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -36,6 +37,9 @@ type Interface interface { // BuildInterface defines the interface for builds stored in the database. build.BuildInterface + // BuildExecutableInterface defines the interface for build executables stored in the database. + executable.BuildExecutableInterface + // HookInterface defines the interface for hooks stored in the database. hook.HookInterface diff --git a/database/resource.go b/database/resource.go index e6784b2e1..482834ecf 100644 --- a/database/resource.go +++ b/database/resource.go @@ -8,6 +8,7 @@ import ( "context" "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -35,6 +36,18 @@ func (e *engine) NewResources(ctx context.Context) error { return err } + // create the database agnostic engine for build_executables + e.BuildExecutableInterface, err = executable.New( + executable.WithClient(e.client), + executable.WithLogger(e.logger), + executable.WithSkipCreation(e.config.SkipCreation), + executable.WithEncryptionKey(e.config.EncryptionKey), + executable.WithDriver(e.config.Driver), + ) + if err != nil { + return err + } + // create the database agnostic engine for hooks e.HookInterface, err = hook.New( hook.WithClient(e.client), diff --git a/database/resource_test.go b/database/resource_test.go index 233dbb800..328f04f58 100644 --- a/database/resource_test.go +++ b/database/resource_test.go @@ -10,6 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -32,6 +33,8 @@ func TestDatabase_Engine_NewResources(t *testing.T) { _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the build executable queries + _mock.ExpectExec(executable.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/go.mod b/go.mod index f36b309d4..2f200eccf 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.20.1 + github.com/go-vela/types v0.20.2-0.20230821135955-6b577f36fdfe github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v53 v53.2.0 diff --git a/go.sum b/go.sum index a8eaa2a36..45416d2f1 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.20.1 h1:hHAX0Iij2J7UZ9f3SlXbwNy481CjKzU9CBfkiLuysVE= -github.com/go-vela/types v0.20.1/go.mod h1:AXO4oQSygOBQ02fPapsKjQHkx2aQO3zTu7clpvVbXBY= +github.com/go-vela/types v0.20.2-0.20230821135955-6b577f36fdfe h1:5lw7hJmwLiymoSI0H8gr9Aiixifv2wOXvtH4NJJZB2k= +github.com/go-vela/types v0.20.2-0.20230821135955-6b577f36fdfe/go.mod h1:AXO4oQSygOBQ02fPapsKjQHkx2aQO3zTu7clpvVbXBY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/mock/server/build.go b/mock/server/build.go index 6d989d8af..cc24af60c 100644 --- a/mock/server/build.go +++ b/mock/server/build.go @@ -150,6 +150,14 @@ const ( "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJidWlsZF9pZCI6MSwicmVwbyI6ImZvby9iYXIiLCJzdWIiOiJPY3RvY2F0IiwiaWF0IjoxNTE2MjM5MDIyfQ.hD7gXpaf9acnLBdOBa4GOEa5KZxdzd0ZvK6fGwaN4bc" }` + // BuildExecutableResp represents a JSON return for requesting a build executable. + BuildExecutableResp = `{ + "id": 1 + "build_id": 1, + "data": "eyAKICAgICJpZCI6ICJzdGVwX25hbWUiLAogICAgInZlcnNpb24iOiAiMSIsCiAgICAibWV0YWRhdGEiOnsKICAgICAgICAiY2xvbmUiOnRydWUsCiAgICAgICAgImVudmlyb25tZW50IjpbInN0ZXBzIiwic2VydmljZXMiLCJzZWNyZXRzIl19LAogICAgIndvcmtlciI6e30sCiAgICAic3RlcHMiOlsKICAgICAgICB7CiAgICAgICAgICAgICJpZCI6InN0ZXBfZ2l0aHViX29jdG9jYXRfMV9pbml0IiwKICAgICAgICAgICAgImRpcmVjdG9yeSI6Ii92ZWxhL3NyYy9naXRodWIuY29tL2dpdGh1Yi9vY3RvY2F0IiwKICAgICAgICAgICAgImVudmlyb25tZW50IjogeyJCVUlMRF9BVVRIT1IiOiJPY3RvY2F0In0KICAgICAgICB9CiAgICBdCn0KCg==" + }` + + // CleanResourcesResp represents a string return for cleaning resources as an admin. CleanResourcesResp = "42 builds cleaned. 42 services cleaned. 42 steps cleaned." ) @@ -330,6 +338,8 @@ func buildToken(c *gin.Context) { if strings.EqualFold(b, "2") { c.AbortWithStatusJSON(http.StatusBadRequest, "") + + return } data := []byte(BuildTokenResp) @@ -340,6 +350,28 @@ func buildToken(c *gin.Context) { c.JSON(http.StatusOK, body) } +// buildExecutable has a param :build returns mock JSON for a http GET. +// +// Pass "0" to :build to test receiving a http 500 response. +func buildExecutable(c *gin.Context) { + b := c.Param("build") + + if strings.EqualFold(b, "0") { + msg := fmt.Sprintf("unable to get build executable for build %s", b) + + c.AbortWithStatusJSON(http.StatusInternalServerError, types.Error{Message: &msg}) + + return + } + + data := []byte(BuildExecutableResp) + + var body library.BuildExecutable + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + // cleanResources has a query param :before returns mock JSON for a http PUT // // Pass "1" to :before to test receiving a http 500 response. Pass "2" to :before diff --git a/mock/server/server.go b/mock/server/server.go index d2f6ccbab..7bc668719 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -42,6 +42,7 @@ func FakeHandler() http.Handler { e.PUT("/api/v1/repos/:org/:repo/builds/:build", updateBuild) e.DELETE("/api/v1/repos/:org/:repo/builds/:build", removeBuild) e.GET("/api/v1/repos/:org/:repo/builds/:build/token", buildToken) + e.GET("/api/v1/repos/:org/:repo/builds/:build/executable", buildExecutable) // mock endpoints for deployment calls e.GET("/api/v1/deployments/:org/:repo", getDeployments) diff --git a/queue/redis/length_test.go b/queue/redis/length_test.go index 95cb21423..c97a5b3f8 100644 --- a/queue/redis/length_test.go +++ b/queue/redis/length_test.go @@ -17,10 +17,9 @@ func TestRedis_Length(t *testing.T) { // setup types // use global variables in redis_test.go _item := &types.Item{ - Build: _build, - Pipeline: _steps, - Repo: _repo, - User: _user, + Build: _build, + Repo: _repo, + User: _user, } // setup queue item diff --git a/queue/redis/pop_test.go b/queue/redis/pop_test.go index 8ae6094b7..cb8209e1d 100644 --- a/queue/redis/pop_test.go +++ b/queue/redis/pop_test.go @@ -18,10 +18,9 @@ func TestRedis_Pop(t *testing.T) { // setup types // use global variables in redis_test.go _item := &types.Item{ - Build: _build, - Pipeline: _steps, - Repo: _repo, - User: _user, + Build: _build, + Repo: _repo, + User: _user, } // setup queue item diff --git a/queue/redis/push_test.go b/queue/redis/push_test.go index 69af784e3..74e815926 100644 --- a/queue/redis/push_test.go +++ b/queue/redis/push_test.go @@ -16,10 +16,9 @@ func TestRedis_Push(t *testing.T) { // setup types // use global variables in redis_test.go _item := &types.Item{ - Build: _build, - Pipeline: _steps, - Repo: _repo, - User: _user, + Build: _build, + Repo: _repo, + User: _user, } // setup queue item diff --git a/router/build.go b/router/build.go index 22cd37aa4..c8cc0d688 100644 --- a/router/build.go +++ b/router/build.go @@ -26,6 +26,7 @@ import ( // DELETE /api/v1/repos/:org/:repo/builds/:build/cancel // GET /api/v1/repos/:org/:repo/builds/:build/logs // GET /api/v1/repos/:org/:repo/builds/:build/token +// GET /api/v1/repos/:org/:repo/builds/:build/executable // POST /api/v1/repos/:org/:repo/builds/:build/services // GET /api/v1/repos/:org/:repo/builds/:build/services // GET /api/v1/repos/:org/:repo/builds/:build/services/:service @@ -61,6 +62,7 @@ func BuildHandlers(base *gin.RouterGroup) { b.DELETE("/cancel", executors.Establish(), perm.MustWrite(), build.CancelBuild) b.GET("/logs", perm.MustRead(), log.ListLogsForBuild) b.GET("/token", perm.MustWorkerAuthToken(), build.GetBuildToken) + b.GET("/executable", perm.MustBuildAccess(), build.GetBuildExecutable) // Service endpoints // * Log endpoints