-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(auto-cancel): server-side logic for auto canceling obsolete buil…
…ds (#911) * init commit * move auto cancel to build pkg * db test file for new func * integration test * linter and fmt debug statements * linter overlord * address feedback * publish before auto cancel and continue upon failure to cancel
- Loading branch information
Showing
14 changed files
with
442 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,181 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package build | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/gin-gonic/gin" | ||
| "github.com/go-vela/server/database" | ||
| "github.com/go-vela/server/internal/token" | ||
| "github.com/go-vela/types/constants" | ||
| "github.com/go-vela/types/library" | ||
| "github.com/go-vela/types/pipeline" | ||
| ) | ||
|
|
||
| // AutoCancel is a helper function that checks to see if any pending or running | ||
| // builds for the repo can be replaced by the current build. | ||
| func AutoCancel(c *gin.Context, b *library.Build, rB *library.Build, r *library.Repo, cancelOpts *pipeline.CancelOptions) (bool, error) { | ||
| // if build is the current build, continue | ||
| if rB.GetID() == b.GetID() { | ||
| return false, nil | ||
| } | ||
|
|
||
| // ensure criteria is met before auto canceling (push to same branch, or pull with same action from same head_ref) | ||
| if (strings.EqualFold(rB.GetEvent(), constants.EventPush) && | ||
| strings.EqualFold(b.GetEvent(), constants.EventPush) && | ||
| strings.EqualFold(b.GetBranch(), rB.GetBranch())) || | ||
| (strings.EqualFold(rB.GetEvent(), constants.EventPull) && | ||
| strings.EqualFold(b.GetEventAction(), rB.GetEventAction()) && | ||
| strings.EqualFold(b.GetHeadRef(), rB.GetHeadRef())) { | ||
| switch { | ||
| case strings.EqualFold(rB.GetStatus(), constants.StatusPending) && cancelOpts.Pending: | ||
| // pending build will be handled gracefully by worker once pulled off queue | ||
| rB.SetStatus(constants.StatusCanceled) | ||
|
|
||
| _, err := database.FromContext(c).UpdateBuild(c, rB) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| case strings.EqualFold(rB.GetStatus(), constants.StatusRunning) && cancelOpts.Running: | ||
| // call cancelRunning routine for builds already running on worker | ||
| err := cancelRunning(c, rB, r) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| default: | ||
| return false, nil | ||
| } | ||
|
|
||
| // set error message that references current build | ||
| rB.SetError(fmt.Sprintf("build was auto canceled in favor of build %d", b.GetNumber())) | ||
|
|
||
| _, err := database.FromContext(c).UpdateBuild(c, rB) | ||
| if err != nil { | ||
| // if this call fails, we still canceled the build, so return true | ||
| return true, err | ||
| } | ||
| } | ||
|
|
||
| return true, nil | ||
| } | ||
|
|
||
| // cancelRunning is a helper function that determines the executor currently running a build and sends an API call | ||
| // to that executor's worker to cancel the build. | ||
| func cancelRunning(c *gin.Context, b *library.Build, r *library.Repo) error { | ||
| e := new([]library.Executor) | ||
| // retrieve the worker | ||
| w, err := database.FromContext(c).GetWorkerForHostname(c, b.GetHost()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // prepare the request to the worker to retrieve executors | ||
| client := http.DefaultClient | ||
| client.Timeout = 30 * time.Second | ||
| endpoint := fmt.Sprintf("%s/api/v1/executors", w.GetAddress()) | ||
|
|
||
| req, err := http.NewRequestWithContext(context.Background(), "GET", endpoint, nil) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| tm := c.MustGet("token-manager").(*token.Manager) | ||
|
|
||
| // set mint token options | ||
| mto := &token.MintTokenOpts{ | ||
| Hostname: "vela-server", | ||
| TokenType: constants.WorkerAuthTokenType, | ||
| TokenDuration: time.Minute * 1, | ||
| } | ||
|
|
||
| // mint token | ||
| tkn, err := tm.MintToken(mto) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // add the token to authenticate to the worker | ||
| req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) | ||
|
|
||
| // make the request to the worker and check the response | ||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| defer resp.Body.Close() | ||
|
|
||
| // Read Response Body | ||
| respBody, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // parse response and validate at least one item was returned | ||
| err = json.Unmarshal(respBody, e) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, executor := range *e { | ||
| // check each executor on the worker running the build to see if it's running the build we want to cancel | ||
| if strings.EqualFold(executor.Repo.GetFullName(), r.GetFullName()) && *executor.GetBuild().Number == b.GetNumber() { | ||
| // prepare the request to the worker | ||
| client := http.DefaultClient | ||
| client.Timeout = 30 * time.Second | ||
|
|
||
| // set the API endpoint path we send the request to | ||
| u := fmt.Sprintf("%s/api/v1/executors/%d/build/cancel", w.GetAddress(), executor.GetID()) | ||
|
|
||
| req, err := http.NewRequestWithContext(context.Background(), "DELETE", u, nil) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| tm := c.MustGet("token-manager").(*token.Manager) | ||
|
|
||
| // set mint token options | ||
| mto := &token.MintTokenOpts{ | ||
| Hostname: "vela-server", | ||
| TokenType: constants.WorkerAuthTokenType, | ||
| TokenDuration: time.Minute * 1, | ||
| } | ||
|
|
||
| // mint token | ||
| tkn, err := tm.MintToken(mto) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // add the token to authenticate to the worker | ||
| req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) | ||
|
|
||
| // perform the request to the worker | ||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| // Read Response Body | ||
| respBody, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| err = json.Unmarshal(respBody, b) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package build | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/go-vela/types/constants" | ||
| "github.com/go-vela/types/database" | ||
| "github.com/go-vela/types/library" | ||
| ) | ||
|
|
||
| // ListPendingAndRunningBuilds gets a list of all pending and running builds in the provided timeframe from the database. | ||
| func (e *engine) ListPendingAndRunningBuildsForRepo(ctx context.Context, repo *library.Repo) ([]*library.Build, error) { | ||
| e.logger.Trace("listing all pending and running builds from the database") | ||
|
|
||
| // variables to store query results and return value | ||
| b := new([]database.Build) | ||
| builds := []*library.Build{} | ||
|
|
||
| // send query to the database and store result in variable | ||
| err := e.client. | ||
| Table(constants.TableBuild). | ||
| Select("*"). | ||
| Where("repo_id = ?", repo.GetID()). | ||
| Where("status = 'running' OR status = 'pending'"). | ||
| Find(&b). | ||
| Error | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // iterate through all query results | ||
| for _, build := range *b { | ||
| // https://golang.org/doc/faq#closures_and_goroutines | ||
| tmp := build | ||
|
|
||
| // convert query result to library type | ||
| // | ||
| // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary | ||
| builds = append(builds, tmp.ToLibrary()) | ||
| } | ||
|
|
||
| return builds, nil | ||
| } |
Oops, something went wrong.