From a0a67704224f4abe4cb932c707b47cab2dd9ab5e Mon Sep 17 00:00:00 2001 From: Ali AKCA Date: Thu, 3 Aug 2023 21:35:17 +0200 Subject: [PATCH] generate unique workflow ids --- .golangci.yaml | 2 +- go.mod | 3 +- go.sum | 2 - internal/config/config.go | 11 +++- internal/idgen/idgen.go | 61 +++++++++++++++++++++++ internal/idgen/idgen_test.go | 97 ++++++++++++++++++++++++++++++++++++ pkg/gale/gale.go | 21 +++++--- 7 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 internal/idgen/idgen.go create mode 100644 internal/idgen/idgen_test.go diff --git a/.golangci.yaml b/.golangci.yaml index 4e3ffc9b..105b33d7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -42,7 +42,7 @@ linters-settings: - github.com/spf13/pflag - github.com/cli/go-gh/v2 - github.com/Masterminds/semver/v3 - - github.com/google/uuid + - github.com/adrg/xdg - github.com/rhysd/actionlint - github.com/magefile/mage/sh - github.com/julienschmidt/httprouter diff --git a/go.mod b/go.mod index 6e1ec8b0..a33b5522 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.20 require ( dagger.io/dagger v0.7.4 + github.com/adrg/xdg v0.4.0 github.com/cli/go-gh/v2 v2.1.0 - github.com/google/uuid v1.3.0 github.com/julienschmidt/httprouter v1.3.0 github.com/magefile/mage v1.15.0 github.com/rhysd/actionlint v1.6.25 @@ -17,7 +17,6 @@ require ( require ( github.com/99designs/gqlgen v0.17.33 // indirect github.com/Khan/genqlient v0.6.0 // indirect - github.com/adrg/xdg v0.4.0 // indirect github.com/cli/safeexec v1.0.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/iancoleman/strcase v0.3.0 // indirect diff --git a/go.sum b/go.sum index 5a88489b..55d25879 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/henvic/httpretty v0.0.6 h1:JdzGzKZBajBfnvlMALXXMVQWxWMF/ofTy8C3/OSUTxs= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= diff --git a/internal/config/config.go b/internal/config/config.go index d5ba9813..75e22446 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,8 @@ import ( "path/filepath" "dagger.io/dagger" + + "github.com/adrg/xdg" ) // cfg is the global configuration for ghx. No other package should access it directly. @@ -14,8 +16,8 @@ func init() { } type config struct { - client *dagger.Client // dagger client for the config. - ghxHome string // ghx home directory where all the data is stored. + client *dagger.Client // client is the dagger client for the config. + ghxHome string // ghxHome directory where all the data is stored. } // SetClient sets the dagger client for the config. @@ -52,3 +54,8 @@ func GhxRunsDir() string { func GhxRunDir(runID string) string { return filepath.Join(GhxRunsDir(), runID) } + +// GaleDataHome returns the path for local data. +func GaleDataHome() string { + return filepath.Join(xdg.DataHome, "gale") +} diff --git a/internal/idgen/idgen.go b/internal/idgen/idgen.go new file mode 100644 index 00000000..e5994e56 --- /dev/null +++ b/internal/idgen/idgen.go @@ -0,0 +1,61 @@ +package idgen + +import ( + "path/filepath" + "strconv" + + "github.com/aweris/gale/internal/config" + "github.com/aweris/gale/internal/core" + "github.com/aweris/gale/internal/fs" +) + +const ( + metadataFile = "metadata.json" + keyWorkflowRunID = "workflow_run_id" + keyJobRunID = "job_run_id" +) + +type counter map[string]int + +// TODO: This is not concurrency safe. Need to use lock file or something similar to make it concurrency safe + +// GenerateWorkflowRunID generates a unique workflow run id for the given repository +func GenerateWorkflowRunID(repo *core.Repository) (string, error) { + dataPath := filepath.Join(config.GaleDataHome(), repo.NameWithOwner, metadataFile) + + return generateID(dataPath, keyWorkflowRunID) +} + +// GenerateJobRunID generates a unique job run id for the given repository +func GenerateJobRunID(repo *core.Repository) (string, error) { + dataPath := filepath.Join(config.GaleDataHome(), repo.NameWithOwner, metadataFile) + + return generateID(dataPath, keyJobRunID) +} + +func generateID(dataPath, key string) (string, error) { + err := fs.EnsureFile(dataPath) + if err != nil { + return "", err + } + + var ids counter + + err = fs.ReadJSONFile(dataPath, &ids) + if err != nil { + return "", err + } + + if ids == nil { + ids = make(counter) + } + + ids[key]++ + + err = fs.WriteJSONFile(dataPath, ids) + if err != nil { + return "", err + } + + return strconv.Itoa(ids[key]), nil +} diff --git a/internal/idgen/idgen_test.go b/internal/idgen/idgen_test.go new file mode 100644 index 00000000..9513b24e --- /dev/null +++ b/internal/idgen/idgen_test.go @@ -0,0 +1,97 @@ +package idgen_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/aweris/gale/internal/config" + "github.com/aweris/gale/internal/core" + "github.com/aweris/gale/internal/idgen" +) + +func TestGenerateWorkflowRunID(t *testing.T) { + // ensure that the test data directory is deleted after the test + defer os.RemoveAll(filepath.Join(config.GaleDataHome(), "testorg")) + + repo := &core.Repository{ + NameWithOwner: "testorg/testrepo", + } + + // Test first workflow run ID generation + wfRunID, err := idgen.GenerateWorkflowRunID(repo) + if err != nil { + t.Fatalf("Error generating workflow run ID: %v", err) + } + + if wfRunID != "1" { + t.Errorf("Expected first workflow run ID to be 1, got %s", wfRunID) + } + + // Test second workflow run ID generation + wfRunID, err = idgen.GenerateWorkflowRunID(repo) + if err != nil { + t.Fatalf("Error generating workflow run ID: %v", err) + } + + if wfRunID != "2" { + t.Errorf("Expected second workflow run ID to be 2, got %s", wfRunID) + } + + repo2 := &core.Repository{ + NameWithOwner: "testorg/testrepo2", + } + + // Test first workflow run ID generation + wfRunID, err = idgen.GenerateWorkflowRunID(repo2) + if err != nil { + t.Fatalf("Error generating workflow run ID: %v", err) + } + + if wfRunID != "1" { + t.Errorf("Expected first workflow run ID to be 1, got %s", wfRunID) + } +} + +func TestGenerateJobRunID(t *testing.T) { + // ensure that the test data directory is deleted after the test + defer os.RemoveAll(filepath.Join(config.GaleDataHome(), "testorg")) + + repo := &core.Repository{ + NameWithOwner: "testorg/testrepo", + } + + // Test first job run ID generation + jobRunID, err := idgen.GenerateJobRunID(repo) + if err != nil { + t.Fatalf("Error generating job run ID: %v", err) + } + + if jobRunID != "1" { + t.Errorf("Expected first job run ID to be 1, got %s", jobRunID) + } + + // Test second job run ID generation + jobRunID, err = idgen.GenerateJobRunID(repo) + if err != nil { + t.Fatalf("Error generating job run ID: %v", err) + } + + if jobRunID != "2" { + t.Errorf("Expected second job run ID to be 2, got %s", jobRunID) + } + + repo2 := &core.Repository{ + NameWithOwner: "testorg/testrepo2", + } + + // Test first workflow run ID generation + jobRunID, err = idgen.GenerateJobRunID(repo2) + if err != nil { + t.Fatalf("Error generating job run ID: %v", err) + } + + if jobRunID != "1" { + t.Errorf("Expected first job run ID to be 1, got %s", jobRunID) + } +} diff --git a/pkg/gale/gale.go b/pkg/gale/gale.go index 14e1ed04..8cca38f3 100644 --- a/pkg/gale/gale.go +++ b/pkg/gale/gale.go @@ -6,12 +6,11 @@ import ( "dagger.io/dagger" - "github.com/google/uuid" - "github.com/aweris/gale/internal/config" "github.com/aweris/gale/internal/core" "github.com/aweris/gale/internal/dagger/services" "github.com/aweris/gale/internal/dagger/tools" + "github.com/aweris/gale/internal/idgen" ) // RunOpts are the options for the Run function. @@ -56,10 +55,18 @@ func Run(ctx context.Context, workflow, job string, opts ...RunOpts) dagger.With jm.Name = job } - runID := uuid.New().String() + workflowRunID, err := idgen.GenerateWorkflowRunID(repo) + if err != nil { + return fail(container, err) + } + + jobRunID, err := idgen.GenerateJobRunID(repo) + if err != nil { + return fail(container, err) + } jr := &core.JobRun{ - RunID: runID, + RunID: jobRunID, Job: jm, } @@ -78,12 +85,12 @@ func Run(ctx context.Context, workflow, job string, opts ...RunOpts) dagger.With container = container.With(core.NewGithubRepositoryContext(repo).Apply) container = container.With(core.NewGithubSecretsContext(token).Apply) container = container.With(core.NewGithubURLContext().Apply) - container = container.With(core.NewGithubWorkflowContext(repo, wf, runID).Apply) // TODO: RunID here is workflow run id, not job run id. It's ok for now, but we need to fix it. + container = container.With(core.NewGithubWorkflowContext(repo, wf, workflowRunID).Apply) container = container.With(core.NewGithubJobInfoContext(job).Apply) // job run configuration - container = container.WithDirectory(config.GhxRunDir(runID), dir) - container = container.WithExec([]string{"/usr/local/bin/ghx", "run", runID}) + container = container.WithDirectory(config.GhxRunDir(jobRunID), dir) + container = container.WithExec([]string{"/usr/local/bin/ghx", "run", jobRunID}) return container }