refactor: move workflow selection to gale

daggerverse/gale/gale.go

@@ -3,6 +3,7 @@ package main
 import (
  "context"
  "fmt"
+ "path/filepath"
  "strings"
 )
 
@@ -34,7 +35,7 @@ func (g *Gale) List(
  sb := strings.Builder{}
 
  err := walkWorkflowDir(ctx, workflowsDir, workflows,
- func(ctx context.Context, path string, file *File) error {
+ func(ctx context.Context, path string, file *File) (bool, error) {
  // dagger do not support maps yet, so we're defining anonymous struct to unmarshal the yaml file to avoid
  // hit this limitation.
  var workflow struct {
@@ -43,7 +44,7 @@ func (g *Gale) List(
  }
 
  if err := unmarshalContentsToYAML(ctx, file, &workflow); err != nil {
- return err
+ return false, err
  }
 
  sb.WriteString("Workflow: ")
@@ -61,7 +62,7 @@ func (g *Gale) List(
 
  sb.WriteString("\n") // extra empty line
 
- return nil
+ return true, nil
  },
  )
 
@@ -104,19 +105,58 @@ func (g *Gale) Run(
  runnerDebug Optional[bool],
  // GitHub token to use for authentication.
  token Optional[*Secret],
-) *WorkflowRun {
+) (*WorkflowRun, error) {
+ wp := ""
+ wf, ok := workflowFile.Get()
+ if !ok {
+ workflow, ok := workflow.Get()
+ if !ok {
+ return nil, fmt.Errorf("workflow or workflow file must be provided")
+ }
+
+ workflows := getWorkflowsDir(source, repo, tag, branch, workflowsDir)
+
+ err := walkWorkflowDir(ctx, workflowsDir, workflows, func(ctx context.Context, path string, file *File) (bool, error) {
+ // when relative path or file name is matches with the workflow name, we assume that it is the workflow
+ // file.
+ if path == workflow || filepath.Base(path) == workflow {
+ wf = file
+ wp = path
+ return false, nil
+ }
+
+ // otherwise, look for matching workflow name in the workflow file.
+ var f struct {
+ Name string `yaml:"name"`
+ }
+
+ if err := unmarshalContentsToYAML(ctx, file, &f); err != nil {
+ return false, err
+ }
+
+ if f.Name == workflow {
+ wf = file
+ wp = path
+ return false, nil
+ }
+
+ return true, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
 
  return &WorkflowRun{
  Runner: g.Runner().Container(ctx, image, container, source, repo, tag, branch),
  Config: WorkflowRunConfig{
- WorkflowsDir: workflowsDir.GetOr(".github/workflows"),
- WorkflowFile: workflowFile.GetOr(nil),
- Workflow: workflow.GetOr(""),
+ WorkflowFile: wf,
+ Workflow: wp,
  Job: job.GetOr(""),
  Event: event.GetOr("push"),
- EventFile: eventFile.GetOr(nil),
+ EventFile: eventFile.GetOr(dag.Directory().WithNewFile("event.json", "{}").File("event.json")),
  RunnerDebug: runnerDebug.GetOr(false),
  Token: token.GetOr(nil),
  },
- }
+ }, nil
 }

daggerverse/gale/repo.go

@@ -26,9 +26,10 @@ func getWorkflowsDir(source Optional[*Directory], repo, tag, branch, workflowsDi
 }
 
 // WorkflowWalkFunc is the type of the function called for each workflow file visited by walkWorkflowDir.
-type WorkflowWalkFunc func(ctx context.Context, path string, file *File) error
+type WorkflowWalkFunc func(ctx context.Context, path string, file *File) (bool, error)
 
-// walkWorkflowDir walks the workflows directory and calls the given function for each workflow file.
+// walkWorkflowDir walks the workflows directory and calls the given function for each workflow file. If walk function
+// returns false, the walk stops.
 func walkWorkflowDir(ctx context.Context, path Optional[string], dir *Directory, fn WorkflowWalkFunc) error {
  entries, err := dir.Entries(ctx)
  if err != nil {
@@ -41,9 +42,14 @@ func walkWorkflowDir(ctx context.Context, path Optional[string], dir *Directory,
  file := dir.File(entry)
  path := filepath.Join(path.GetOr(".github/workflows"), entry)
 
- if err := fn(ctx, path, file); err != nil {
+ walk, err := fn(ctx, path, file)
+ if err != nil {
  return err
  }
+
+ if !walk {
+ return nil
+ }
  }
  }
 

daggerverse/gale/runner.go

@@ -77,6 +77,19 @@ func (r *Runner) Container(
  ctr = ctr.WithEnvVariable("GALE_RUNNER_CACHE", path)
  ctr = ctr.WithMountedCache(path, cache, ContainerWithMountedCacheOpts{Sharing: Shared})
 
+ // ghx specific directory configuration -- TODO: refactor this later to be more generic for runners
+ var (
+ metadata = "/home/runner/_temp/gale/metadata"
+ actions = "/home/runner/_temp/gale/actions"
+ cacheOpts = ContainerWithMountedCacheOpts{Sharing: Shared}
+ )
+
+ ctr = ctr.WithEnvVariable("GHX_METADATA_DIR", metadata)
+ ctr = ctr.WithMountedCache(metadata, dag.CacheVolume("gale-metadata"), cacheOpts)
+
+ ctr = ctr.WithEnvVariable("GHX_ACTIONS_DIR", actions)
+ ctr = ctr.WithMountedCache(actions, dag.CacheVolume("gale-actions"), cacheOpts)
+
  // add env variable to the container to indicate container is configured
  ctr = ctr.WithEnvVariable("GALE_RUNNER_ID", id)
  ctr = ctr.WithEnvVariable("GALE_CONFIGURED", "true")

daggerverse/gale/workflow_run.go

@@ -5,21 +5,26 @@ import (
  "fmt"
  "path/filepath"
  "time"
+
+ "github.com/google/uuid"
 )
 
 type WorkflowRun struct {
  // Base container to use for the workflow run.
  Runner *RunnerContainer
 
+ // Workflow run cache path to mount to runner containers.
+ RunCachePath string
+
+ // Workflow run cache volume to share data between jobs in the same workflow run and keep the data after the workflow
+ RunCacheVolume *CacheVolume
+
  // Configuration of the workflow run.
  Config WorkflowRunConfig
 }
 
 // WorkflowRunConfig holds the configuration of a workflow run.
 type WorkflowRunConfig struct {
- // Path to the workflow directory.
- WorkflowsDir string
-
  // WorkflowFile is external workflow file to run.
  WorkflowFile *File
 
@@ -79,14 +84,16 @@ func (wr *WorkflowRun) Directory(
  return nil, err
  }
 
- dir := dag.Directory().WithDirectory("run", container.Directory("/home/runner/_temp/ghx/run"))
+ rd := container.WithExec([]string{"cp", "-r", wr.RunCachePath, "/exported_run"}).Directory("/exported_run")
 
- if includeRepo.GetOr(false) {
- dir = dir.WithDirectory("repo", container.Directory("."))
- }
+ dir := dag.Directory().WithDirectory("run", rd.Directory("run"))
 
  if includeSecrets.GetOr(false) {
- dir = dir.WithDirectory("secrets", container.Directory("/home/runner/_temp/ghx/secrets"))
+ dir = dir.WithDirectory("secrets", rd.Directory("secrets"))
+ }
+
+ if includeRepo.GetOr(false) {
+ dir = dir.WithDirectory("repo", container.Directory("."))
  }
 
  if includeEvent.GetOr(false) && wr.Config.EventFile != nil {
@@ -114,66 +121,53 @@ func (wr *WorkflowRun) Directory(
 func (wr *WorkflowRun) run() (*Container, error) {
  container := wr.Runner.Ctr
 
- // loading request scoped configs
+ var (
+ id = uuid.New().String()
+ wrPath = filepath.Join("/home/runner/_temp/_gale/runs", id)
+ cache = dag.CacheVolume(fmt.Sprintf("ghx-run-%s", id))
+ cacheOpts = ContainerWithMountedCacheOpts{Sharing: Shared}
+ )
 
- // configure workflow run configuration
- container = container.With(wr.configure)
+ // mount workflow run cache volume
+ wr.RunCachePath = wrPath
+ wr.RunCacheVolume = cache
 
- // ghx specific directory configuration
- container = container.WithEnvVariable("GHX_HOME", "/home/runner/_temp/ghx")
- container = container.WithMountedDirectory("/home/runner/_temp/ghx", dag.Directory())
- container = container.WithMountedCache("/home/runner/_temp/ghx/metadata", dag.CacheVolume("gale-metadata"), ContainerWithMountedCacheOpts{Sharing: Shared})
- container = container.WithMountedCache("/home/runner/_temp/ghx/actions", dag.CacheVolume("gale-actions"), ContainerWithMountedCacheOpts{Sharing: Shared})
-
- // workaround for disabling cache
- container = container.WithEnvVariable("CACHE_BUSTER", time.Now().Format(time.RFC3339Nano))
-
- // execute the workflow
- container = container.WithExec([]string{"ghx"}, ContainerWithExecOpts{ExperimentalPrivilegedNesting: true})
-
- // unloading request scoped configs
- container = container.WithoutEnvVariable("GHX_WORKFLOW")
- container = container.WithoutEnvVariable("GHX_JOB")
- container = container.WithoutEnvVariable("GHX_WORKFLOWS_DIR")
-
- return container, nil
-}
-
-func (wr *WorkflowRun) configure(c *Container) *Container {
- container := c
+ container = container.WithMountedCache(wrPath, cache, cacheOpts)
+ container = container.WithEnvVariable("GHX_HOME", wrPath)
 
  // set github token as secret if provided
  if wr.Config.Token != nil {
  container = container.WithSecretVariable("GITHUB_TOKEN", wr.Config.Token)
  }
 
- if wr.Config.WorkflowFile != nil {
- path := "/home/runner/_temp/_github_workflow/.gale/dagger.yaml"
-
- container = container.WithMountedFile(path, wr.Config.WorkflowFile)
- container = container.WithEnvVariable("GHX_WORKFLOWS_DIR", filepath.Dir(path))
-
- if wr.Config.Workflow != "" {
- container = container.WithEnvVariable("GHX_WORKFLOW", wr.Config.Workflow)
- } else {
- container = container.WithEnvVariable("GHX_WORKFLOW", path)
- }
- } else {
- container = container.WithEnvVariable("GHX_WORKFLOWS_DIR", wr.Config.WorkflowsDir)
- container = container.WithEnvVariable("GHX_WORKFLOW", wr.Config.Workflow)
+ // set runner debug mode if enabled
+ if wr.Config.RunnerDebug {
+ container = container.WithEnvVariable("RUNNER_DEBUG", "1")
  }
 
+ // set workflow config
+ path := filepath.Join(wrPath, "run", "workflow.yaml")
+
+ container = container.WithMountedFile(path, wr.Config.WorkflowFile)
+ container = container.WithEnvVariable("GHX_WORKFLOW", wr.Config.Workflow)
  container = container.WithEnvVariable("GHX_JOB", wr.Config.Job)
 
+ // event config
+ eventPath := filepath.Join(wrPath, "run", "event.json")
+
  container = container.WithEnvVariable("GITHUB_EVENT_NAME", wr.Config.Event)
+ container = container.WithEnvVariable("GITHUB_EVENT_PATH", eventPath)
+ container = container.WithMountedFile(eventPath, wr.Config.EventFile)
 
- if wr.Config.EventFile != nil {
- container = container.WithMountedFile("/home/runner/_temp/_github_workflow/event.json", wr.Config.EventFile)
- }
+ // workaround for disabling cache
+ container = container.WithEnvVariable("CACHE_BUSTER", time.Now().Format(time.RFC3339Nano))
 
- if wr.Config.RunnerDebug {
- container = container.WithEnvVariable("RUNNER_DEBUG", "1")
- }
+ // execute the workflow
+ container = container.WithExec([]string{"ghx"}, ContainerWithExecOpts{ExperimentalPrivilegedNesting: true})
 
- return container
+ // unloading request scoped configs
+ container = container.WithoutEnvVariable("GHX_JOB")
+ container = container.WithoutEnvVariable("GHX_WORKFLOWS_DIR")
+
+ return container, nil
 }

ghx/context/ast.go

@@ -13,11 +13,14 @@ type GhxConfig struct {
  // Job name to run. If not specified, the all jobs will be run.
  Job string `env:"GHX_JOB"`
 
- // Directory to look for workflows.
- WorkflowsDir string `env:"GHX_WORKFLOWS_DIR" envDefault:".github/workflows"`
-
  // Home directory for the ghx to use for storing execution related files.
  HomeDir string `env:"GHX_HOME" envDefault:"/home/runner/_temp/ghx"`
+
+ // ActionsDir is the directory to look for actions.
+ ActionsDir string `env:"GHX_ACTIONS_DIR" envDefault:"/home/runner/_temp/gale/actions"`
+
+ // MetadataDir is the directory to look for metadata.
+ MetadataDir string `env:"GHX_METADATA_DIR" envDefault:"/home/runner/_temp/gale/metadata"`
 }
 
 // DaggerContext is the context holding the dagger client.

ghx/context/context.go

@@ -63,12 +63,9 @@ func New(std context.Context, client *dagger.Client) (*Context, error) {
  // set the dagger client
  ctx.Dagger.Client = client
 
- // set non environment config for github ctx
- if ctx.Github.EventPath != "" {
- err := fs.ReadJSONFile(ctx.Github.EventPath, &ctx.Github.Event)
- if err != nil {
- return nil, err
- }
+ // read event file
+ if err := fs.ReadJSONFile(ctx.Github.EventPath, &ctx.Github.Event); err != nil {
+ return nil, err
  }
 
  // just to be sure that we have a initialized event map

ghx/context/execution.go

@@ -45,18 +45,6 @@ func (c *Context) UnsetWorkflow(result RunResult) {
  if err := fs.WriteJSONFile(filepath.Join(dir, "workflow_run.json"), report); err != nil {
  log.Errorf("failed to write workflow run", "error", err, "workflow", c.Execution.WorkflowRun.Workflow.Name)
  }
-
- // copy file to the workflow run directory
- var (
- src = c.Execution.WorkflowRun.Workflow.Path
- dst = filepath.Join(dir, "workflow.yaml")
- )
-
- // copy the workflow file to the workflow run directory to keep the workflow file as it is to prevent potential
- // changes when marshaling the workflow file again from context
- if err := fs.CopyFile(src, dst); err != nil {
- log.Errorf("failed to write workflow", "error", err, "workflow", c.Execution.WorkflowRun.Workflow.Name)
- }
 }
 
 // SetJob sets the given job to the execution context.

ghx/context/paths.go

@@ -11,13 +11,13 @@ import (
 // GetMetadataPath returns the path of the metadata path. If the path does not exist, it creates it. This path assumes
 // zenith module set a cache for metadata, otherwise it will be empty every time it runs.
 func (c *Context) GetMetadataPath() (string, error) {
- return EnsureDir(c.GhxConfig.HomeDir, "metadata")
+ return EnsureDir(c.GhxConfig.MetadataDir)
 }
 
 // GetActionsPath returns the path of the custom action repositories clones and stores. If the path does not exist, it
 // creates it.
 func (c *Context) GetActionsPath() (string, error) {
- return EnsureDir(c.GhxConfig.HomeDir, "actions")
+ return EnsureDir(c.GhxConfig.ActionsDir)
 }
 
 // GetSecretsPath returns the path of the secrets.json file containing the secrets. If the path does not exist, it