diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 5065136..4c9cd76 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -43,8 +43,8 @@ var _ = Describe("API Client", func() { initRunConfig := api.InitiateRunConfig{ InitializationParameters: []api.InitializationParameter{}, - TaskDefinitions: []api.TaskDefinition{ - {Path: "foo", FileContents: "echo 'bar'"}, + TaskDefinitions: []api.MintDirectoryEntry{ + {Path: "foo", FileContents: "echo 'bar'", Permissions: 0o644, Type: "file"}, }, TargetedTaskKeys: []string{}, UseCache: false, @@ -82,8 +82,8 @@ var _ = Describe("API Client", func() { initRunConfig := api.InitiateRunConfig{ InitializationParameters: []api.InitializationParameter{}, - TaskDefinitions: []api.TaskDefinition{ - {Path: "foo", FileContents: "echo 'bar'"}, + TaskDefinitions: []api.MintDirectoryEntry{ + {Path: "foo", FileContents: "echo 'bar'", Permissions: 0o644, Type: "file"}, }, TargetedTaskKeys: []string{}, UseCache: false, @@ -191,7 +191,7 @@ var _ = Describe("API Client", func() { It("makes the request", func() { body := api.SetSecretsInVaultConfig{ VaultName: "default", - Secrets: []api.Secret{{Name: "ABC", Secret: "123"}}, + Secrets: []api.Secret{{Name: "ABC", Secret: "123"}}, } bodyBytes, _ := json.Marshal(body) diff --git a/internal/api/config.go b/internal/api/config.go index 6386d8f..2a84e2d 100644 --- a/internal/api/config.go +++ b/internal/api/config.go @@ -26,8 +26,8 @@ func (c Config) Validate() error { type InitiateRunConfig struct { InitializationParameters []InitializationParameter `json:"initialization_parameters"` - TaskDefinitions []TaskDefinition `json:"task_definitions"` - MintDirectory []TaskDefinition `json:"mint_directory"` + TaskDefinitions []MintDirectoryEntry `json:"task_definitions"` + MintDirectory []MintDirectoryEntry `json:"mint_directory"` TargetedTaskKeys []string `json:"targeted_task_keys,omitempty"` Title string `json:"title,omitempty"` UseCache bool `json:"use_cache"` @@ -86,8 +86,8 @@ func (lf LintProblem) FileLocation() string { line := lf.Line column := lf.Column - if (len(lf.StackTrace) > 0) { - lastStackEntry := lf.StackTrace[len(lf.StackTrace) - 1] + if len(lf.StackTrace) > 0 { + lastStackEntry := lf.StackTrace[len(lf.StackTrace)-1] fileName = lastStackEntry.FileName line = NullInt{ Value: lastStackEntry.Line, diff --git a/internal/api/mint_directory_entry.go b/internal/api/mint_directory_entry.go new file mode 100644 index 0000000..9e74a1c --- /dev/null +++ b/internal/api/mint_directory_entry.go @@ -0,0 +1,9 @@ +package api + +type MintDirectoryEntry struct { + OriginalPath string `json:"-"` + Path string `json:"path"` + Type string `json:"type"` + Permissions uint32 `json:"permissions"` + FileContents string `json:"file_contents"` +} diff --git a/internal/cli/service.go b/internal/cli/service.go index 2ce9fc8..39363a0 100644 --- a/internal/cli/service.go +++ b/internal/cli/service.go @@ -99,7 +99,7 @@ func (s Service) InitiateRun(cfg InitiateRunConfig) (*api.InitiateRunResult, err return nil, errors.Wrap(err, "validation failed") } - mintDirectoryYamlPaths := make([]string, 0) + mintDirectory := make([]api.MintDirectoryEntry, 0) taskDefinitionYamlPath := cfg.MintFilePath mintDirectoryPath, err := s.findMintDirectoryPath(cfg.MintDirectory) @@ -109,37 +109,22 @@ func (s Service) InitiateRun(cfg InitiateRunConfig) (*api.InitiateRunResult, err // It's possible (when no directory is specified) that there is no .mint directory found during traversal if mintDirectoryPath != "" { - paths, err := s.yamlFilePathsInDirectory(mintDirectoryPath) + mintDirectoryEntries, err := s.mintDirectoryEntries(mintDirectoryPath) if err != nil { if errors.Is(err, errors.ErrFileNotExists) { return nil, fmt.Errorf("You specified --dir %q, but %q could not be found", cfg.MintDirectory, cfg.MintDirectory) } - return nil, errors.Wrap(err, "unable to find yaml files in directory") + return nil, err } - mintDirectoryYamlPaths = paths + mintDirectory = mintDirectoryEntries } - mintDirectory, err := s.taskDefinitionsFromPaths(mintDirectoryYamlPaths) + taskDefinitions, err := s.mintDirectoryEntriesFromPaths([]string{taskDefinitionYamlPath}) if err != nil { return nil, errors.Wrap(err, "unable to read provided files") } - taskDefinitions, err := s.taskDefinitionsFromPaths([]string{taskDefinitionYamlPath}) - if err != nil { - return nil, errors.Wrap(err, "unable to read provided files") - } - - // mintDirectory task definitions must have their paths relative to the .mint directory - for i, taskDefinition := range mintDirectory { - relPath, err := filepath.Rel(mintDirectoryPath, taskDefinition.Path) - if err != nil { - return nil, errors.Wrapf(err, "unable to determine relative path of %q", taskDefinition.Path) - } - taskDefinition.Path = filepath.Join(".mint", relPath) - mintDirectory[i] = taskDefinition - } - i := 0 initializationParameters := make([]api.InitializationParameter, len(cfg.InitParameters)) for key, value := range cfg.InitParameters { @@ -178,11 +163,12 @@ func (s Service) Lint(cfg LintConfig) (*api.LintResult, error) { if err != nil { return nil, fmt.Errorf("You specified a mint directory of %q, but %q could not be found", cfg.MintDirectory, cfg.MintDirectory) } - configFilePaths = append(configFilePaths, mintDirectoryPath) + if mintDirectoryPath != "" { + configFilePaths = append(configFilePaths, mintDirectoryPath) + } configFilePaths = removeDuplicateStrings(configFilePaths) taskDefinitionYamlPaths := make([]string, 0) - for _, fileOrDir := range configFilePaths { fi, err := os.Stat(fileOrDir) if err != nil { @@ -193,15 +179,19 @@ func (s Service) Lint(cfg LintConfig) (*api.LintResult, error) { } if fi.IsDir() { - paths, err := s.yamlFilePathsInDirectory(fileOrDir) + mintDirectory, err := s.mintDirectoryEntries(fileOrDir) if err != nil { return nil, errors.Wrap(err, "unable to find yaml files in directory") } - taskDefinitionYamlPaths = append(taskDefinitionYamlPaths, paths...) + + for _, entry := range s.yamlFilesInMintDirectory(mintDirectory) { + taskDefinitionYamlPaths = append(taskDefinitionYamlPaths, entry.OriginalPath) + } } else { taskDefinitionYamlPaths = append(taskDefinitionYamlPaths, fileOrDir) } } + taskDefinitionYamlPaths = removeDuplicateStrings(taskDefinitionYamlPaths) wd, err := os.Getwd() if err != nil { @@ -487,11 +477,17 @@ func (s Service) UpdateLeaves(cfg UpdateLeavesConfig) error { if len(cfg.Files) > 0 { files = cfg.Files } else { - yamlFilePathsInDirectory, err := s.yamlFilePathsInDirectory(cfg.DefaultDir) + mintDirectory, err := s.mintDirectoryEntries(cfg.DefaultDir) if err != nil { - return errors.Wrap(err, fmt.Sprintf("unable to find yaml files in directory %s", cfg.DefaultDir)) + return err + } + + yamlFiles := make([]string, 0) + for _, entry := range s.yamlFilesInMintDirectory(mintDirectory) { + yamlFiles = append(yamlFiles, entry.OriginalPath) } - files = yamlFilePathsInDirectory + + files = yamlFiles } if len(files) == 0 { @@ -650,31 +646,139 @@ func (s Service) taskDefinitionsFromPaths(paths []string) ([]api.TaskDefinition, return taskDefinitions, nil } -// yamlFilePathsInDirectory returns any *.yml and *.yaml files in a given directory, ignoring any sub-directories. -func (s Service) yamlFilePathsInDirectory(dir string) ([]string, error) { - paths := make([]string, 0) +func (s Service) mintDirectoryEntries(dir string) ([]api.MintDirectoryEntry, error) { + mintDirectoryEntries := make([]api.MintDirectoryEntry, 0) + contentLength := 0 - files, err := os.ReadDir(dir) + err := filepath.Walk(dir, func(pathInDir string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error reading %q: %w", pathInDir, err) + } + + entry, entryContentLength, err := s.mintDirectoryEntry(pathInDir, info, dir) + if err != nil { + return err + } + contentLength += entryContentLength + mintDirectoryEntries = append(mintDirectoryEntries, entry) + + return nil + }) if err != nil { - return nil, errors.Wrapf(err, "unable to read %q", dir) + return nil, fmt.Errorf("unable to retrieve the entire contents of the .mint directory %q: %w", dir, err) } + if contentLength > 5*1024*1024 { + return nil, fmt.Errorf("the size of the .mint directory at %q exceeds 5MiB", dir) + } + + return mintDirectoryEntries, nil +} + +// taskDefinitionsFromPaths opens each file specified in `paths` and reads their content as a string. +// No validation takes place here. +func (s Service) mintDirectoryEntriesFromPaths(paths []string) ([]api.MintDirectoryEntry, error) { + mintDirectoryEntries := make([]api.MintDirectoryEntry, 0) + + for _, path := range paths { + fd, err := os.Open(path) + if err != nil { + return nil, errors.Wrapf(err, "error while opening %q", path) + } + defer fd.Close() + + info, err := os.Lstat(path) + if err != nil { + return nil, errors.Wrapf(err, "error while stating %q", path) + } + + entry, _, err := s.mintDirectoryEntry(path, info, "") + if err != nil { + return nil, err + } + + mintDirectoryEntries = append(mintDirectoryEntries, entry) + } + + return mintDirectoryEntries, nil +} + +func (s Service) mintDirectoryEntry(path string, info os.FileInfo, makePathRelativeTo string) (api.MintDirectoryEntry, int, error) { + mode := info.Mode() + permissions := mode.Perm() + + var entryType string + switch mode.Type() { + case os.ModeDir: + entryType = "dir" + case os.ModeSymlink: + entryType = "symlink" + case os.ModeNamedPipe: + entryType = "named-pipe" + case os.ModeSocket: + entryType = "socket" + case os.ModeDevice: + entryType = "device" + case os.ModeCharDevice: + entryType = "char-device" + case os.ModeIrregular: + entryType = "irregular" + default: + if mode.IsRegular() { + entryType = "file" + } else { + entryType = "unknown" + } + } + + var fileContents string + var contentLength int + if entryType == "file" { + contents, err := os.ReadFile(path) + if err != nil { + return api.MintDirectoryEntry{}, contentLength, fmt.Errorf("unable to read file %q: %w", path, err) + } + + contentLength = len(contents) + fileContents = string(contents) + } + + relPath := path + if makePathRelativeTo != "" { + rel, err := filepath.Rel(makePathRelativeTo, path) + if err != nil { + return api.MintDirectoryEntry{}, contentLength, fmt.Errorf("unable to determine relative path of %q: %w", path, err) + } + relPath = filepath.ToSlash(filepath.Join(".mint", rel)) // Mint only supports unix-style path separators + } + + return api.MintDirectoryEntry{ + Type: entryType, + OriginalPath: path, + Path: relPath, + Permissions: uint32(permissions), + FileContents: fileContents, + }, contentLength, nil +} + +// yamlFilesInMintDirectory returns any *.yml and *.yaml files in a given mint directory +func (s Service) yamlFilesInMintDirectory(mintDirectory []api.MintDirectoryEntry) []api.MintDirectoryEntry { + yamlEntries := make([]api.MintDirectoryEntry, 0) - for _, file := range files { - if file.IsDir() { + for _, entry := range mintDirectory { + if entry.Type != "file" { continue } - if !strings.HasSuffix(file.Name(), ".yml") && !strings.HasSuffix(file.Name(), ".yaml") { + if !strings.HasSuffix(entry.OriginalPath, ".yml") && !strings.HasSuffix(entry.OriginalPath, ".yaml") { continue } - paths = append(paths, filepath.Join(dir, file.Name())) + yamlEntries = append(yamlEntries, entry) } - return paths, nil + return yamlEntries } -// yamlFilePathsInDirectory returns any *.yml and *.yaml files in a given directory, ignoring any sub-directories. func (s Service) findMintDirectoryPath(configuredDirectory string) (string, error) { if configuredDirectory != "" { return configuredDirectory, nil diff --git a/internal/cli/service_test.go b/internal/cli/service_test.go index 9e20848..55b2a2c 100644 --- a/internal/cli/service_test.go +++ b/internal/cli/service_test.go @@ -82,13 +82,12 @@ var _ = Describe("CLI Service", func() { var originalSpecifiedFileContent string var originalMintDirFileContent string var receivedSpecifiedFileContent string - var receivedMintDirFileContent string + var receivedMintDir []api.MintDirectoryEntry BeforeEach(func() { originalSpecifiedFileContent = "tasks:\n - key: foo\n run: echo 'bar'\n" originalMintDirFileContent = "tasks:\n - key: mintdir\n run: echo 'mintdir'\n" receivedSpecifiedFileContent = "" - receivedMintDirFileContent = "" var err error @@ -112,17 +111,30 @@ var _ = Describe("CLI Service", func() { err = os.WriteFile(filepath.Join(mintDir, "mintdir-tasks.json"), []byte("some json"), 0o644) Expect(err).NotTo(HaveOccurred()) + nestedDir := filepath.Join(mintDir, "some", "nested", "path") + err = os.MkdirAll(nestedDir, 0o755) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(filepath.Join(nestedDir, "tasks.yaml"), []byte("some nested yaml"), 0o644) + Expect(err).NotTo(HaveOccurred()) + runConfig.MintFilePath = "mint.yml" runConfig.MintDirectory = "" mockAPI.MockInitiateRun = func(cfg api.InitiateRunConfig) (*api.InitiateRunResult, error) { Expect(cfg.TaskDefinitions).To(HaveLen(1)) Expect(cfg.TaskDefinitions[0].Path).To(Equal(runConfig.MintFilePath)) - Expect(cfg.MintDirectory).To(HaveLen(1)) - Expect(cfg.MintDirectory[0].Path).To(Equal(".mint/mintdir-tasks.yml")) + Expect(cfg.MintDirectory).To(HaveLen(7)) + Expect(cfg.MintDirectory[0].Path).To(Equal(".mint")) + Expect(cfg.MintDirectory[1].Path).To(Equal(".mint/mintdir-tasks.json")) + Expect(cfg.MintDirectory[2].Path).To(Equal(".mint/mintdir-tasks.yml")) + Expect(cfg.MintDirectory[3].Path).To(Equal(".mint/some")) + Expect(cfg.MintDirectory[4].Path).To(Equal(".mint/some/nested")) + Expect(cfg.MintDirectory[5].Path).To(Equal(".mint/some/nested/path")) + Expect(cfg.MintDirectory[6].Path).To(Equal(".mint/some/nested/path/tasks.yaml")) Expect(cfg.UseCache).To(BeTrue()) receivedSpecifiedFileContent = cfg.TaskDefinitions[0].FileContents - receivedMintDirFileContent = cfg.MintDirectory[0].FileContents + receivedMintDir = cfg.MintDirectory return &api.InitiateRunResult{ RunId: "785ce4e8-17b9-4c8b-8869-a55e95adffe7", RunURL: "https://cloud.rwx.com/mint/rwx/runs/785ce4e8-17b9-4c8b-8869-a55e95adffe7", @@ -139,7 +151,14 @@ var _ = Describe("CLI Service", func() { It("sends the file contents to cloud", func() { Expect(receivedSpecifiedFileContent).To(Equal(originalSpecifiedFileContent)) - Expect(receivedMintDirFileContent).To(Equal(originalMintDirFileContent)) + Expect(receivedMintDir).NotTo(BeNil()) + Expect(receivedMintDir[0].FileContents).To(Equal("")) + Expect(receivedMintDir[1].FileContents).To(Equal("some json")) + Expect(receivedMintDir[2].FileContents).To(Equal(originalMintDirFileContent)) + Expect(receivedMintDir[3].FileContents).To(Equal("")) + Expect(receivedMintDir[4].FileContents).To(Equal("")) + Expect(receivedMintDir[5].FileContents).To(Equal("")) + Expect(receivedMintDir[6].FileContents).To(Equal("some nested yaml")) }) }) @@ -173,7 +192,8 @@ var _ = Describe("CLI Service", func() { mockAPI.MockInitiateRun = func(cfg api.InitiateRunConfig) (*api.InitiateRunResult, error) { Expect(cfg.TaskDefinitions).To(HaveLen(1)) Expect(cfg.TaskDefinitions[0].Path).To(Equal(runConfig.MintFilePath)) - Expect(cfg.MintDirectory).To(HaveLen(0)) + Expect(cfg.MintDirectory).To(HaveLen(1)) + Expect(cfg.MintDirectory[0].Path).To(Equal(".mint")) Expect(cfg.UseCache).To(BeTrue()) receivedSpecifiedFileContent = cfg.TaskDefinitions[0].FileContents return &api.InitiateRunResult{ @@ -262,13 +282,12 @@ var _ = Describe("CLI Service", func() { var originalSpecifiedFileContent string var originalMintDirFileContent string var receivedSpecifiedFileContent string - var receivedMintDirFileContent string + var receivedMintDir []api.MintDirectoryEntry BeforeEach(func() { originalSpecifiedFileContent = "tasks:\n - key: foo\n run: echo 'bar'\n" originalMintDirFileContent = "tasks:\n - key: mintdir\n run: echo 'mintdir'\n" receivedSpecifiedFileContent = "" - receivedMintDirFileContent = "" var err error @@ -299,11 +318,13 @@ var _ = Describe("CLI Service", func() { mockAPI.MockInitiateRun = func(cfg api.InitiateRunConfig) (*api.InitiateRunResult, error) { Expect(cfg.TaskDefinitions).To(HaveLen(1)) Expect(cfg.TaskDefinitions[0].Path).To(Equal(runConfig.MintFilePath)) - Expect(cfg.MintDirectory).To(HaveLen(1)) - Expect(cfg.MintDirectory[0].Path).To(Equal(".mint/mintdir-tasks.yml")) + Expect(cfg.MintDirectory).To(HaveLen(3)) + Expect(cfg.MintDirectory[0].Path).To(Equal(".mint")) + Expect(cfg.MintDirectory[1].Path).To(Equal(".mint/mintdir-tasks.json")) + Expect(cfg.MintDirectory[2].Path).To(Equal(".mint/mintdir-tasks.yml")) Expect(cfg.UseCache).To(BeTrue()) receivedSpecifiedFileContent = cfg.TaskDefinitions[0].FileContents - receivedMintDirFileContent = cfg.MintDirectory[0].FileContents + receivedMintDir = cfg.MintDirectory return &api.InitiateRunResult{ RunId: "785ce4e8-17b9-4c8b-8869-a55e95adffe7", RunURL: "https://cloud.rwx.com/mint/rwx/runs/785ce4e8-17b9-4c8b-8869-a55e95adffe7", @@ -320,7 +341,10 @@ var _ = Describe("CLI Service", func() { It("sends the file contents to cloud", func() { Expect(receivedSpecifiedFileContent).To(Equal(originalSpecifiedFileContent)) - Expect(receivedMintDirFileContent).To(Equal(originalMintDirFileContent)) + Expect(receivedMintDir).NotTo(BeNil()) + Expect(receivedMintDir[0].FileContents).To(Equal("")) + Expect(receivedMintDir[1].FileContents).To(Equal("some json")) + Expect(receivedMintDir[2].FileContents).To(Equal(originalMintDirFileContent)) }) }) @@ -355,7 +379,8 @@ var _ = Describe("CLI Service", func() { mockAPI.MockInitiateRun = func(cfg api.InitiateRunConfig) (*api.InitiateRunResult, error) { Expect(cfg.TaskDefinitions).To(HaveLen(1)) Expect(cfg.TaskDefinitions[0].Path).To(Equal(runConfig.MintFilePath)) - Expect(cfg.MintDirectory).To(HaveLen(0)) + Expect(cfg.MintDirectory).To(HaveLen(1)) + Expect(cfg.MintDirectory[0].Path).To(Equal(".mint")) Expect(cfg.UseCache).To(BeTrue()) receivedSpecifiedFileContent = cfg.TaskDefinitions[0].FileContents return &api.InitiateRunResult{ @@ -1132,6 +1157,17 @@ AAAEC6442PQKevgYgeT0SIu9zwlnEMl6MF59ZgM+i0ByMv4eLJPqG3xnZcEQmktHj/GY2i call: mint/setup-node 1.2.3 `), 0o644) Expect(err).NotTo(HaveOccurred()) + + nestedDir := filepath.Join(mintDir, "some", "nested", "dir") + err = os.MkdirAll(nestedDir, 0o755) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(filepath.Join(nestedDir, "tasks.yaml"), []byte(` + tasks: + - key: foo + call: mint/setup-node 1.2.3 + `), 0o644) + Expect(err).NotTo(HaveOccurred()) }) BeforeEach(func() { @@ -1156,11 +1192,15 @@ AAAEC6442PQKevgYgeT0SIu9zwlnEMl6MF59ZgM+i0ByMv4eLJPqG3xnZcEQmktHj/GY2i var contents []byte - contents, err = os.ReadFile(filepath.Join(tmp, "bar.yaml")) + contents, err = os.ReadFile(filepath.Join(mintDir, "bar.yaml")) Expect(err).NotTo(HaveOccurred()) Expect(string(contents)).To(ContainSubstring("mint/setup-node 1.3.0")) - contents, err = os.ReadFile(filepath.Join(tmp, "baz.yaml")) + contents, err = os.ReadFile(filepath.Join(mintDir, "baz.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(contents)).To(ContainSubstring("mint/setup-node 1.3.0")) + + contents, err = os.ReadFile(filepath.Join(mintDir, "some", "nested", "dir", "tasks.yaml")) Expect(err).NotTo(HaveOccurred()) Expect(string(contents)).To(ContainSubstring("mint/setup-node 1.3.0")) }) @@ -1796,5 +1836,31 @@ Checked 2 files and found 4 problems. }) }) }) + + Context("when specific files are not targeted", func() { + var lintedDefinitions []api.TaskDefinition + + BeforeEach(func() { + Expect(os.WriteFile("mint1.yml", []byte("mint1 contents"), 0o644)).NotTo(HaveOccurred()) + Expect(os.WriteFile(".mint/base.yml", []byte(".mint/base.yml contents"), 0o644)).NotTo(HaveOccurred()) + Expect(os.WriteFile(".mint/base.json", []byte(".mint/base.json contents"), 0o644)).NotTo(HaveOccurred()) + Expect(os.MkdirAll(".mint/some/nested/dir", 0o755)).NotTo(HaveOccurred()) + Expect(os.WriteFile(".mint/some/nested/dir/tasks.yml", []byte(".mint/some/nested/dir/tasks.yml contents"), 0o644)).NotTo(HaveOccurred()) + Expect(os.WriteFile(".mint/some/nested/dir/tasks.json", []byte(".mint/some/nested/dir/tasks.json contents"), 0o644)).NotTo(HaveOccurred()) + + mockAPI.MockLint = func(cfg api.LintConfig) (*api.LintResult, error) { + lintedDefinitions = cfg.TaskDefinitions + return &api.LintResult{Problems: []api.LintProblem{}}, nil + } + }) + + It("targets yaml files in the .mint dir recursively", func() { + _, err := service.Lint(lintConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(lintedDefinitions).To(HaveLen(2)) + Expect(lintedDefinitions[0].Path).To(Equal(".mint/base.yml")) + Expect(lintedDefinitions[1].Path).To(Equal(".mint/some/nested/dir/tasks.yml")) + }) + }) }) })