From f37011b03287caf55dcb85c3538d000ddd5d68dc Mon Sep 17 00:00:00 2001 From: Aron Kyle Date: Mon, 22 Jul 2024 10:33:38 -0400 Subject: [PATCH] Updated COMPOSE_ENV_FILES in env files Updated the docker compose command to recursively search through any specified .env files for the COMPOSE_ENV_FILES parameter. - When found, the paths are followed recursively to extend the list of .env files used in the command. - A cache is kept that prevents circular dependencies. - Recursion is depth first. Signed-off-by: Aron Kyle --- .gitignore | 1 + cmd/compose/compose.go | 84 ++++++++++++++++++- pkg/e2e/compose_environment_test.go | 35 ++++++++ .../fixtures/environment/env-recursion/.env | 2 + .../fixtures/environment/env-recursion/.env.2 | 2 + .../fixtures/environment/env-recursion/.env.3 | 2 + .../env-recursion/.env.test-missing | 2 + .../env-recursion/.env.test-missing.2 | 2 + .../env-recursion/.env.test-missing.3 | 1 + .../environment/env-recursion/Dockerfile | 17 ++++ .../environment/env-recursion/compose.yaml | 5 ++ 11 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 pkg/e2e/fixtures/environment/env-recursion/.env create mode 100644 pkg/e2e/fixtures/environment/env-recursion/.env.2 create mode 100644 pkg/e2e/fixtures/environment/env-recursion/.env.3 create mode 100644 pkg/e2e/fixtures/environment/env-recursion/.env.test-missing create mode 100644 pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.2 create mode 100644 pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.3 create mode 100644 pkg/e2e/fixtures/environment/env-recursion/Dockerfile create mode 100644 pkg/e2e/fixtures/environment/env-recursion/compose.yaml diff --git a/.gitignore b/.gitignore index 7bfe97f718..5b12b9f63f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ bin/ coverage.out covdatafiles/ .DS_Store +/.idea diff --git a/cmd/compose/compose.go b/cmd/compose/compose.go index 17008a1751..e6efc2e339 100644 --- a/cmd/compose/compose.go +++ b/cmd/compose/compose.go @@ -647,12 +647,12 @@ func RootCommand(dockerCli command.Cli, backend Backend) *cobra.Command { //noli c.Flags().MarkHidden("verbose") //nolint:errcheck return c } - func setEnvWithDotEnv(opts ProjectOptions) error { options, err := cli.NewProjectOptions(opts.ConfigPaths, cli.WithWorkingDirectory(opts.ProjectDir), cli.WithOsEnv, cli.WithEnvFiles(opts.EnvFiles...), + WithExtendedEnvFiles, cli.WithDotEnv, ) if err != nil { @@ -672,6 +672,88 @@ func setEnvWithDotEnv(opts ProjectOptions) error { return err } +func WithExtendedEnvFiles(o *cli.ProjectOptions) error { + + absEnvFiles := make([]string, 0) + fileLookupCache := make(map[string]string) + + for _, file := range o.EnvFiles { + absFile, err := filepath.Abs(file) + if err != nil { + return err + } + absEnvFiles = append(absEnvFiles, absFile) + fileLookupCache[absFile] = absFile + + recursedFiles, err := recurseEnvFiles(absFile, fileLookupCache) + if err != nil { + return err + } + absEnvFiles = append(absEnvFiles, recursedFiles...) + } + o.EnvFiles = absEnvFiles + + return nil +} + +func recurseEnvFiles(envFile string, fileLookup map[string]string) ([]string, error) { + + newEnvFiles := make([]string, 0) + + _, err := os.Stat(envFile) + // This indicates that the specified file does not exist + // In this specific case it's safe to ignore loading the file i.e. the file is optional. + if os.IsNotExist(err) { + return newEnvFiles, nil + } + + // Parse the .env file + envFromFile, err := dotenv.GetEnvFromFile(make(map[string]string), []string{envFile}) + if err != nil { + return nil, err + } + + // If the file contains a COMPOSE_ENV_FILES key, add the files to the list. + // Remote any files that don't exist i.e. the file is optional. + // Filter any files we've already seen in the fileLookup. + // Depth first recursion into the new files + if extraEnvFiles, ok := envFromFile[ComposeEnvFiles]; ok { + + for _, newFile := range strings.Split(extraEnvFiles, ",") { + // Handle relative paths + if !filepath.IsAbs(newFile) { + newFile, err = filepath.Abs(filepath.Join(filepath.Dir(envFile), newFile)) + if err != nil { + return nil, err + } + } + + _, err := os.Stat(newFile) + // This indicates that the specified file does not exist + // In this specific case it's safe to ignore using the file as an env file + // i.e. the file is optional. + if os.IsNotExist(err) { + continue + } + + // if we haven't seen this file before, add it to the list + // and recurse into it + if _, ok := fileLookup[newFile]; !ok { + newEnvFiles = append(newEnvFiles, newFile) + fileLookup[newFile] = newFile + recursedFiles, recurseErr := recurseEnvFiles(newFile, fileLookup) + if recurseErr != nil { + return nil, recurseErr + } + + newEnvFiles = append(newEnvFiles, recursedFiles...) + } + } + } + + return newEnvFiles, nil +} + var printerModes = []string{ ui.ModeAuto, ui.ModeTTY, diff --git a/pkg/e2e/compose_environment_test.go b/pkg/e2e/compose_environment_test.go index 62d6c9ebc5..0ecdc1c50c 100644 --- a/pkg/e2e/compose_environment_test.go +++ b/pkg/e2e/compose_environment_test.go @@ -171,6 +171,41 @@ func TestEnvPriority(t *testing.T) { assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File") }) + // Recursing through multiple env files based on the COMPOSE_ENV_FILES variable + // Chain of env files: + // 1. .env + // 2. .env.2 + // 3. .env.3 + t.Run("recurse env files with COMPOSE_ENV_FILES", func(t *testing.T) { + res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-recursion/compose.yaml", + "run", "--rm", "-e", "WHEREAMI", "env-compose-recursion") + assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File 3") + }) + + // Recursing through multiple env files based on the COMPOSE_ENV_FILES variable + // Chain of env files: + // 1. .env.3 + // 2. .env + // 3. .env.2 + t.Run("recurse env files with COMPOSE_ENV_FILES with --env-file", func(t *testing.T) { + res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-recursion/compose.yaml", + "--env-file", "./fixtures/environment/env-recursion/.env.3", + "run", "--rm", "-e", "WHEREAMI", "env-compose-recursion") + assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File 2") + }) + // Recursing through multiple env files based on the COMPOSE_ENV_FILES variable with missing file + // Chain of env files: + // 1. .env.test-missing + // 2. .env.test-missing.2 + // 3. .env.test-missing.idontexist -> Skipped because it does not exist + // 4. .env.test-missing.3 + t.Run("recurse env files with COMPOSE_ENV_FILES with --env-file and missing file", func(t *testing.T) { + res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-recursion/compose.yaml", + "--env-file", "./fixtures/environment/env-recursion/.env.test-missing", + "run", "--rm", "-e", "WHEREAMI", "env-compose-recursion") + assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File Test Missing 3") + }) + // No Compose file & no env variable, using an empty override env file // 1. Command Line (docker compose run --env ) // 2. Compose File (service::environment section) diff --git a/pkg/e2e/fixtures/environment/env-recursion/.env b/pkg/e2e/fixtures/environment/env-recursion/.env new file mode 100644 index 0000000000..af12bbe898 --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/.env @@ -0,0 +1,2 @@ +COMPOSE_ENV_FILES=./.env.2 +WHEREAMI="Env File" diff --git a/pkg/e2e/fixtures/environment/env-recursion/.env.2 b/pkg/e2e/fixtures/environment/env-recursion/.env.2 new file mode 100644 index 0000000000..6b7753fc69 --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/.env.2 @@ -0,0 +1,2 @@ +COMPOSE_ENV_FILES=.env.3,.env.2 +WHEREAMI="Env File 2" diff --git a/pkg/e2e/fixtures/environment/env-recursion/.env.3 b/pkg/e2e/fixtures/environment/env-recursion/.env.3 new file mode 100644 index 0000000000..8573b2c15e --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/.env.3 @@ -0,0 +1,2 @@ +COMPOSE_ENV_FILES=.env +WHEREAMI="Env File 3" diff --git a/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing b/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing new file mode 100644 index 0000000000..d105ad35a3 --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing @@ -0,0 +1,2 @@ +COMPOSE_ENV_FILES=.env.test-missing.2 +WHEREAMI="Env File Test Missing" diff --git a/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.2 b/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.2 new file mode 100644 index 0000000000..4226cb7c7a --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.2 @@ -0,0 +1,2 @@ +COMPOSE_ENV_FILES=.env.test-missing.idontexist,.env.test-missing.3 +WHEREAMI="Env File Test Missing 2" diff --git a/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.3 b/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.3 new file mode 100644 index 0000000000..054934f084 --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/.env.test-missing.3 @@ -0,0 +1 @@ +WHEREAMI="Env File Test Missing 3" diff --git a/pkg/e2e/fixtures/environment/env-recursion/Dockerfile b/pkg/e2e/fixtures/environment/env-recursion/Dockerfile new file mode 100644 index 0000000000..0901119f7d --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/Dockerfile @@ -0,0 +1,17 @@ +# Copyright 2020 Docker Compose CLI authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine +ENV WHEREAMI=Dockerfile +CMD ["printenv", "WHEREAMI"] diff --git a/pkg/e2e/fixtures/environment/env-recursion/compose.yaml b/pkg/e2e/fixtures/environment/env-recursion/compose.yaml new file mode 100644 index 0000000000..66369e7d8b --- /dev/null +++ b/pkg/e2e/fixtures/environment/env-recursion/compose.yaml @@ -0,0 +1,5 @@ +services: + env-compose-recursion: + image: env-compose-recursion + build: + context: .