Skip to content

Commit

Permalink
Merge pull request #51 from uselagoon/envfile-checks
Browse files Browse the repository at this point in the history
  • Loading branch information
shreddedbacon authored Jun 16, 2022
2 parents 7805ec3 + 7fb04e4 commit 2f27606
Show file tree
Hide file tree
Showing 14 changed files with 494 additions and 20 deletions.
2 changes: 1 addition & 1 deletion cmd/helpers_values.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func collectBuildValues(debug bool, activeEnv, standbyEnv *bool,

// lCompose := composetypes.Project{}
// unmarshal the docker-compose.yml file
lCompose, err := lagoon.UnmarshaDockerComposeYAML(lYAML.DockerComposeYAML, ignoreNonStringKeyErrors, composeVars)
lCompose, err := lagoon.UnmarshaDockerComposeYAML(lYAML.DockerComposeYAML, ignoreNonStringKeyErrors, ignoreMissingEnvFiles, composeVars)
if err != nil {
return err
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ func init() {
"The fastly secret prefix to use")
rootCmd.PersistentFlags().BoolVarP(&ignoreNonStringKeyErrors, "ignore-non-string-key-errors", "", true,
"Ignore non-string-key docker-compose errors (true by default, subject to change).")
rootCmd.PersistentFlags().BoolVarP(&ignoreMissingEnvFiles, "ignore-missing-env-files", "", true,
"Ignore missing env_file files (true by default, subject to change).")
}

// initConfig reads in config file and ENV variables if set.
Expand Down
20 changes: 20 additions & 0 deletions cmd/template_autogen_ingress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,26 @@ func TestAutogeneratedIngressGeneration(t *testing.T) {
emptyDir: false,
want: "../test-resources/template-autogenerated/test20-results",
},
{
name: "test21 autogenerated routes where docker-compose env_file has missing file references",
args: args{
alertContact: "alertcontact",
statusPageID: "statuspageid",
projectName: "test21-example-com",
environmentName: "feature",
environmentType: "development",
buildType: "branch",
lagoonVersion: "v2.7.x",
branch: "feature",
projectVars: `[{"name":"LAGOON_SYSTEM_ROUTER_PATTERN","value":"${environment}.${project}.example.com","scope":"internal_system"}]`,
envVars: `[]`,
secretPrefix: "fastly-api-",
lagoonYAML: "../test-resources/template-autogenerated/test21/lagoon.yml",
templatePath: "../test-resources/template-autogenerated/output",
},
emptyDir: false,
want: "../test-resources/template-autogenerated/test21-results",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
7 changes: 4 additions & 3 deletions cmd/validate_compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
var (
dockerComposeFile string
ignoreNonStringKeyErrors bool
ignoreMissingEnvFiles bool
)

var validateDockerCompose = &cobra.Command{
Expand All @@ -20,7 +21,7 @@ var validateDockerCompose = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
// @TODO: ignoreNonStringKeyErrors is `true` by default because Lagoon doesn't enforce
// docker-compose compliance yet
err := ValidateDockerCompose(dockerComposeFile, ignoreNonStringKeyErrors)
err := ValidateDockerCompose(dockerComposeFile, ignoreNonStringKeyErrors, ignoreMissingEnvFiles)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
Expand All @@ -29,8 +30,8 @@ var validateDockerCompose = &cobra.Command{
}

// ValidateDockerCompose validate a docker-compose file
func ValidateDockerCompose(file string, ignoreErrors bool) error {
_, err := lagoon.UnmarshaDockerComposeYAML(file, ignoreNonStringKeyErrors, map[string]string{})
func ValidateDockerCompose(file string, ignoreErrors, ignoreMisEnvFiles bool) error {
_, err := lagoon.UnmarshaDockerComposeYAML(file, ignoreErrors, ignoreMisEnvFiles, map[string]string{})
if err != nil {
return err
}
Expand Down
30 changes: 24 additions & 6 deletions cmd/validate_compose_test.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package cmd

import (
"strings"
"testing"
)

func TestValidateDockerCompose(t *testing.T) {
type args struct {
file string
ignoreErrors bool
file string
ignoreNonStringKeyErrors bool
ignoreMissingEnvFiles bool
}
tests := []struct {
name string
Expand Down Expand Up @@ -50,16 +52,32 @@ func TestValidateDockerCompose(t *testing.T) {
{
name: "test6 check an invalid docker-compose (same as test5 but ignoring the errors)",
args: args{
file: "../test-resources/docker-compose/test8/docker-compose.yml",
ignoreErrors: true,
file: "../test-resources/docker-compose/test8/docker-compose.yml",
ignoreNonStringKeyErrors: true,
},
},
{
name: "test7 check an valid docker-compose with missing env_files ",
args: args{
file: "../test-resources/docker-compose/test10/docker-compose.yml",
},
wantErr: true,
wantErrMsg: "no such file or directory",
},
{
name: "test8 check an valid docker-compose with missing env_files (same as test7 but ignoring the missing file errors)",
args: args{
file: "../test-resources/docker-compose/test9/docker-compose.yml",
ignoreNonStringKeyErrors: true,
ignoreMissingEnvFiles: true,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := ValidateDockerCompose(tt.args.file, tt.args.ignoreErrors); err != nil {
if err := ValidateDockerCompose(tt.args.file, tt.args.ignoreNonStringKeyErrors, tt.args.ignoreMissingEnvFiles); err != nil {
if tt.wantErr {
if err.Error() != tt.wantErrMsg {
if !strings.Contains(err.Error(), tt.wantErrMsg) {
t.Errorf("ValidateDockerCompose() error = %v, wantErr %v", err, tt.wantErr)
}
} else {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,6 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
)

replace github.com/compose-spec/compose-go v1.2.7 => github.com/shreddedbacon/compose-go v0.0.0-20220610063019-17f447c8c1aa
replace github.com/compose-spec/compose-go v1.2.7 => github.com/shreddedbacon/compose-go v0.0.0-20220616064547-4e908a2865c1

// replace github.com/compose-spec/compose-go v1.2.7 => ../../compose-spec/compose-go
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -296,8 +296,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/shreddedbacon/compose-go v0.0.0-20220610063019-17f447c8c1aa h1:vVqWIv8lPg7H3GgSYAqOfVLT5MzFVr/WPxeXRWAGqbk=
github.com/shreddedbacon/compose-go v0.0.0-20220610063019-17f447c8c1aa/go.mod h1:Jl9L8zJrt4aGY1XAz03DvHAu8V3/f00TK+uJL4BayDU=
github.com/shreddedbacon/compose-go v0.0.0-20220616064547-4e908a2865c1 h1:/OYATO27ywKZ506enKV8Fd8gjLYFU9uQqCMmcVOUpW0=
github.com/shreddedbacon/compose-go v0.0.0-20220616064547-4e908a2865c1/go.mod h1:Jl9L8zJrt4aGY1XAz03DvHAu8V3/f00TK+uJL4BayDU=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
Expand Down
4 changes: 3 additions & 1 deletion internal/lagoon/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import (
)

// UnmarshaDockerComposeYAML unmarshal the lagoon.yml file into a YAML and map for consumption.
func UnmarshaDockerComposeYAML(file string, ignoreErrors bool, envvars map[string]string) (*composetypes.Project, error) {
func UnmarshaDockerComposeYAML(file string, ignoreErrors, ignoreMissingEnvFiles bool, envvars map[string]string) (*composetypes.Project, error) {
options, err := cli.NewProjectOptions([]string{file},
cli.WithResolvedPaths(false),
cli.WithLoadOptions(
loader.WithSkipValidation,
loader.WithDiscardEnvFiles,
func(o *loader.Options) {
o.IgnoreNonStringKeyErrors = ignoreErrors
o.IgnoreMissingEnvFileCheck = ignoreMissingEnvFiles
},
),
)
Expand Down
31 changes: 25 additions & 6 deletions internal/lagoon/compose_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,17 @@ package lagoon

import (
"encoding/json"
"strings"
"testing"

"github.com/google/go-cmp/cmp"
)

func TestUnmarshaDockerComposeYAML(t *testing.T) {
type args struct {
file string
ignoreErrors bool
file string
ignoreNonStringKeyErrors bool
ignoreMissingEnvFiles bool
}
tests := []struct {
name string
Expand Down Expand Up @@ -64,8 +66,8 @@ func TestUnmarshaDockerComposeYAML(t *testing.T) {
{
name: "test7 check an invalid docker-compose with ignoring non-string key errors",
args: args{
file: "../../test-resources/docker-compose/test7/docker-compose.yml",
ignoreErrors: true,
file: "../../test-resources/docker-compose/test7/docker-compose.yml",
ignoreNonStringKeyErrors: true,
},
want: `{"name":"test7","services":{"cli":{"build":{"context":".","dockerfile":".lagoon/cli.dockerfile","args":{"DOCKER_CLI_IMAGE_URI":"","ENVIRONMENT_TYPE_ID":""}},"container_name":"_cli","environment":{"ENVIRONMENT_TYPE_ID":"","LAGOON_ENVIRONMENT_TYPE":"","LAGOON_PROJECT":"","LAGOON_ROUTE":"http://","PHP_MEMORY_LIMIT":"768M","XDEBUG_ENABLE":""},"labels":{"lagoon.persistent":"/app/docroot/sites/default/files/","lagoon.persistent.name":"nginx","lagoon.type":"cli-persistent"},"networks":{"default":null},"user":"root","volumes":[{"type":"bind","source":"./.lagoon/scripts/bash_prompts.rc","target":"/home/.bashrc","bind":{"create_host_path":true}},{"type":"bind","source":"./.lagoon/scripts/color_grid.sh","target":"/home/color_grid.sh","bind":{"create_host_path":true}}],"volumes_from":["container:amazeeio-ssh-agent"]},"mariadb":{"container_name":"_db","environment":{"ENVIRONMENT_TYPE_ID":"","LAGOON_ENVIRONMENT_TYPE":"","LAGOON_PROJECT":"","LAGOON_ROUTE":"http://","PHP_MEMORY_LIMIT":"768M","XDEBUG_ENABLE":""},"image":"amazeeio/mariadb-drupal","labels":{"lagoon.type":"mariadb"},"networks":{"default":null},"ports":[{"mode":"ingress","target":3306,"protocol":"tcp"}],"volumes":[{"type":"volume","source":"mysql","target":"/var/lib/mysql","volume":{}}]},"nginx":{"build":{"context":".","dockerfile":".lagoon/nginx.dockerfile","args":{"CLI_IMAGE":"","DOCKER_NGINX_IMAGE_URI":"","LAGOON_GIT_BRANCH":null}},"container_name":"_nginx","depends_on":{"cli":{"condition":"service_started"}},"environment":{"ENVIRONMENT_TYPE_ID":"","LAGOON_ENVIRONMENT_TYPE":"","LAGOON_LOCALDEV_URL":"http://","LAGOON_PROJECT":"","LAGOON_ROUTE":"http://","PHP_MEMORY_LIMIT":"768M","XDEBUG_ENABLE":""},"labels":{"lagoon.name":"nginx","lagoon.persistent":"/app/docroot/sites/default/files/","lagoon.type":"nginx-php-persistent"},"networks":{"amazeeio-network":null,"default":null},"volumes":[{"type":"bind","source":"./.lagoon/nginx/nginx-http.conf","target":"/etc/nginx/conf.d/000-nginx-http.conf","bind":{"create_host_path":true}},{"type":"bind","source":"./.lagoon/nginx/app.conf","target":"/etc/nginx/conf.d/app.conf","bind":{"create_host_path":true}}]},"php":{"build":{"context":".","dockerfile":".lagoon/php.dockerfile","args":{"CLI_IMAGE":"","DOCKER_PHP_IMAGE_URI":""}},"container_name":"_php","depends_on":{"cli":{"condition":"service_started"}},"environment":{"ENVIRONMENT_TYPE_ID":"","LAGOON_ENVIRONMENT_TYPE":"","LAGOON_PROJECT":"","LAGOON_ROUTE":"http://","PHP_MEMORY_LIMIT":"768M","XDEBUG_ENABLE":""},"labels":{"lagoon.deployment.servicetype":"php","lagoon.name":"nginx","lagoon.persistent":"/app/docroot/sites/default/files","lagoon.type":"nginx-php-persistent"},"networks":{"default":null}}},"networks":{"amazeeio-network":{"name":"amazeeio-network","ipam":{},"external":true},"default":{"name":"test7_default","ipam":{},"external":false}},"volumes":{"app":{"name":"test7_app","external":false},"mysql":{"name":"test7_mysql","external":false},"solr7":{"name":"test7_solr7","external":false}}}`,
},
Expand All @@ -77,15 +79,32 @@ func TestUnmarshaDockerComposeYAML(t *testing.T) {
wantErr: true,
wantErrMsg: "Non-string key in x-site-branch: <nil>",
},
{
name: "test9 check an valid docker-compose with missing env_files",
args: args{
file: "../../test-resources/docker-compose/test9/docker-compose.yml",
ignoreNonStringKeyErrors: true,
ignoreMissingEnvFiles: true,
},
want: `{"name":"test9","services":{"cli":{"build":{"context":".","dockerfile":"lagoon/cli.dockerfile"},"container_name":"-cli","environment":{"DRUSH_OPTIONS_URI":"https://","LAGOON_PROJECT":"","LAGOON_ROUTE":"https://","SIMPLETEST_BASE_URL":"http://nginx:8080","SIMPLETEST_DB":"mysql://drupal:drupal@mariadb:3306/drupal","SSMTP_MAILHUB":"host.docker.internal:1025"},"labels":{"lagoon.persistent":"/app/public/sites/default/files/","lagoon.persistent.name":"nginx","lagoon.type":"cli-persistent"},"networks":{"default":null},"volumes":[{"type":"bind","source":".","target":"/app","bind":{"create_host_path":true}},{"type":"volume","source":"ssh","target":"/tmp/amazeeio_ssh-agent","volume":{}}]},"mariadb":{"container_name":"-db","environment":{"LAGOON_PROJECT":"","LAGOON_ROUTE":"https://","SSMTP_MAILHUB":"host.docker.internal:1025"},"image":"uselagoon/mariadb-drupal:latest","labels":{"lagoon.type":"mariadb"},"networks":{"default":null},"ports":[{"mode":"ingress","target":3306,"protocol":"tcp"}]},"nginx":{"build":{"context":".","dockerfile":"lagoon/nginx.dockerfile","args":{"CLI_IMAGE":""}},"container_name":"-nginx","depends_on":{"cli":{"condition":"service_started"}},"environment":{"LAGOON_LOCALDEV_URL":"","LAGOON_PROJECT":"","LAGOON_ROUTE":"https://","SSMTP_MAILHUB":"host.docker.internal:1025"},"labels":{"lagoon.persistent":"/app/public/sites/default/files/","lagoon.type":"nginx-php-persistent"},"networks":{"default":null,"stonehenge-network":null},"volumes":[{"type":"bind","source":".","target":"/app","bind":{"create_host_path":true}}]},"php":{"build":{"context":".","dockerfile":"lagoon/php.dockerfile","args":{"CLI_IMAGE":""}},"container_name":"-php","depends_on":{"cli":{"condition":"service_started"}},"environment":{"LAGOON_PROJECT":"","LAGOON_ROUTE":"https://","SSMTP_MAILHUB":"host.docker.internal:1025"},"labels":{"lagoon.name":"nginx","lagoon.persistent":"/app/public/sites/default/files/","lagoon.type":"nginx-php-persistent"},"networks":{"default":null},"volumes":[{"type":"bind","source":".","target":"/app","bind":{"create_host_path":true}}]},"pma":{"container_name":"-pma","environment":{"PMA_HOST":"mariadb","PMA_PASSWORD":"drupal","PMA_USER":"drupal","UPLOAD_LIMIT":"1G"},"image":"phpmyadmin/phpmyadmin","labels":{"lagoon.type":"none"},"networks":{"default":null,"stonehenge-network":null}}},"networks":{"default":{"name":"test9_default","ipam":{},"external":false},"stonehenge-network":{"name":"stonehenge-network","ipam":{},"external":true}},"volumes":{"es_data":{"name":"test9_es_data","external":false},"ssh":{"name":"stonehenge-ssh","external":true}}}`,
},
{
name: "test10 check an valid docker-compose with missing env_files (same as test9 but not ignoring the errors)",
args: args{
file: "../../test-resources/docker-compose/test10/docker-compose.yml",
},
wantErr: true,
wantErrMsg: "no such file or directory",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
l, err := UnmarshaDockerComposeYAML(tt.args.file, tt.args.ignoreErrors, map[string]string{})
l, err := UnmarshaDockerComposeYAML(tt.args.file, tt.args.ignoreNonStringKeyErrors, tt.args.ignoreMissingEnvFiles, map[string]string{})
if err != nil && !tt.wantErr {
t.Errorf("UnmarshaDockerComposeYAML() error = %v, wantErr %v", err, tt.wantErr)
}
if tt.wantErr {
if err.Error() != tt.wantErrMsg {
if !strings.Contains(err.Error(), tt.wantErrMsg) {
t.Errorf("UnmarshaDockerComposeYAML() error = %v, wantErrMsg %v", err.Error(), tt.wantErrMsg)
}
} else {
Expand Down
111 changes: 111 additions & 0 deletions test-resources/docker-compose/test10/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
version: '2.3'

x-lagoon-project:
# Lagoon project name (leave `&lagoon-project` when you edit this)
&lagoon-project "${COMPOSE_PROJECT_NAME}"

x-environment:
&default-environment
LAGOON_PROJECT: *lagoon-project
# Route that should be used locally
LAGOON_ROUTE: "https://${DRUPAL_HOSTNAME}"
SSMTP_MAILHUB: "host.docker.internal:1025"

services:

cli: # cli container, will be used for executing composer and any local commands (drush, drupal, etc.)
container_name: "${COMPOSE_PROJECT_NAME}-cli"
build:
context: .
dockerfile: lagoon/cli.dockerfile
image: *lagoon-project # this image will be reused as `CLI_IMAGE` in subsequent Docker builds
labels:
# Lagoon Labels
lagoon.type: cli-persistent
lagoon.persistent.name: nginx # mount the persistent storage of nginx into this container
lagoon.persistent: "/app/public/sites/default/files/" # location where the persistent storage should be mounted
volumes:
- .:/app:delegated
- ssh:/tmp/amazeeio_ssh-agent
environment:
<< : *default-environment # loads the defined environment variables from the top
SIMPLETEST_BASE_URL: "http://nginx:8080"
SIMPLETEST_DB: "mysql://drupal:drupal@mariadb:3306/drupal"
DRUSH_OPTIONS_URI: "https://${DRUPAL_HOSTNAME}"
env_file:
- .env.local

nginx:
container_name: "${COMPOSE_PROJECT_NAME}-nginx"
build:
context: .
dockerfile: lagoon/nginx.dockerfile
args:
CLI_IMAGE: *lagoon-project # Inject the name of the cli image
labels:
lagoon.type: nginx-php-persistent
lagoon.persistent: "/app/public/sites/default/files/" # define where the persistent file storage should be mounted too
volumes:
- .:/app:delegated
depends_on:
- cli # basically just tells docker-compose to build the cli first
environment:
<< : *default-environment # loads the defined environment variables from the top
LAGOON_LOCALDEV_URL: "${DRUPAL_HOSTNAME}" # generate another route for nginx, by default we go to varnish
networks:
- stonehenge-network
- default

php:
container_name: "${COMPOSE_PROJECT_NAME}-php"
build:
context: .
dockerfile: lagoon/php.dockerfile
args:
CLI_IMAGE: *lagoon-project
labels:
lagoon.type: nginx-php-persistent
lagoon.name: nginx # we want this service be part of the nginx pod in Lagoon
lagoon.persistent: /app/public/sites/default/files/ # define where the persistent storage should be mounted too
volumes:
- .:/app:delegated
depends_on:
- cli # basically just tells docker-compose to build the cli first
environment:
<< : *default-environment # loads the defined environment variables from the top
env_file:
- .env.local

mariadb:
container_name: "${COMPOSE_PROJECT_NAME}-db"
image: uselagoon/mariadb-drupal:latest
labels:
lagoon.type: mariadb
ports:
- "3306" # exposes the port 3306 with a random local port, find it with `docker-compose port mariadb 3306`
environment:
<< : *default-environment

pma:
image: phpmyadmin/phpmyadmin
container_name: "${COMPOSE_PROJECT_NAME}-pma"
environment:
PMA_HOST: mariadb
PMA_USER: drupal
PMA_PASSWORD: drupal
UPLOAD_LIMIT: 1G
labels:
lagoon.type: none
networks:
- default
- stonehenge-network

networks:
stonehenge-network:
external: true

volumes:
es_data:
ssh:
name: stonehenge-ssh
external: true
Loading

0 comments on commit 2f27606

Please sign in to comment.