chore: support newer compose-go library with lagoon workarounds

go.mod

@@ -6,11 +6,12 @@ require (
 	dario.cat/mergo v1.0.0
 	github.com/PaesslerAG/gval v1.2.2
 	github.com/amazeeio/dbaas-operator v0.3.0
-	github.com/compose-spec/compose-go v1.2.7
+	github.com/compose-spec/compose-go/v2 v2.0.0-rc.8
 	github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f
 	github.com/google/go-cmp v0.5.9
 	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/k8up-io/k8up/v2 v2.7.2
+	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/uselagoon/machinery v0.0.12
 	github.com/vshn/k8up v1.99.99
@@ -24,7 +25,7 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/distribution/distribution/v3 v3.0.0-20210316161203-a01c71e2477e // indirect
+	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
@@ -45,20 +46,21 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
@@ -79,6 +81,6 @@ require (
 
 replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16
 
-replace github.com/compose-spec/compose-go v1.2.7 => github.com/shreddedbacon/compose-go v0.0.0-20220616064547-4e908a2865c1
+replace github.com/compose-spec/compose-go/v2 v2.0.0-rc.8 => github.com/shreddedbacon/compose-go/v2 v2.0.0-20240312080248-f885cd14d500
 
 // replace github.com/compose-spec/compose-go v1.2.7 => ../../compose-spec/compose-go

internal/generator/services.go

@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"strings"
 
-	composetypes "github.com/compose-spec/compose-go/types"
+	composetypes "github.com/compose-spec/compose-go/v2/types"
 	"github.com/uselagoon/build-deploy-tool/internal/helpers"
 	"github.com/uselagoon/build-deploy-tool/internal/lagoon"
 )

internal/generator/services_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	composetypes "github.com/compose-spec/compose-go/types"
+	composetypes "github.com/compose-spec/compose-go/v2/types"
 	"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
 	"github.com/uselagoon/build-deploy-tool/internal/helpers"
 	"github.com/uselagoon/build-deploy-tool/internal/lagoon"

internal/lagoon/compose.go

@@ -1,13 +1,16 @@
 package lagoon
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 
-	"github.com/compose-spec/compose-go/cli"
-	"github.com/compose-spec/compose-go/loader"
-	composetypes "github.com/compose-spec/compose-go/types"
+	"github.com/compose-spec/compose-go/v2/cli"
+	"github.com/compose-spec/compose-go/v2/loader"
+	composetypes "github.com/compose-spec/compose-go/v2/types"
+	"github.com/sirupsen/logrus"
 	goyaml "gopkg.in/yaml.v2"
 	goyamlv3 "gopkg.in/yaml.v3"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
@@ -20,6 +23,8 @@ type OriginalServiceOrder struct {
 
 // UnmarshaDockerComposeYAML unmarshal the lagoon.yml file into a YAML and map for consumption.
 func UnmarshaDockerComposeYAML(file string, ignoreErrors, ignoreMissingEnvFiles bool, envvars map[string]string) (*composetypes.Project, []OriginalServiceOrder, error) {
+	// disable logging output for now, maybe capture this in a buffer for later analysis
+	logrus.SetOutput(io.Discard)
 	options, err := cli.NewProjectOptions([]string{file},
 		cli.WithResolvedPaths(false),
 		cli.WithLoadOptions(
@@ -31,8 +36,11 @@ func UnmarshaDockerComposeYAML(file string, ignoreErrors, ignoreMissingEnvFiles
 			},
 		),
 	)
+	if err != nil {
+		return nil, nil, err
+	}
 	options.Environment = envvars
-	l, err := cli.ProjectFromOptions(options)
+	l, err := cli.ProjectFromOptions(context.Background(), options)
 	if err != nil {
 		return nil, nil, err
 	}

internal/testdata/complex/docker-compose.multi-db.yml

@@ -55,8 +55,7 @@ services:
       lagoon.type: nginx-php-persistent
       lagoon.persistent: /app/web/sites/default/files/ # define where the persistent storage should be mounted too
       lando.type: nginx-drupal
-    << : *default-volumes # loads the defined volumes from the top
-    << : *default-user # uses the defined user from top
+    << : [*default-volumes, *default-user]
     depends_on:
       - cli # basically just tells docker-compose to build the cli first
     environment:
@@ -77,8 +76,7 @@ services:
       lagoon.name: nginx # we want this service be part of the nginx pod in Lagoon
       lagoon.persistent: /app/web/sites/default/files/ # define where the persistent storage should be mounted too
       lando.type: php-fpm
-    << : *default-volumes # loads the defined volumes from the top
-    << : *default-user # uses the defined user from top
+    << : [*default-volumes, *default-user]
     depends_on:
       - cli # basically just tells docker-compose to build the cli first
     environment:

legacy/build-deploy-docker-compose.sh

@@ -248,6 +248,8 @@ You can run docker compose config locally to check that your docker-compose file
 "
   if [[ "${dccOutput}" =~ "no such file or directory" ]]; then
     echo "> an env_file is defined in your docker-compose file, but no matching file found."
+    echo ">> See https://docs.docker.com/compose/compose-file/05-services/#env_file if and the new env_file attributes for 'required' env_files."
+    echo ">> Consider setting these files to 'required: false' when using Lagoon to prevent this warning from appearing again"
   fi
   if [[ "${dccOutput}" =~ "Non-string key" ]]; then
     echo "> an invalid string key was detected in your docker-compose file."

test-resources/docker-compose/test1/docker-compose.yml

@@ -55,8 +55,7 @@ services:
       lagoon.type: nginx-php-persistent
       lagoon.persistent: /app/web/sites/default/files/ # define where the persistent storage should be mounted too
       lando.type: nginx-drupal
-    << : *default-volumes # loads the defined volumes from the top
-    << : *default-user # uses the defined user from top
+    << : [*default-volumes, *default-user]
     depends_on:
       - cli # basically just tells docker-compose to build the cli first
     environment:
@@ -77,8 +76,7 @@ services:
       lagoon.name: nginx # we want this service be part of the nginx pod in Lagoon
       lagoon.persistent: /app/web/sites/default/files/ # define where the persistent storage should be mounted too
       lando.type: php-fpm
-    << : *default-volumes # loads the defined volumes from the top
-    << : *default-user # uses the defined user from top
+    << : [*default-volumes, *default-user]
     depends_on:
       - cli # basically just tells docker-compose to build the cli first
     environment:

test-resources/docker-compose/test12/docker-compose.yml

@@ -0,0 +1,113 @@
+version: '2.3'
+
+x-lagoon-project:
+  # Lagoon project name (leave `&lagoon-project` when you edit this)
+  &lagoon-project "${COMPOSE_PROJECT_NAME}"
+
+x-environment:
+  &default-environment
+  LAGOON_PROJECT: *lagoon-project
+  # Route that should be used locally
+  LAGOON_ROUTE: "https://${DRUPAL_HOSTNAME}"
+  SSMTP_MAILHUB: "host.docker.internal:1025"
+
+services:
+
+  cli: # cli container, will be used for executing composer and any local commands (drush, drupal, etc.)
+    container_name: "${COMPOSE_PROJECT_NAME}-cli"
+    build:
+      context: .
+      dockerfile: lagoon/cli.dockerfile
+    image: *lagoon-project # this image will be reused as `CLI_IMAGE` in subsequent Docker builds
+    labels:
+      # Lagoon Labels
+      lagoon.type: cli-persistent
+      lagoon.persistent.name: nginx # mount the persistent storage of nginx into this container
+      lagoon.persistent: "/app/public/sites/default/files/" # location where the persistent storage should be mounted
+    volumes:
+      - .:/app:delegated
+      - ssh:/tmp/amazeeio_ssh-agent
+    environment:
+      << : *default-environment # loads the defined environment variables from the top
+      SIMPLETEST_BASE_URL: "http://nginx:8080"
+      SIMPLETEST_DB: "mysql://drupal:drupal@mariadb:3306/drupal"
+      DRUSH_OPTIONS_URI: "https://${DRUPAL_HOSTNAME}"
+    env_file:
+      - path: .env.local
+        required: true
+
+  nginx:
+    container_name: "${COMPOSE_PROJECT_NAME}-nginx"
+    build:
+      context: .
+      dockerfile: lagoon/nginx.dockerfile
+      args:
+        CLI_IMAGE: *lagoon-project # Inject the name of the cli image
+    labels:
+      lagoon.type: nginx-php-persistent
+      lagoon.persistent: "/app/public/sites/default/files/" # define where the persistent file storage should be mounted too
+    volumes:
+      - .:/app:delegated
+    depends_on:
+      - cli # basically just tells docker-compose to build the cli first
+    environment:
+      << : *default-environment # loads the defined environment variables from the top
+      LAGOON_LOCALDEV_URL: "${DRUPAL_HOSTNAME}" # generate another route for nginx, by default we go to varnish
+    networks:
+      - stonehenge-network
+      - default
+
+  php:
+    container_name: "${COMPOSE_PROJECT_NAME}-php"
+    build:
+      context: .
+      dockerfile: lagoon/php.dockerfile
+      args:
+        CLI_IMAGE: *lagoon-project
+    labels:
+      lagoon.type: nginx-php-persistent
+      lagoon.name: nginx # we want this service be part of the nginx pod in Lagoon
+      lagoon.persistent: /app/public/sites/default/files/ # define where the persistent storage should be mounted too
+    volumes:
+      - .:/app:delegated
+    depends_on:
+      - cli # basically just tells docker-compose to build the cli first
+    environment:
+      << : *default-environment # loads the defined environment variables from the top
+    env_file:
+      - path: .env.local
+        required: false
+
+  mariadb:
+    container_name: "${COMPOSE_PROJECT_NAME}-db"
+    image: uselagoon/mariadb-drupal:latest
+    labels:
+      lagoon.type: mariadb
+    ports:
+      - "3306" # exposes the port 3306 with a random local port, find it with `docker-compose port mariadb 3306`
+    environment:
+      << : *default-environment
+
+  pma:
+    image: phpmyadmin/phpmyadmin
+    container_name: "${COMPOSE_PROJECT_NAME}-pma"
+    environment:
+      PMA_HOST: mariadb
+      PMA_USER: drupal
+      PMA_PASSWORD: drupal
+      UPLOAD_LIMIT: 1G
+    labels:
+      lagoon.type: none
+    networks:
+      - default
+      - stonehenge-network
+
+networks:
+  stonehenge-network:
+    external: true
+
+volumes:
+  es_data:
+  ssh:
+    name: stonehenge-ssh
+    external: true

test-resources/docker-compose/test13/docker-compose.yml

@@ -0,0 +1,113 @@
+version: '2.3'
+
+x-lagoon-project:
+  # Lagoon project name (leave `&lagoon-project` when you edit this)
+  &lagoon-project "${COMPOSE_PROJECT_NAME}"
+
+x-environment:
+  &default-environment
+  LAGOON_PROJECT: *lagoon-project
+  # Route that should be used locally
+  LAGOON_ROUTE: "https://${DRUPAL_HOSTNAME}"
+  SSMTP_MAILHUB: "host.docker.internal:1025"
+
+services:
+
+  cli: # cli container, will be used for executing composer and any local commands (drush, drupal, etc.)
+    container_name: "${COMPOSE_PROJECT_NAME}-cli"
+    build:
+      context: .
+      dockerfile: lagoon/cli.dockerfile
+    image: *lagoon-project # this image will be reused as `CLI_IMAGE` in subsequent Docker builds
+    labels:
+      # Lagoon Labels
+      lagoon.type: cli-persistent
+      lagoon.persistent.name: nginx # mount the persistent storage of nginx into this container
+      lagoon.persistent: "/app/public/sites/default/files/" # location where the persistent storage should be mounted
+    volumes:
+      - .:/app:delegated
+      - ssh:/tmp/amazeeio_ssh-agent
+    environment:
+      << : *default-environment # loads the defined environment variables from the top
+      SIMPLETEST_BASE_URL: "http://nginx:8080"
+      SIMPLETEST_DB: "mysql://drupal:drupal@mariadb:3306/drupal"
+      DRUSH_OPTIONS_URI: "https://${DRUPAL_HOSTNAME}"
+    env_file:
+      - path: .env.local
+        required: true
+
+  nginx:
+    container_name: "${COMPOSE_PROJECT_NAME}-nginx"
+    build:
+      context: .
+      dockerfile: lagoon/nginx.dockerfile
+      args:
+        CLI_IMAGE: *lagoon-project # Inject the name of the cli image
+    labels:
+      lagoon.type: nginx-php-persistent
+      lagoon.persistent: "/app/public/sites/default/files/" # define where the persistent file storage should be mounted too
+    volumes:
+      - .:/app:delegated
+    depends_on:
+      - cli # basically just tells docker-compose to build the cli first
+    environment:
+      << : *default-environment # loads the defined environment variables from the top
+      LAGOON_LOCALDEV_URL: "${DRUPAL_HOSTNAME}" # generate another route for nginx, by default we go to varnish
+    networks:
+      - stonehenge-network
+      - default
+
+  php:
+    container_name: "${COMPOSE_PROJECT_NAME}-php"
+    build:
+      context: .
+      dockerfile: lagoon/php.dockerfile
+      args:
+        CLI_IMAGE: *lagoon-project
+    labels:
+      lagoon.type: nginx-php-persistent
+      lagoon.name: nginx # we want this service be part of the nginx pod in Lagoon
+      lagoon.persistent: /app/public/sites/default/files/ # define where the persistent storage should be mounted too
+    volumes:
+      - .:/app:delegated
+    depends_on:
+      - cli # basically just tells docker-compose to build the cli first
+    environment:
+      << : *default-environment # loads the defined environment variables from the top
+    env_file:
+      - path: .env.local
+        required: false
+
+  mariadb:
+    container_name: "${COMPOSE_PROJECT_NAME}-db"
+    image: uselagoon/mariadb-drupal:latest
+    labels:
+      lagoon.type: mariadb
+    ports:
+      - "3306" # exposes the port 3306 with a random local port, find it with `docker-compose port mariadb 3306`
+    environment:
+      << : *default-environment
+
+  pma:
+    image: phpmyadmin/phpmyadmin
+    container_name: "${COMPOSE_PROJECT_NAME}-pma"
+    environment:
+      PMA_HOST: mariadb
+      PMA_USER: drupal
+      PMA_PASSWORD: drupal
+      UPLOAD_LIMIT: 1G
+    labels:
+      lagoon.type: none
+    networks:
+      - default
+      - stonehenge-network
+
+networks:
+  stonehenge-network:
+    external: true
+
+volumes:
+  es_data:
+  ssh:
+    name: stonehenge-ssh
+    external: true