feat: variable replacement support for lagoon.base.image label (#378)

* Adds ability to specify tags in refreshimages

* removes println

* refactor: just use one tag with variable replacement

* test: add test with lagoon and docker-compose files

---------

Co-authored-by: Blaize Kaye <[email protected]>
Co-authored-by: shreddedbacon <[email protected]>

cmd/identify_imagebuild_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/andreyvit/diff"
 	"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
 	"github.com/uselagoon/build-deploy-tool/internal/generator"
 	"github.com/uselagoon/build-deploy-tool/internal/helpers"
@@ -752,6 +753,61 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "test12 Force Pull Base Images with variable replacement",
+			args: testdata.GetSeedData(
+				testdata.TestData{
+					Namespace:       "example-project-main",
+					ProjectName:     "example-project",
+					EnvironmentName: "main",
+					Branch:          "main",
+					LagoonYAML:      "internal/testdata/basic/lagoon.forcebaseimagepull-2.yml",
+					ProjectVariables: []lagoon.EnvironmentVariable{
+						{
+							Name:  "BASE_IMAGE_TAG",
+							Value: "my-tag",
+							Scope: "build",
+						},
+						{
+							Name:  "BASE_IMAGE_REPO",
+							Value: "my-repo",
+							Scope: "build",
+						},
+					},
+				}, true),
+			want: imageBuild{
+				BuildKit: helpers.BoolPtr(true),
+				BuildArguments: map[string]string{
+					"BASE_IMAGE_TAG":               "my-tag",
+					"BASE_IMAGE_REPO":              "my-repo",
+					"LAGOON_BUILD_NAME":            "lagoon-build-abcdefg",
+					"LAGOON_PROJECT":               "example-project",
+					"LAGOON_ENVIRONMENT":           "main",
+					"LAGOON_ENVIRONMENT_TYPE":      "production",
+					"LAGOON_BUILD_TYPE":            "branch",
+					"LAGOON_GIT_SOURCE_REPOSITORY": "ssh://[email protected]/lagoon-demo.git",
+					"LAGOON_KUBERNETES":            "remote-cluster1",
+					"LAGOON_GIT_SHA":               "abcdefg123456",
+					"LAGOON_GIT_BRANCH":            "main",
+					"NODE_IMAGE":                   "example-project-main-node",
+					"LAGOON_SSH_PRIVATE_KEY":       "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
+				},
+				ForcePullImages: []string{
+					"registry.com/my-repo/imagename:my-tag",
+				},
+				Images: []imageBuilds{
+					{
+						Name: "node",
+						ImageBuild: generator.ImageBuild{
+							BuildImage:     "harbor.example/example-project/main/node:latest",
+							Context:        "internal/testdata/basic/docker",
+							DockerFile:     "basic.dockerfile",
+							TemporaryImage: "example-project-main-node",
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -788,10 +844,10 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
 				t.Errorf("%v", err)
 			}
 
-			oJ, _ := json.Marshal(out)
-			wJ, _ := json.Marshal(tt.want)
+			oJ, _ := json.MarshalIndent(out, "", "  ")
+			wJ, _ := json.MarshalIndent(tt.want, "", "  ")
 			if string(oJ) != string(wJ) {
-				t.Errorf("returned output %v doesn't match want %v", string(oJ), string(wJ))
+				t.Errorf("ImageBuildConfigurationIdentification() = \n%v", diff.LineDiff(string(oJ), string(wJ)))
 			}
 			t.Cleanup(func() {
 				helpers.UnsetEnvVars(tt.vars)

internal/generator/helpers_generator.go

@@ -8,6 +8,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/distribution/reference"
+
 	"github.com/spf13/cobra"
 	"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
 	"github.com/uselagoon/build-deploy-tool/internal/lagoon"
@@ -271,3 +273,44 @@ func getDBaasEnvironment(
 	}
 	return exists, nil
 }
+
+var exp = regexp.MustCompile(`(\\*)\$\{(.+?)(?:(\:\-)(.*?))?\}`)
+
+func determineRefreshImage(serviceName, imageName string, envVars []lagoon.EnvironmentVariable) (string, []error) {
+	errs := []error{}
+	parsed := exp.ReplaceAllStringFunc(string(imageName), func(match string) string {
+		tagvalue := ""
+		re := regexp.MustCompile(`\${?(\w+)?(?::-(\w+))?}?`)
+		matches := re.FindStringSubmatch(match)
+		if len(matches) > 0 {
+			tv := ""
+			envVarKey := matches[1]
+			defaultVal := matches[2] //This could be empty
+			for _, v := range envVars {
+				if v.Name == envVarKey {
+					tv = v.Value
+				}
+			}
+			if tv == "" {
+				if defaultVal != "" {
+					tagvalue = defaultVal
+				} else {
+					errs = append(errs, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - no matching variable or fallback found to replace requested variable %s", serviceName, imageName, envVarKey))
+				}
+			} else {
+				tagvalue = tv
+			}
+		}
+		return tagvalue
+	})
+	if parsed == imageName {
+		if !reference.ReferenceRegexp.MatchString(parsed) {
+			if strings.Contains(parsed, "$") {
+				errs = append(errs, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - variables are defined incorrectly, must contain curly brackets (example: '${VARIABLE}')", serviceName, imageName))
+			} else {
+				errs = append(errs, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - please ensure it conforms to the structure `[REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]`", serviceName, imageName))
+			}
+		}
+	}
+	return parsed, errs
+}

internal/generator/helpers_generator_test.go

@@ -81,3 +81,89 @@ func Test_checkDuplicateCronjobs(t *testing.T) {
 		})
 	}
 }
+
+func Test_determineRefreshImage(t *testing.T) {
+	type args struct {
+		serviceName string
+		imageName   string
+		envVars     []lagoon.EnvironmentVariable
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "Identity function",
+			args: args{
+				serviceName: "testservice",
+				imageName:   "image/name:latest",
+				envVars:     nil,
+			},
+			want:    "image/name:latest",
+			wantErr: false,
+		},
+		{
+			name: "Fails with no matching variable in envvars",
+			args: args{
+				serviceName: "testservice",
+				imageName:   "image/name:${NOENVVAR}",
+				envVars:     nil,
+			},
+			want:    "",
+			wantErr: true,
+		},
+		{
+			name: "Fails with variable missing curly brackets",
+			args: args{
+				serviceName: "testservice",
+				imageName:   "image/name:$NOENVVAR",
+				envVars:     nil,
+			},
+			want:    "",
+			wantErr: true,
+		},
+		{
+			name: "Tag with simple arg - fallback to default",
+			args: args{
+				serviceName: "testservice",
+				imageName:   "image/name:${ENVVAR:-sometag}",
+				envVars:     nil,
+			},
+			want:    "image/name:sometag",
+			wantErr: false,
+		},
+		{
+			name: "Tag with env var that works",
+			args: args{
+				serviceName: "testservice",
+				imageName:   "image/name:${ENVVAR:-sometag}",
+				envVars: []lagoon.EnvironmentVariable{
+					{
+						Name:  "ENVVAR",
+						Value: "injectedTag",
+					},
+				},
+			},
+			want:    "image/name:injectedTag",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, errs := determineRefreshImage(tt.args.serviceName, tt.args.imageName, tt.args.envVars)
+			if len(errs) > 0 && !tt.wantErr {
+				for idx, err := range errs {
+					t.Errorf("determineRefreshImage() error = %v, wantErr %v", err, tt.wantErr)
+					if idx+1 == len(errs) {
+						return
+					}
+				}
+			}
+			if got != tt.want && !tt.wantErr {
+				t.Errorf("determineRefreshImage() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/generator/services.go

@@ -7,8 +7,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/distribution/reference"
-
 	composetypes "github.com/compose-spec/compose-go/types"
 	"github.com/uselagoon/build-deploy-tool/internal/helpers"
 	"github.com/uselagoon/build-deploy-tool/internal/lagoon"
@@ -286,13 +284,22 @@ func composeToServiceValues(
 			}
 		}
 
+		// if any `lagoon.base.image` labels are set, we note them for docker pulling
+		// this allows us to refresh the docker-host's cache in cases where an image
+		// may have an update without a change in tag (i.e. "latest" tagged images)
 		baseimage := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.base.image")
 		if baseimage != "" {
-			// First, let's ensure that the structure of the base image is valid
-			if !reference.ReferenceRegexp.MatchString(baseimage) {
-				return nil, fmt.Errorf("the 'lagoon.base.image' label defined on service %s in the docker-compose file is invalid ('%s') - please ensure it conforms to the structure `[REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]`", composeService, baseimage)
+			baseImageWithTag, errs := determineRefreshImage(composeService, baseimage, buildValues.EnvironmentVariables)
+			if len(errs) > 0 {
+				for idx, err := range errs {
+					if idx+1 == len(errs) {
+						return nil, err
+					} else {
+						fmt.Println(err)
+					}
+				}
 			}
-			buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseimage)
+			buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseImageWithTag)
 		}
 
 		// if there are overrides defined in the lagoon API `LAGOON_SERVICE_TYPES`

internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml

@@ -0,0 +1,23 @@
+version: '2'
+services:
+  node:
+    networks:
+      - amazeeio-network
+      - default
+    build:
+      context: internal/testdata/basic/docker
+      dockerfile: basic.dockerfile
+    labels:
+      lagoon.type: basic
+      lagoon.service.usecomposeports: true
+      lagoon.base.image: registry.com/${BASE_IMAGE_REPO:-namespace}/imagename:${BASE_IMAGE_TAG:-latest}
+    volumes:
+      - .:/app:delegated
+    ports:
+      - '1234'
+      - '8191'
+      - '9001/udp'
+
+networks:
+  amazeeio-network:
+    external: true

internal/testdata/basic/lagoon.forcebaseimagepull-2.yml

@@ -0,0 +1,10 @@
+docker-compose-yaml: internal/testdata/basic/docker-compose.forcebaseimagepull-2.yml
+
+environment_variables:
+  git_sha: "true"
+
+environments:
+  main:
+    routes:
+      - node:
+          - example.com