Allow for overriding default dependency source

dependency_cache.go

@@ -18,6 +18,7 @@ package libpak
 
 import (
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -187,7 +188,7 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque
 	}
 
 	if !isBinding {
-		d.overrideDependencySource(urlP)
+		d.setDependencyMirror(urlP)
 	}
 
 	if dependency.SHA256 == "" {
@@ -293,6 +294,28 @@ func (d DependencyCache) downloadFile(source string, destination string, mods ..
 }
 
 func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...RequestModifierFunc) error {
+	var httpClient *http.Client
+	if (strings.EqualFold(url.Hostname(), "localhost")) || (strings.EqualFold(url.Hostname(), "127.0.0.1")) {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	} else {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				Dial: (&net.Dialer{
+					Timeout:   d.HttpClientTimeouts.DialerTimeout,
+					KeepAlive: d.HttpClientTimeouts.DialerKeepAlive,
+				}).Dial,
+				TLSHandshakeTimeout:   d.HttpClientTimeouts.TLSHandshakeTimeout,
+				ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout,
+				ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout,
+				Proxy:                 http.ProxyFromEnvironment,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url.String(), nil)
 	if err != nil {
 		return fmt.Errorf("unable to create new GET request for %s\n%w", url.Redacted(), err)
@@ -309,19 +332,7 @@ func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...
 		}
 	}
 
-	client := http.Client{
-		Transport: &http.Transport{
-			Dial: (&net.Dialer{
-				Timeout:   d.HttpClientTimeouts.DialerTimeout,
-				KeepAlive: d.HttpClientTimeouts.DialerKeepAlive,
-			}).Dial,
-			TLSHandshakeTimeout:   d.HttpClientTimeouts.TLSHandshakeTimeout,
-			ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout,
-			ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout,
-			Proxy:                 http.ProxyFromEnvironment,
-		},
-	}
-	resp, err := client.Do(req)
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return fmt.Errorf("unable to request %s\n%w", url.Redacted(), err)
 	}
@@ -370,19 +381,27 @@ func (DependencyCache) verify(path string, expected string) error {
 	return nil
 }
 
-func (d DependencyCache) overrideDependencySource(urlD *url.URL) {
-	dependencySourceOverride := sherpa.GetEnvWithDefault("BP_DEPENDENCY_SOURCE_OVERRIDE", "")
-	if dependencySourceOverride != "" {
-		d.Logger.Infof("variable BP_DEPENDENCY_SOURCE_OVERRIDE found. overriding download uri.")
-		urlOverride, err := url.ParseRequestURI(dependencySourceOverride)
-		if err == nil {
+func (d DependencyCache) setDependencyMirror(urlD *url.URL) {
+	dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "")
+	dependencyMirrorPreserveHost := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "false")
+	if dependencyMirror != "" {
+		var originalHost string
+
+		d.Logger.Infof("variable BP_DEPENDENCY_MIRROR found. overriding download uri.")
+		urlOverride, err := url.ParseRequestURI(dependencyMirror)
+
+		if strings.EqualFold(dependencyMirrorPreserveHost, "true") && urlD.Hostname() != "" {
+			originalHost = "/" + urlD.Hostname()
+		}
+
+		if strings.EqualFold(urlOverride.Scheme, "https") || strings.EqualFold(urlOverride.Scheme, "file") {
 			urlD.Scheme = urlOverride.Scheme
 			urlD.User = urlOverride.User
 			urlD.Host = urlOverride.Host
-			urlD.Path = urlOverride.Path + urlD.Path
+			urlD.Path = urlOverride.Path + originalHost + urlD.Path
 		} else {
-			d.Logger.Debugf("environment variable BP_DEPENDENCY_SOURCE_OVERRIDE could not be parsed. value: %s\n%w", dependencySourceOverride, err)
-			d.Logger.Infof("ignoring invalid variable BP_DEPENDENCY_SOURCE_OVERRIDE. continuing without override...")
+			d.Logger.Debugf("environment variable BP_DEPENDENCY_MIRROR has invalid value: %s\n%w", dependencyMirror, err)
+			d.Logger.Infof("ignoring invalid variable BP_DEPENDENCY_MIRROR. have you used one of the supported schemes 'https://' or 'file://'? continuing without override.")
 		}
 	}
 }

dependency_cache_test.go

@@ -298,16 +298,22 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
 			})
 		})
 
-		context("source is overridden", func() {
-			serverOverride := ghttp.NewServer()
-			url, _ := url.Parse(serverOverride.URL())
+		context("dependency mirror is used https", func() {
+			var mirrorServer *ghttp.Server
 
 			it.Before(func() {
-				t.Setenv("BP_DEPENDENCY_SOURCE_OVERRIDE", url.Scheme+"://"+"username:password@"+url.Host+"/foo/bar")
+				mirrorServer = ghttp.NewTLSServer()
 			})
 
-			it("downloads from override source", func() {
-				serverOverride.AppendHandlers(ghttp.CombineHandlers(
+			it.After(func() {
+				mirrorServer.Close()
+			})
+
+			it("downloads from https mirror", func() {
+				url, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				t.Setenv("BP_DEPENDENCY_MIRROR", url.Scheme+"://"+"username:password@"+url.Host+"/foo/bar")
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
 					ghttp.VerifyBasicAuth("username", "password"),
 					ghttp.VerifyRequest(http.MethodGet, "/foo/bar/test-path", ""),
 					ghttp.RespondWith(http.StatusOK, "test-fixture"),
@@ -318,6 +324,66 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
 
 				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
 			})
+
+			it("downloads from https mirror preserving hostname", func() {
+				url, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				t.Setenv("BP_DEPENDENCY_MIRROR", url.Scheme+"://"+url.Host)
+				t.Setenv("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "true")
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyRequest(http.MethodGet, "/"+url.Hostname()+"/test-path", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+		})
+
+		context("dependency mirror is used file", func() {
+			var (
+				mirrorPath              string
+				mirrorPathPreservedHost string
+			)
+
+			it.Before(func() {
+				var err error
+				mirrorPath, err = os.MkdirTemp("", "mirror-path")
+				Expect(err).NotTo(HaveOccurred())
+				mirrorPathPreservedHost = mirrorPath + "/127.0.0.1"
+				Expect(os.Mkdir(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred())
+			})
+
+			it.After(func() {
+				Expect(os.RemoveAll(mirrorPath)).To(Succeed())
+			})
+
+			it("downloads from file mirror", func() {
+				mirrorFile := filepath.Join(mirrorPath, "test-path")
+				Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())
+
+				t.Setenv("BP_DEPENDENCY_MIRROR", "file://"+mirrorPath)
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("downloads from file mirror preserving hostname", func() {
+				mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path")
+				Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())
+
+				t.Setenv("BP_DEPENDENCY_MIRROR", "file://"+mirrorPath)
+				t.Setenv("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "true")
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
 		})
 
 		it("fails with invalid SHA256", func() {