Allow for overriding default dependency source

dependency_cache.go

@@ -18,6 +18,7 @@ package libpak
 
 import (
 	"crypto/sha256"
+	"crypto/tls"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -164,15 +165,17 @@ type RequestModifierFunc func(request *http.Request) (*http.Request, error)
 func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...RequestModifierFunc) (*os.File, error) {
 
 	var (
-		actual   BuildpackDependency
-		artifact string
-		file     string
-		uri      = dependency.URI
-		urlP     *url.URL
+		actual    BuildpackDependency
+		artifact  string
+		file      string
+		isBinding bool
+		uri       = dependency.URI
+		urlP      *url.URL
 	)
 
 	for d, u := range d.Mappings {
 		if d == dependency.SHA256 {
+			isBinding = true
 			uri = u
 			break
 		}
@@ -184,6 +187,10 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque
 		return nil, fmt.Errorf("unable to parse URI. see DEBUG log level")
 	}
 
+	if !isBinding {
+		d.setDependencyMirror(urlP)
+	}
+
 	if dependency.SHA256 == "" {
 		d.Logger.Headerf("%s Dependency has no SHA256. Skipping cache.",
 			color.New(color.FgYellow, color.Bold).Sprint("Warning:"))
@@ -287,6 +294,28 @@ func (d DependencyCache) downloadFile(source string, destination string, mods ..
 }
 
 func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...RequestModifierFunc) error {
+	var httpClient *http.Client
+	if (strings.EqualFold(url.Hostname(), "localhost")) || (strings.EqualFold(url.Hostname(), "127.0.0.1")) {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	} else {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				Dial: (&net.Dialer{
+					Timeout:   d.HttpClientTimeouts.DialerTimeout,
+					KeepAlive: d.HttpClientTimeouts.DialerKeepAlive,
+				}).Dial,
+				TLSHandshakeTimeout:   d.HttpClientTimeouts.TLSHandshakeTimeout,
+				ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout,
+				ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout,
+				Proxy:                 http.ProxyFromEnvironment,
+			},
+		}
+	}
+
 	req, err := http.NewRequest("GET", url.String(), nil)
 	if err != nil {
 		return fmt.Errorf("unable to create new GET request for %s\n%w", url.Redacted(), err)
@@ -303,19 +332,7 @@ func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...
 		}
 	}
 
-	client := http.Client{
-		Transport: &http.Transport{
-			Dial: (&net.Dialer{
-				Timeout:   d.HttpClientTimeouts.DialerTimeout,
-				KeepAlive: d.HttpClientTimeouts.DialerKeepAlive,
-			}).Dial,
-			TLSHandshakeTimeout:   d.HttpClientTimeouts.TLSHandshakeTimeout,
-			ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout,
-			ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout,
-			Proxy:                 http.ProxyFromEnvironment,
-		},
-	}
-	resp, err := client.Do(req)
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return fmt.Errorf("unable to request %s\n%w", url.Redacted(), err)
 	}
@@ -363,3 +380,28 @@ func (DependencyCache) verify(path string, expected string) error {
 
 	return nil
 }
+
+func (d DependencyCache) setDependencyMirror(urlD *url.URL) {
+	dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "")
+	dependencyMirrorPreserveHost := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "false")
+	if dependencyMirror != "" {
+		var originalHost string
+
+		d.Logger.Infof("variable BP_DEPENDENCY_MIRROR found. overriding download uri.")
+		urlOverride, err := url.ParseRequestURI(dependencyMirror)
+
+		if strings.EqualFold(dependencyMirrorPreserveHost, "true") && urlD.Hostname() != "" {
+			originalHost = "/" + urlD.Hostname()
+		}
+
+		if strings.EqualFold(urlOverride.Scheme, "https") || strings.EqualFold(urlOverride.Scheme, "file") {
+			urlD.Scheme = urlOverride.Scheme
+			urlD.User = urlOverride.User
+			urlD.Host = urlOverride.Host
+			urlD.Path = urlOverride.Path + originalHost + urlD.Path
+		} else {
+			d.Logger.Debugf("environment variable BP_DEPENDENCY_MIRROR has invalid value: %s\n%w", dependencyMirror, err)
+			d.Logger.Infof("ignoring invalid variable BP_DEPENDENCY_MIRROR. have you used one of the supported schemes 'https://' or 'file://'? continuing without override.")
+		}
+	}
+}

dependency_cache_test.go

@@ -298,6 +298,94 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
 			})
 		})
 
+		context("dependency mirror is used https", func() {
+			var mirrorServer *ghttp.Server
+
+			it.Before(func() {
+				mirrorServer = ghttp.NewTLSServer()
+			})
+
+			it.After(func() {
+				mirrorServer.Close()
+			})
+
+			it("downloads from https mirror", func() {
+				url, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				t.Setenv("BP_DEPENDENCY_MIRROR", url.Scheme+"://"+"username:password@"+url.Host+"/foo/bar")
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyBasicAuth("username", "password"),
+					ghttp.VerifyRequest(http.MethodGet, "/foo/bar/test-path", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("downloads from https mirror preserving hostname", func() {
+				url, err := url.Parse(mirrorServer.URL())
+				Expect(err).NotTo(HaveOccurred())
+				t.Setenv("BP_DEPENDENCY_MIRROR", url.Scheme+"://"+url.Host)
+				t.Setenv("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "true")
+				mirrorServer.AppendHandlers(ghttp.CombineHandlers(
+					ghttp.VerifyRequest(http.MethodGet, "/"+url.Hostname()+"/test-path", ""),
+					ghttp.RespondWith(http.StatusOK, "test-fixture"),
+				))
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+		})
+
+		context("dependency mirror is used file", func() {
+			var (
+				mirrorPath              string
+				mirrorPathPreservedHost string
+			)
+
+			it.Before(func() {
+				var err error
+				mirrorPath, err = os.MkdirTemp("", "mirror-path")
+				Expect(err).NotTo(HaveOccurred())
+				mirrorPathPreservedHost = mirrorPath + "/127.0.0.1"
+				Expect(os.Mkdir(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred())
+			})
+
+			it.After(func() {
+				Expect(os.RemoveAll(mirrorPath)).To(Succeed())
+			})
+
+			it("downloads from file mirror", func() {
+				mirrorFile := filepath.Join(mirrorPath, "test-path")
+				Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())
+
+				t.Setenv("BP_DEPENDENCY_MIRROR", "file://"+mirrorPath)
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+
+			it("downloads from file mirror preserving hostname", func() {
+				mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path")
+				Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())
+
+				t.Setenv("BP_DEPENDENCY_MIRROR", "file://"+mirrorPath)
+				t.Setenv("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "true")
+
+				a, err := dependencyCache.Artifact(dependency)
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
+			})
+		})
+
 		it("fails with invalid SHA256", func() {
 			server.AppendHandlers(ghttp.RespondWith(http.StatusOK, "invalid-fixture"))