From 81b9df86ffa48a190e991d6d85be552d40e07d62 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Fri, 28 Jul 2023 15:26:07 +0530 Subject: [PATCH 1/4] adds logic to remove http query params from destination file name Signed-off-by: Jayashree O --- dependency_cache.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dependency_cache.go b/dependency_cache.go index 7453ad5..f93a89e 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -182,7 +182,19 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque color.New(color.FgYellow, color.Bold).Sprint("Warning:")) d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) - artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) + + // ensure query parameters are not included in the downloaded file name if the uri is http type + downloadUri, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("unable to parse the download uri %s\n%w", uri, err) + } + + if(downloadUri.Scheme == "http" || downloadUri.Scheme == "https") { + artifact = filepath.Join(d.DownloadPath, filepath.Base(downloadUri.Path)) + } else { + artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) + } + if err := d.download(uri, artifact, mods...); err != nil { return nil, fmt.Errorf("unable to download %s\n%w", uri, err) } From ff207dfb3df818c582637edc594eec5b59ed1fd2 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Fri, 4 Aug 2023 16:35:00 +0530 Subject: [PATCH 2/4] sets downloaded http artifact names to uri's sha256 and adds a corresponding test Signed-off-by: Amit Singh --- dependency_cache.go | 5 ++++- dependency_cache_test.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dependency_cache.go b/dependency_cache.go index f93a89e..aecb588 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -190,7 +190,10 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque } if(downloadUri.Scheme == "http" || downloadUri.Scheme == "https") { - artifact = filepath.Join(d.DownloadPath, filepath.Base(downloadUri.Path)) + hasher := sha256.New() + hasher.Write([]byte(uri)) + + artifact = filepath.Join(d.DownloadPath, hex.EncodeToString(hasher.Sum(nil))) } else { artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) } diff --git a/dependency_cache_test.go b/dependency_cache_test.go index e9e99c9..e101579 100644 --- a/dependency_cache_test.go +++ b/dependency_cache_test.go @@ -17,6 +17,8 @@ package libpak_test import ( + "crypto/sha256" + "encoding/hex" "fmt" "io" "net/http" @@ -317,6 +319,21 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { Expect(io.ReadAll(a)).To(Equal([]byte("alternate-fixture"))) }) + it("sets downloaded file name to uri's sha256 with empty SHA256 and query parameters in the uri", func() { + dependency.SHA256 = "" + dependency.URI = fmt.Sprintf("%s/test-path?param1=value1¶m2=value2", server.URL()) + server.AppendHandlers(ghttp.RespondWith(http.StatusOK, "alternate-fixture")) + + hasher := sha256.New() + hasher.Write([]byte(dependency.URI)) + + a, err := dependencyCache.Artifact(dependency) + Expect(err).NotTo(HaveOccurred()) + + Expect(io.ReadAll(a)).To(Equal([]byte("alternate-fixture"))) + Expect(filepath.Base(a.Name())).To(Equal(hex.EncodeToString(hasher.Sum(nil)))) + }) + it("sets User-Agent", func() { server.AppendHandlers(ghttp.CombineHandlers( ghttp.VerifyHeaderKV("User-Agent", "test-user-agent"), From 59a60989f09af60c831a9e755c10481a2ab17b6e Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Thu, 7 Sep 2023 18:48:36 +0530 Subject: [PATCH 3/4] sets uri sha as the downloaded artifact filename and removes checks for http(s) uri Signed-off-by: Amit Singh --- dependency_cache.go | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/dependency_cache.go b/dependency_cache.go index aecb588..03f94fb 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -167,7 +167,9 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque actual BuildpackDependency artifact string file string + hasher = sha256.New() uri = dependency.URI + uriSha string ) for d, u := range d.Mappings { @@ -177,26 +179,17 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque } } + // downloaded artifact will have the uri sha as its filename + hasher.Write([]byte(uri)) + uriSha = hex.EncodeToString(hasher.Sum(nil)) + if dependency.SHA256 == "" { d.Logger.Headerf("%s Dependency has no SHA256. Skipping cache.", color.New(color.FgYellow, color.Bold).Sprint("Warning:")) d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) - // ensure query parameters are not included in the downloaded file name if the uri is http type - downloadUri, err := url.Parse(uri) - if err != nil { - return nil, fmt.Errorf("unable to parse the download uri %s\n%w", uri, err) - } - - if(downloadUri.Scheme == "http" || downloadUri.Scheme == "https") { - hasher := sha256.New() - hasher.Write([]byte(uri)) - - artifact = filepath.Join(d.DownloadPath, hex.EncodeToString(hasher.Sum(nil))) - } else { - artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) - } + artifact = filepath.Join(d.DownloadPath, uriSha) if err := d.download(uri, artifact, mods...); err != nil { return nil, fmt.Errorf("unable to download %s\n%w", uri, err) @@ -234,7 +227,7 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque } d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) - artifact = filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(uri)) + artifact = filepath.Join(d.DownloadPath, dependency.SHA256, uriSha) if err := d.download(uri, artifact, mods...); err != nil { return nil, fmt.Errorf("unable to download %s\n%w", uri, err) } From aaf573f26233f0013dabfbc191b56feb98f7a407 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Fri, 8 Sep 2023 10:48:27 +0530 Subject: [PATCH 4/4] adds test to verify a downloaded artifact has expected sha256 filename Signed-off-by: Amit Singh --- dependency_cache_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dependency_cache_test.go b/dependency_cache_test.go index e101579..c1c6ae7 100644 --- a/dependency_cache_test.go +++ b/dependency_cache_test.go @@ -20,6 +20,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "hash" "io" "net/http" "os" @@ -157,6 +158,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { downloadPath string dependency libpak.BuildpackDependency dependencyCache libpak.DependencyCache + hasher hash.Hash server *ghttp.Server ) @@ -166,6 +168,8 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { cachePath = t.TempDir() Expect(err).NotTo(HaveOccurred()) + hasher = sha256.New() + downloadPath = t.TempDir() Expect(err).NotTo(HaveOccurred()) @@ -319,12 +323,23 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { Expect(io.ReadAll(a)).To(Equal([]byte("alternate-fixture"))) }) + it("sets downloaded file name to uri's sha256", func() { + server.AppendHandlers(ghttp.RespondWith(http.StatusOK, "test-fixture")) + + hasher.Write([]byte(dependency.URI)) + + a, err := dependencyCache.Artifact(dependency) + Expect(err).NotTo(HaveOccurred()) + + Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture"))) + Expect(filepath.Base(a.Name())).To(Equal(hex.EncodeToString(hasher.Sum(nil)))) + }) + it("sets downloaded file name to uri's sha256 with empty SHA256 and query parameters in the uri", func() { dependency.SHA256 = "" dependency.URI = fmt.Sprintf("%s/test-path?param1=value1¶m2=value2", server.URL()) server.AppendHandlers(ghttp.RespondWith(http.StatusOK, "alternate-fixture")) - hasher := sha256.New() hasher.Write([]byte(dependency.URI)) a, err := dependencyCache.Artifact(dependency)