From 0357c136ab1d4acb847bfdb8a1e27db320e80952 Mon Sep 17 00:00:00 2001
From: Volker Theile <vtheile@suse.com>
Date: Mon, 12 Feb 2024 13:13:20 +0100
Subject: [PATCH] BackingImage does not download URL correctly in some
 situation

Remove the `Referer` header that is automatically added by the http client. Without this header it is possible to download files that are redirected several times. This mimics the behaviour of `curl` and `wget` which do not submit the `Referer` header by default.

Signed-off-by: Volker Theile <vtheile@suse.com>
---
 pkg/sync/handler.go      | 26 ++++++++++++++++++++++++--
 pkg/sync/handler_test.go | 20 ++++++++++++++++++++
 2 files changed, 44 insertions(+), 2 deletions(-)
 create mode 100644 pkg/sync/handler_test.go

diff --git a/pkg/sync/handler.go b/pkg/sync/handler.go
index e558f3fd..d3256d07 100644
--- a/pkg/sync/handler.go
+++ b/pkg/sync/handler.go
@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,7 +40,7 @@ func (h *HTTPHandler) GetSizeFromURL(url string) (size int64, err error) {
 		return 0, err
 	}
 
-	client := http.Client{}
+	client := NewDownloadHttpClient()
 	resp, err := client.Do(rr)
 	if err != nil {
 		return 0, err
@@ -72,7 +73,7 @@ func (h *HTTPHandler) DownloadFromURL(ctx context.Context, url, filePath string,
 		return 0, err
 	}
 
-	client := http.Client{}
+	client := NewDownloadHttpClient()
 	resp, err := client.Do(rr)
 	if err != nil {
 		return 0, err
@@ -176,6 +177,27 @@ func IdleTimeoutCopy(ctx context.Context, cancel context.CancelFunc, src io.Read
 	return copied, err
 }
 
+func removeReferer(req *http.Request) {
+	for k, _ := range req.Header {
+		if strings.ToLower(k) == "referer" {
+			delete(req.Header, k)
+		}
+	}
+}
+
+func NewDownloadHttpClient() http.Client {
+	return http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// Remove the "Referer" header to enable downloads of files
+			// that are delivered via CDN and therefor may be redirected
+			// several times. This is the same behaviour of curl or wget
+			// in their default configuration.
+			removeReferer(req)
+			return nil
+		},
+	}
+}
+
 const (
 	MockFileSize = 4096
 )
diff --git a/pkg/sync/handler_test.go b/pkg/sync/handler_test.go
new file mode 100644
index 00000000..bc612058
--- /dev/null
+++ b/pkg/sync/handler_test.go
@@ -0,0 +1,20 @@
+package sync
+
+import (
+	. "gopkg.in/check.v1"
+	"net/http"
+)
+
+type TestSuite struct{}
+
+var _ = Suite(&TestSuite{})
+
+func (s *TestSuite) TestRemoveReferer(c *C) {
+	req, err := http.NewRequest("HEAD", "https://foo.bar", nil)
+	c.Assert(err, IsNil)
+	req.Header.Set("Referer", "https://foo.bar")
+	req.Header.Set("Foo", "foo")
+	removeReferer(req)
+	c.Assert(req.Referer(), Equals, "")
+	c.Assert(req.Header, HasLen, 1)
+}