Automatically gzip responses (#29)

It seems convenient to do this in CTile rather than Nginx because
(currently) it's easier for us to scale the number of CTile instances
than the number of Nginx instances.

Using NYTimes/gziphandler simplifies a little bit the handling of
different cases (like whether or not the requester accepts gzip
encoding).

I incorporated the gzip handler into `tileCachingHandler` rather than
setting it up in main, in order to make it easier to test inside the
current integration test.

go.mod

@@ -3,6 +3,7 @@ module github.com/letsencrypt/ctile
 go 1.20
 
 require (
+	github.com/NYTimes/gziphandler v1.1.1
 	github.com/aws/aws-sdk-go-v2 v1.21.0
 	github.com/aws/aws-sdk-go-v2/config v1.18.37
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.35

go.sum

@@ -1,3 +1,5 @@
+github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
 github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
@@ -55,6 +57,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
 github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
@@ -65,6 +68,8 @@ github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -79,3 +84,4 @@ google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cn
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

integration_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/binary"
 	"encoding/json"
@@ -275,7 +276,11 @@ func TestIntegration(t *testing.T) {
 	if resp.StatusCode != 400 {
 		t.Errorf("expected 400 got %d", resp.StatusCode)
 	}
-	body, _ := io.ReadAll(resp.Body)
+	gzReader, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, _ := io.ReadAll(gzReader)
 	if !strings.Contains(string(body), testLogSaysPastTheEnd) {
 		t.Errorf("expected response to contain %q got %q", testLogSaysPastTheEnd, body)
 	}
@@ -310,6 +315,7 @@ func TestIntegration(t *testing.T) {
 
 func getResp(ctile *tileCachingHandler, url string) *http.Response {
 	req := httptest.NewRequest("GET", url, nil)
+	req.Header.Set("Accept-Encoding", "gzip")
 	w := httptest.NewRecorder()
 
 	ctile.ServeHTTP(w, req)
@@ -324,8 +330,20 @@ func getAndParseResp(t *testing.T, ctile *tileCachingHandler, url string) (entri
 	if resp.StatusCode != 200 {
 		t.Fatalf("%q: expected status code 200 got %d with body: %q", url, resp.StatusCode, body)
 	}
+	if resp.Header.Get("Content-Encoding") != "gzip" {
+		t.Fatalf("expected Content-Encoding: gzip, got %q", resp.Header.Get("Content-Encoding"))
+	}
+	gzipReader, err := gzip.NewReader(bytes.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	jsonBytes, err := io.ReadAll(gzipReader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	var entries entries
-	err := json.Unmarshal(body, &entries)
+	err = json.Unmarshal(jsonBytes, &entries)
 	return entries, resp.Header, err
 }
 

main.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/NYTimes/gziphandler"
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -294,6 +295,8 @@ type tileCachingHandler struct {
 	backendLatencyMetric *prometheus.HistogramVec
 
 	fullRequestTimeout time.Duration
+
+	gzipHandler http.Handler
 }
 
 func newTileCachingHandler(
@@ -363,7 +366,7 @@ func newTileCachingHandler(
 		[]string{"backend"})
 	promRegisterer.MustRegister(backendLatencyMetric)
 
-	return &tileCachingHandler{
+	tch := tileCachingHandler{
 		logURL:               logURL,
 		tileSize:             tileSize,
 		s3Service:            s3Service,
@@ -376,10 +379,23 @@ func newTileCachingHandler(
 		fullRequestTimeout:   fullRequestTimeout,
 		latencyMetric:        latencyMetric,
 		backendLatencyMetric: backendLatencyMetric,
-	}, nil
+	}
+
+	handlerMaker, err := gziphandler.NewGzipLevelAndMinSize(gzip.BestSpeed, 100)
+	if err != nil {
+		return nil, err
+	}
+
+	tch.gzipHandler = handlerMaker(http.HandlerFunc(tch.serveHTTPInner))
+
+	return &tch, nil
 }
 
 func (tch *tileCachingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	tch.gzipHandler.ServeHTTP(w, r)
+}
+
+func (tch *tileCachingHandler) serveHTTPInner(w http.ResponseWriter, r *http.Request) {
 	begin := time.Now()
 	defer func() {
 		tch.latencyMetric.Observe(time.Since(begin).Seconds())