Decompress request body before recording and processing

core/modes/diff_mode.go

@@ -1,26 +1,22 @@
 package modes
 
 import (
-	"net/http"
-
-	"github.com/SpectoLabs/hoverfly/core/errors"
-
-	log "github.com/sirupsen/logrus"
-
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"reflect"
-	"time"
-
+	"github.com/SpectoLabs/hoverfly/core/errors"
 	"github.com/SpectoLabs/hoverfly/core/handlers/v2"
 	"github.com/SpectoLabs/hoverfly/core/models"
 	"github.com/SpectoLabs/hoverfly/core/util"
 	"github.com/dsnet/compress/brotli"
+	log "github.com/sirupsen/logrus"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"reflect"
+	"time"
 )
 
 type HoverflyDiff interface {
@@ -203,14 +199,47 @@ func unmarshalResponseToInterface(response *models.ResponseDetails, output inter
 	return err
 }
 
+func (this *DiffMode) JsonDiff(prefix string, expected map[string]interface{}, actual map[string]interface{}) bool {
+	same := true
+	for k := range expected {
+		param := prefix + "/" + k
+		if _, ok := actual[k]; !ok {
+			this.addEntry(param, expected[k], nil)
+			same = false
+		} else if reflect.TypeOf(expected[k]) != reflect.TypeOf(actual[k]) {
+			this.addEntry(param, expected[k], actual[k])
+			same = false
+		} else {
+			switch expected[k].(type) {
+			default:
+				if expected[k] != actual[k] {
+					this.addEntry(param, expected[k], actual[k])
+					same = false
+				}
+			case map[string]interface{}:
+				if !this.JsonDiff(param, expected[k].(map[string]interface{}), actual[k].(map[string]interface{})) {
+					same = false
+				}
+			case []interface{}:
+				if !reflect.DeepEqual(expected[k], actual[k]) {
+					this.addEntry(param, expected[k], actual[k])
+					same = false
+				}
+			}
+		}
+	}
+
+	return same
+}
+
 func decompress(body []byte, encodings []string) ([]byte, error) {
 	var err error
 	var reader io.ReadCloser
 	if len(encodings) > 0 {
 		for index := range encodings {
 			switch encodings[index] {
 			case "gzip":
-				reader, err = gzip.NewReader(ioutil.NopCloser(bytes.NewBuffer(body)))
+				reader, err = gzip.NewReader(bytes.NewBuffer(body))
 				if err != nil {
 					return body, err
 				}
@@ -241,36 +270,3 @@ func decompress(body []byte, encodings []string) ([]byte, error) {
 	}
 	return body, err
 }
-
-func (this *DiffMode) JsonDiff(prefix string, expected map[string]interface{}, actual map[string]interface{}) bool {
-	same := true
-	for k := range expected {
-		param := prefix + "/" + k
-		if _, ok := actual[k]; !ok {
-			this.addEntry(param, expected[k], nil)
-			same = false
-		} else if reflect.TypeOf(expected[k]) != reflect.TypeOf(actual[k]) {
-			this.addEntry(param, expected[k], actual[k])
-			same = false
-		} else {
-			switch expected[k].(type) {
-			default:
-				if expected[k] != actual[k] {
-					this.addEntry(param, expected[k], actual[k])
-					same = false
-				}
-			case map[string]interface{}:
-				if !this.JsonDiff(param, expected[k].(map[string]interface{}), actual[k].(map[string]interface{})) {
-					same = false
-				}
-			case []interface{}:
-				if !reflect.DeepEqual(expected[k], actual[k]) {
-					this.addEntry(param, expected[k], actual[k])
-					same = false
-				}
-			}
-		}
-	}
-
-	return same
-}

core/modes/modes.go

@@ -3,6 +3,7 @@ package modes
 import (
 	"bytes"
 	"fmt"
+	"github.com/SpectoLabs/hoverfly/core/util"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -53,15 +54,34 @@ func ReconstructRequest(pair models.RequestResponsePair) (*http.Request, error)
 		return nil, fmt.Errorf("failed to reconstruct request, destination not specified")
 	}
 
+	bodyBytes := []byte(pair.Request.Body)
+	// recompress the request body if the original was encoded
+	if values, found := pair.Request.Headers["Content-Encoding"]; found {
+		for _, value := range values {
+			// Only gzip is supported at the moment
+			if value == "gzip" {
+				compressedBody, err := util.CompressGzip(bodyBytes)
+				if err == nil {
+					bodyBytes = compressedBody
+				} else {
+					// Fail to compress, we should remove the encoding header
+					delete(pair.Request.Headers, "Content-Encoding")
+				}
+				break
+			}
+		}
+	}
+
 	newRequest, err := http.NewRequest(
 		pair.Request.Method,
 		fmt.Sprintf("%s://%s%s", pair.Request.Scheme, pair.Request.Destination, pair.Request.Path),
-		bytes.NewBuffer([]byte(pair.Request.Body)))
+		bytes.NewBuffer(bodyBytes))
 
 	if err != nil {
 		return nil, err
 	}
 
+
 	newRequest.Method = pair.Request.Method
 	newRequest.Header = pair.Request.Headers
 

core/modes/modes_test.go

@@ -2,6 +2,7 @@ package modes_test
 
 import (
 	"errors"
+	"github.com/SpectoLabs/hoverfly/core/util"
 	"io/ioutil"
 	"net/http"
 	"testing"
@@ -80,6 +81,32 @@ func Test_ReconstructRequest_BodyRequestResponsePair(t *testing.T) {
 	Expect(string(body)).To(Equal("new request body here"))
 }
 
+func Test_ReconstructRequest_ShouldRecompressGzipBody(t *testing.T) {
+	RegisterTestingT(t)
+
+	request := models.RequestDetails{
+		Scheme:      "http",
+		Path:        "/another-path",
+		Method:      "POST",
+		Destination: "test-destination.com",
+		Body:        "new request body here",
+		Headers: 	 map[string][]string{"Content-Encoding": {"gzip"}},
+	}
+	pair := models.RequestResponsePair{Request: request}
+
+	newRequest, err := modes.ReconstructRequest(pair)
+	Expect(err).To(BeNil())
+
+	body, err := ioutil.ReadAll(newRequest.Body)
+	Expect(err).To(BeNil())
+
+	Expect(string(body)).To(Not(Equal("new request body here")))
+
+	decompressedBody, err := util.DecompressGzip(body)
+	Expect(err).To(BeNil())
+	Expect(string(decompressedBody)).To(Equal("new request body here"))
+}
+
 func Test_ReconstructRequest_HeadersInPair(t *testing.T) {
 	RegisterTestingT(t)
 

core/util/util.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
@@ -20,12 +21,21 @@ import (
 // GetRequestBody will read the http.Request body io.ReadCloser
 // and will also set the buffer to the original value as the
 // buffer will be empty after reading it.
+// It also decompress if any Content-Encoding is applied
 func GetRequestBody(request *http.Request) (string, error) {
 	bodyBytes, err := ioutil.ReadAll(request.Body)
 	if err != nil {
 		return "", err
 	}
 
+	// Will add more compression support in the future
+	if request.Header.Get("Content-Encoding") == "gzip" {
+		decompressedBody, err := DecompressGzip(bodyBytes)
+		if err == nil {
+			bodyBytes = decompressedBody
+		}
+	}
+
 	request.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
 
 	return string(bodyBytes), nil
@@ -186,3 +196,33 @@ func CopyMap(originalMap map[string]string) map[string]string {
 	}
 	return newMap
 }
+
+func DecompressGzip(body []byte) ([]byte, error) {
+	reader, err := gzip.NewReader(bytes.NewBuffer(body))
+	if err != nil {
+		return body, err
+	}
+	defer reader.Close()
+	body, err = ioutil.ReadAll(reader)
+	if err != nil {
+		return body, err
+	}
+	return body, err
+}
+
+func CompressGzip(body []byte) ([]byte, error) {
+	var byteBuffer bytes.Buffer
+	var err error
+	gzWriter := gzip.NewWriter(&byteBuffer)
+	if _, err := gzWriter.Write(body); err != nil {
+		return body, err
+	}
+	if err := gzWriter.Flush(); err != nil {
+		return body, err
+	}
+	if err := gzWriter.Close(); err != nil {
+		return body, err
+	}
+
+	return byteBuffer.Bytes(), err
+}

core/util/util_test.go

@@ -36,6 +36,28 @@ func Test_GetRequestBody_GettingTheRequestBodySetsTheSameBodyAgain(t *testing.T)
 	Expect(string(newRequestBody)).To(Equal("test-preserve"))
 }
 
+func Test_GetRequestBody_DecompressGzipContent(t *testing.T) {
+	RegisterTestingT(t)
+
+	request, _ := http.NewRequest("POST", "", nil)
+	request.Header.Set("Content-Encoding", "gzip")
+	originalBody := "hello_world"
+
+	compressedBody, err := CompressGzip([]byte(originalBody))
+	Expect(err).To(BeNil())
+	Expect(string(compressedBody)).To(Not(Equal(originalBody)))
+	request.Body = ioutil.NopCloser(bytes.NewBuffer(compressedBody))
+
+	_, err = GetRequestBody(request)
+	Expect(err).To(BeNil())
+
+	newRequestBody, err := ioutil.ReadAll(request.Body)
+	Expect(err).To(BeNil())
+
+	Expect(string(newRequestBody)).To(Equal(originalBody))
+}
+
+
 func Test_GetResponseBody_GettingTheResponseBodyGetsTheCorrectData(t *testing.T) {
 	RegisterTestingT(t)