Merge pull request #66 from matrix-org/kegan/mitmdump-introspection

Replace tcpdump with mitmdump

ENVIRONMENT.md

@@ -4,16 +4,16 @@
 Complement-Crypto is configured exclusively through the use of environment variables. These variables are described below. Additional environment variables can be used, and are outlined at https://github.com/matrix-org/complement/blob/main/ENVIRONMENT.md 
 Complement-Crypto always runs in dirty mode (homeservers exist for the entire duration of the test suite) for performance reasons.
 
+#### `COMPLEMENT_CRYPTO_MITMDUMP`
+The path to dump the output from `mitmdump`. This file can then be used with mitmweb to view all the HTTP flows in the test.  
+- Type: `string`
+- Default: ""
+
 #### `COMPLEMENT_CRYPTO_RPC_BINARY`
 The absolute path to the pre-built rpc binary file. This binary is generated via `go build -tags=jssdk,rust ./cmd/rpc`. This binary is used when running multiprocess tests. If this environment variable is not supplied, tests which try to use multiprocess clients will be skipped, making this environment variable optional.  
 - Type: `string`
 - Default: ""
 
-#### `COMPLEMENT_CRYPTO_TCPDUMP`
-If 1, automatically attempts to run `tcpdump` when the containers are running. Stops dumping when tests complete. This will probably require you to run `go test` with `sudo -E`. The `.pcap` file is written to `tests/test.pcap`.  
-- Type: `bool`
-- Default: 0
-
 #### `COMPLEMENT_CRYPTO_TEST_CLIENT_MATRIX`
 The client test matrix to run. Every test is run for each given permutation. The default matrix tests all JS/Rust permutations _ignoring federation_. 
 ```

FAQ.md

@@ -23,7 +23,7 @@ Now you can look around those log lines for any warnings/errors or unexpected be
 
 Sometimes the bug cannot be found via log files alone. You may want to see server logs. To do this, [enable writing container logs](https://github.com/matrix-org/complement-crypto/blob/main/ENVIRONMENT.md#complement_crypto_write_container_logs) then re-run the test. 
 
-Sometimes, even that isn't enough. Perhaps server logs aren't giving enough information. In that case, [enable tcpdump](https://github.com/matrix-org/complement-crypto/blob/main/ENVIRONMENT.md#complement_crypto_tcpdump) and open the `.pcap` file in Wireshark to see the raw HTTP request/responses made by all clients.
+Sometimes, even that isn't enough. Perhaps server logs aren't giving enough information. In that case, [enable mitmdump](https://github.com/matrix-org/complement-crypto/blob/main/ENVIRONMENT.md#complement_crypto_mitmdump) and open the dump file in mitmweb to see the raw HTTP request/responses made by all clients. If you don't have mitmweb, run `./open_mitmweb.sh` which will use the mitmproxy image.
 
 If you need to add console logging to clients, see below.
 

README.md

@@ -77,7 +77,7 @@ go test -v -count=1 -tags=jssdk -timeout 15m ./tests
 `COMPLEMENT_CRYPTO_TEST_CLIENT_MATRIX` controls which SDK is used to create test clients, and the `-tags` option
 controls conditional compilation so other SDKs don't need to be compiled for the tests to run.
 
-To test interoperability between the SDKs, `tcpdump` the traffic, run extra multiprocess tests and more,
+To test interoperability between the SDKs, `mitmdump` the traffic, run extra multiprocess tests and more,
 see [ENVIRONMENT.md](ENVIRONMENT.md) for the full configuration options.
 
 *See [FAQ.md](FAQ.md) for more information around debugging.*

internal/config/config.go

@@ -35,12 +35,11 @@ type ComplementCrypto struct {
 	// Derived from TestClientMatrix
 	clientLangs map[api.ClientTypeLang]bool
 
-	// Name: COMPLEMENT_CRYPTO_TCPDUMP
-	// Default: 0
-	// Description: If 1, automatically attempts to run `tcpdump` when the containers are running. Stops dumping when
-	// tests complete. This will probably require you to run `go test` with `sudo -E`. The `.pcap` file is written to
-	// `tests/test.pcap`.
-	TCPDump bool
+	// Name: COMPLEMENT_CRYPTO_MITMDUMP
+	// Default: ""
+	// Description: The path to dump the output from `mitmdump`. This file can then be used with mitmweb to view
+	// all the HTTP flows in the test.
+	MITMDump string
 
 	// Name: COMPLEMENT_CRYPTO_RPC_BINARY
 	// Default: ""
@@ -123,7 +122,7 @@ func NewComplementCryptoConfigFromEnvVars() *ComplementCrypto {
 		}
 	}
 	return &ComplementCrypto{
-		TCPDump:          os.Getenv("COMPLEMENT_CRYPTO_TCPDUMP") == "1",
+		MITMDump:         os.Getenv("COMPLEMENT_CRYPTO_MITMDUMP"),
 		RPCBinaryPath:    rpcBinaryPath,
 		TestClientMatrix: testClientMatrix,
 		clientLangs:      clientLangs,

internal/deploy/deploy.go

@@ -11,7 +11,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -35,14 +34,16 @@ import (
 // must match the value in tests/addons/__init__.py
 const magicMITMURL = "http://mitm.code"
 
+const mitmDumpFilePathOnContainer = "/tmp/mitm.dump"
+
 type SlidingSyncDeployment struct {
 	complement.Deployment
 	extraContainers      map[string]testcontainers.Container
 	mitmClient           *http.Client
 	ControllerURL        string
 	dnsToReverseProxyURL map[string]string
 	mu                   sync.RWMutex
-	tcpdump              *exec.Cmd
+	mitmDumpFile         string
 }
 
 func (d *SlidingSyncDeployment) WithSniffedEndpoint(t *testing.T, partialPath string, onSniff func(CallbackData), inner func()) {
@@ -142,7 +143,29 @@ func (d *SlidingSyncDeployment) withReverseProxyURL(hsName string, c *client.CSA
 	return c
 }
 
+func (d *SlidingSyncDeployment) writeMITMDump() {
+	if d.mitmDumpFile == "" {
+		return
+	}
+	log.Printf("dumping mitmdump to '%s'\n", d.mitmDumpFile)
+	fileContents, err := d.extraContainers["mitmproxy"].CopyFileFromContainer(context.Background(), mitmDumpFilePathOnContainer)
+	if err != nil {
+		log.Printf("failed to copy mitmdump from container: %s", err)
+		return
+	}
+	contents, err := io.ReadAll(fileContents)
+	if err != nil {
+		log.Printf("failed to read mitmdump: %s", err)
+		return
+	}
+	if err = os.WriteFile(d.mitmDumpFile, contents, os.ModePerm); err != nil {
+		log.Printf("failed to write mitmdump to %s: %s", d.mitmDumpFile, err)
+		return
+	}
+}
+
 func (d *SlidingSyncDeployment) Teardown() {
+	d.writeMITMDump()
 	for name, c := range d.extraContainers {
 		filename := fmt.Sprintf("container-%s.log", name)
 		logs, err := c.Logs(context.Background())
@@ -186,13 +209,9 @@ func (d *SlidingSyncDeployment) Teardown() {
 			log.Fatalf("failed to stop %s: %s", name, err)
 		}
 	}
-	if d.tcpdump != nil {
-		fmt.Println("Sent SIGINT to tcpdump, waiting for it to exit, err=", d.tcpdump.Process.Signal(os.Interrupt))
-		fmt.Println("tcpdump finished, err=", d.tcpdump.Wait())
-	}
 }
 
-func RunNewDeployment(t *testing.T, mitmProxyAddonsDir string, shouldTCPDump bool) *SlidingSyncDeployment {
+func RunNewDeployment(t *testing.T, mitmProxyAddonsDir string, mitmDumpFile string) *SlidingSyncDeployment {
 	// allow time for everything to deploy
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
@@ -261,6 +280,7 @@ func RunNewDeployment(t *testing.T, mitmProxyAddonsDir string, shouldTCPDump boo
 			"--mode", "reverse:http://ssproxy1:6789@3002",
 			"--mode", "reverse:http://ssproxy2:6789@3003",
 			"--mode", "regular",
+			"-w", mitmDumpFilePathOnContainer,
 		},
 		Networks: []string{networkName},
 		NetworkAliases: map[string][]string{
@@ -353,25 +373,6 @@ func RunNewDeployment(t *testing.T, mitmProxyAddonsDir string, shouldTCPDump boo
 	t.Logf("  synapse:      hs2          %s (rp=%s)", csapi2.BaseURL, rpHS2URL)
 	t.Logf("  postgres:     postgres")
 	t.Logf("  mitmproxy:    mitmproxy    controller=%s", controllerURL)
-	var cmd *exec.Cmd
-	if shouldTCPDump {
-		t.Log("Running tcpdump...")
-		urlsToTCPDump := []string{
-			rpSS1URL, rpSS2URL, csapi1.BaseURL, csapi2.BaseURL, rpHS1URL, rpHS2URL, controllerURL,
-		}
-		tcpdumpFilter := []string{}
-		for _, u := range urlsToTCPDump {
-			parsedURL, _ := url.Parse(u)
-			tcpdumpFilter = append(tcpdumpFilter, fmt.Sprintf("port %s", parsedURL.Port()))
-		}
-		filter := fmt.Sprintf("tcp " + strings.Join(tcpdumpFilter, " or "))
-		cmd = exec.Command("tcpdump", "-i", "any", "-s", "0", filter, "-w", "test.pcap")
-		t.Log(cmd.String())
-		if err := cmd.Start(); err != nil {
-			t.Fatalf("tcpdump failed: %v", err)
-		}
-		t.Logf("Started tcpdumping (requires sudo): PID %d", cmd.Process.Pid)
-	}
 	// without this, GHA will fail when trying to hit the controller with "Post "http://mitm.code/options/lock": EOF"
 	// suspected IPv4 vs IPv6 problems in Docker as Flask is listening on v4/v6.
 	controllerURL = strings.Replace(controllerURL, "localhost", "127.0.0.1", 1)
@@ -385,7 +386,6 @@ func RunNewDeployment(t *testing.T, mitmProxyAddonsDir string, shouldTCPDump boo
 			"postgres":  postgresContainer,
 			"mitmproxy": mitmproxyContainer,
 		},
-		tcpdump:       cmd,
 		ControllerURL: controllerURL,
 		mitmClient: &http.Client{
 			Timeout: 5 * time.Second,
@@ -399,6 +399,7 @@ func RunNewDeployment(t *testing.T, mitmProxyAddonsDir string, shouldTCPDump boo
 			"ssproxy1": rpSS1URL,
 			"ssproxy2": rpSS2URL,
 		},
+		mitmDumpFile: mitmDumpFile,
 	}
 }
 

internal/tests/client_test.go

@@ -36,7 +36,7 @@ func Deploy(t *testing.T) *deploy.SlidingSyncDeployment {
 	if ssDeployment != nil {
 		return ssDeployment
 	}
-	ssDeployment = deploy.RunNewDeployment(t, "", false)
+	ssDeployment = deploy.RunNewDeployment(t, "", "")
 	return ssDeployment
 }
 

open_mitmweb.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+if [ "$1" = "-h" ] || [ "$1" = "--help" ];
+then
+    echo "Opens a browser with mitmweb. Then you can open a dump file made via COMPLEMENT_CRYPTO_MITMDUMP. (requires on PATH: docker)"
+    exit 1
+fi
+
+# - use python3 instead of xdg-open because it's more portable (xdg-open doesn't work on MacOS). Sleep 1s and do it in the background.
+(sleep 1 && python3 -m webbrowser http://localhost:1445) &
+
+# - use same version as tests so we don't need to pull any new image. When the user CTRL+Cs this, the container quits.
+docker run --rm -p 1445:8081 mitmproxy/mitmproxy:10.1.5  mitmweb --web-host 0.0.0.0

tests/main_test.go

@@ -59,7 +59,7 @@ func Deploy(t *testing.T) *deploy.SlidingSyncDeployment {
 		t.Fatalf("failed to find working directory: %s", err)
 	}
 	mitmProxyAddonsDir := filepath.Join(workingDir, "mitmproxy_addons")
-	ssDeployment = deploy.RunNewDeployment(t, mitmProxyAddonsDir, complementCryptoConfig.TCPDump)
+	ssDeployment = deploy.RunNewDeployment(t, mitmProxyAddonsDir, complementCryptoConfig.MITMDump)
 	return ssDeployment
 }