datadog: fix race in test

While the metrics are set in the correct order, the use of a HTTP
server to receive them means that the metrics can be received on the
server side in any order, which fails the tests 1 time out of 200,
potentially higher.

This means we can't rely on any ordering on the server side in the
test, and need to manually sort events.

datadog/server_test.go

@@ -3,6 +3,8 @@ package datadog
 import (
 	"io"
 	"net"
+	"sort"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -14,16 +16,28 @@ func TestServer(t *testing.T) {
 	engine := stats.NewEngine("datadog.test", nil)
 
 	a := uint32(0)
-	b := uint32(0)
 	c := uint32(0)
 
+	seenGauges := make([]Metric, 0)
+	var mu sync.Mutex
+
 	addr, closer := startTestServer(t, HandlerFunc(func(m Metric, _ net.Addr) {
 		switch m.Name {
 		case "datadog.test.A":
 			atomic.AddUint32(&a, uint32(m.Value))
 
 		case "datadog.test.B":
-			atomic.StoreUint32(&b, uint32(m.Value)) // gauge
+			// Because it's the other side of a HTTP server, these can arrive
+			// out of order, even if the client sends them in the right order
+			// - there aren't any guarantees about which connection the server
+			// will activate first.
+			//
+			// Previously this used atomic.StoreInt32 to do last write wins, but
+			// occasionally the last write would be "2" or "1" and fail the
+			// test, easily reproducible by running this test 200 times.
+			mu.Lock()
+			seenGauges = append(seenGauges, m)
+			mu.Unlock()
 
 		case "datadog.test.C":
 			atomic.AddUint32(&c, uint32(m.Value))
@@ -42,25 +56,35 @@ func TestServer(t *testing.T) {
 	engine.Incr("A")
 	engine.Incr("A")
 
-	engine.Set("B", 1)
-	engine.Set("B", 2)
-	engine.Set("B", 3)
+	now := time.Now()
+	engine.Set("B", float64(time.Since(now)))
+	engine.Set("B", float64(time.Since(now)))
+	last := float64(time.Since(now))
+	engine.Set("B", last)
 
 	engine.Observe("C", 1)
 	engine.Observe("C", 2)
 	engine.Observe("C", 3)
 
+	// because this is "last write wins" it's possible it runs before the reads
+	// of 1 or 2; add a sleep to try to ensure it loses the race
 	engine.Flush()
 
 	// Give time for the server to receive the metrics.
-	time.Sleep(100 * time.Millisecond)
+	time.Sleep(20 * time.Millisecond)
 
 	if n := atomic.LoadUint32(&a); n != 3 { // two increments (+1, +1, +1)
 		t.Error("datadog.test.A: bad value:", n)
 	}
 
-	if n := atomic.LoadUint32(&b); n != 3 { // three assignments (=1, =2, =3)
-		t.Error("datadog.test.B: bad value:", n)
+	if len(seenGauges) != 3 {
+		t.Errorf("datadog.test.B: expected 3 values, got %d", len(seenGauges))
+	}
+	sort.Slice(seenGauges, func(i, j int) bool {
+		return seenGauges[i].Value < seenGauges[j].Value
+	})
+	if seenGauges[2].Value != last {
+		t.Errorf("expected highest value to be the latest value set, got %v", seenGauges[2])
 	}
 
 	if n := atomic.LoadUint32(&c); n != 6 { // observed values, all reported (+1, +2, +3)