Investigate flaky tests on CI

.github/workflows/test-flaky.yml

@@ -0,0 +1,61 @@
+name: Test Repeat Flaky
+
+on:
+  pull_request:
+
+jobs:
+  #  cache-test-bins:
+  #    name: Prepare go test cache
+  #    runs-on: ubuntu-latest
+  #    strategy:
+  #      matrix:
+  #        go:
+  #          - 1.22
+  #          - 1.23
+  #    steps:
+  #      - name: Checkout
+  #        uses: actions/checkout@v4
+  #      - name: Setup Go ${{ matrix.go }}
+  #        uses: actions/setup-go@v5
+  #        with:
+  #          go-version: ${{ matrix.go }}
+  #          cache: false
+  #      - name: Compile tests
+  #        run: go test -c -o f3-tests
+  #      - name: Cache compiled tests
+  #        uses: actions/cache/save@v4
+  #        with:
+  #          path: './f3-tests'
+  #          key: f3-tests-flaky-${{ matrix.go }}
+  test:
+    name: ${{ matrix.test }} ${{ matrix.go }}
+    runs-on: ubuntu-latest
+    #    needs: cache-test-bins
+    strategy:
+      fail-fast: false
+      matrix:
+        go:
+          - 1.22
+          - 1.23
+#        test:
+#          - TestF3LateBootstrap
+#          - TestF3PauseResumeCatchup
+#          - TestF3WithLookback
+#          - TestF3FailRecover
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      #      - name: Restore cached built F3 test binaries
+      #        uses: actions/cache/restore@v4
+      #        with:
+      #          path: './f3-tests'
+      #          key: f3-tests-flaky-${{ matrix.go }}
+      #      - name: Repeat ${{ matrix.test }}
+      #        run: ./f3-tests -test.run=${{ matrix.test }} -test.count 100
+      - name: Setup Go ${{ matrix.go }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go }}
+          cache: false
+      - name: Repeat ${{ matrix.test }}
+        run: go test -v -timeout 30m -count 50 . --tags=theflakybunch

f3_test.go

@@ -1,3 +1,6 @@
+//go:build theflakybunch
+// +build theflakybunch
+
 package f3_test
 
 import (
@@ -37,13 +40,11 @@ func init() {
 var manifestSenderTimeout = 10 * time.Second
 
 func TestF3Simple(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).start()
 	env.waitForInstanceNumber(5, 10*time.Second, false)
 }
 
 func TestF3WithLookback(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).withManifest(func(m *manifest.Manifest) {
 		m.EC.HeadLookback = 20
 	}).start()
@@ -89,7 +90,6 @@ func TestF3WithLookback(t *testing.T) {
 }
 
 func TestF3PauseResumeCatchup(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(3).start()
 
 	env.waitForInstanceNumber(1, 30*time.Second, true)
@@ -128,7 +128,6 @@ func TestF3PauseResumeCatchup(t *testing.T) {
 }
 
 func TestF3FailRecover(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2)
 
 	// Make it possible to fail a single write for node 0.
@@ -159,7 +158,6 @@ func TestF3FailRecover(t *testing.T) {
 }
 
 func TestF3DynamicManifest_WithoutChanges(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).withDynamicManifest()
 
 	env.start()
@@ -172,7 +170,6 @@ func TestF3DynamicManifest_WithoutChanges(t *testing.T) {
 }
 
 func TestF3DynamicManifest_WithRebootstrap(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).withDynamicManifest().start()
 
 	prev := env.nodes[0].f3.Manifest()
@@ -214,7 +211,6 @@ func TestF3DynamicManifest_WithRebootstrap(t *testing.T) {
 }
 
 func TestF3DynamicManifest_WithPauseAndRebootstrap(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).withDynamicManifest().start()
 
 	env.waitForInstanceNumber(10, 30*time.Second, true)
@@ -242,7 +238,6 @@ func TestF3DynamicManifest_WithPauseAndRebootstrap(t *testing.T) {
 }
 
 func TestF3LateBootstrap(t *testing.T) {
-	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).start()
 
 	// Wait till we're "caught up".