refactor: extend the integration testing framework to support multipl…

…e languages Previously we had full test coverage in each language, so we would just run them all, but as we're building up support for other languages for now, this change allows multiple languages to be supported per-test.
TBD54566975 · Aug 14, 2024 · 0bebc0f · 0bebc0f
1 parent 4a24b88
commit 0bebc0f
Show file tree

Hide file tree

Showing 21 changed files with 217 additions and 132 deletions.
diff --git a/backend/controller/admin/local_client_test.go b/backend/controller/admin/local_client_test.go
@@ -6,15 +6,18 @@ import (
 	"context"
 	"testing"
 
+	"github.com/alecthomas/assert/v2"
+	"github.com/alecthomas/types/optional"
+
 	cf "github.com/TBD54566975/ftl/common/configuration"
 	in "github.com/TBD54566975/ftl/integration"
 	"github.com/TBD54566975/ftl/internal/log"
-	"github.com/alecthomas/assert/v2"
-	"github.com/alecthomas/types/optional"
 )
 
 func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) {
-	in.RunWithoutController(t, "ftl-project-dr.toml",
+	in.Run(t,
+		in.WithFTLConfig("ftl-project-dr.toml"),
+		in.WithoutController(),
 		in.CopyModule("dischema"),
 		in.Build("dischema"),
 		func(t testing.TB, ic in.TestContext) {
@@ -30,7 +33,9 @@ func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) {
 }
 
 func TestDiskSchemaRetrieverWithNoSchema(t *testing.T) {
-	in.RunWithoutController(t, "ftl-project-dr.toml",
+	in.Run(t,
+		in.WithFTLConfig("ftl-project-dr.toml"),
+		in.WithoutController(),
 		in.CopyModule("dischema"),
 		func(t testing.TB, ic in.TestContext) {
 			dsr := &diskSchemaRetriever{}

diff --git a/backend/controller/console/console_integration_test.go b/backend/controller/console/console_integration_test.go
@@ -6,9 +6,10 @@ import (
 	"testing"
 
 	"connectrpc.com/connect"
+	"github.com/alecthomas/assert/v2"
+
 	pbconsole "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/console"
 	in "github.com/TBD54566975/ftl/integration"
-	"github.com/alecthomas/assert/v2"
 )
 
 // GetModules calls console service GetModules and returns the response.
@@ -24,7 +25,7 @@ func GetModules(onResponse func(t testing.TB, resp *connect.Response[pbconsole.G
 }
 
 func TestConsoleGetModules(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("console"),
 		in.Deploy("console"),
 		GetModules(func(t testing.TB, resp *connect.Response[pbconsole.GetModulesResponse]) {

diff --git a/backend/controller/cronjobs/cronjobs_integration_test.go b/backend/controller/cronjobs/cronjobs_integration_test.go
@@ -53,7 +53,7 @@ func TestCron(t *testing.T) {
 
 	t.Cleanup(func() { _ = os.Remove(tmpFile) })
 
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("cron"),
 		in.Deploy("cron"),
 		func(t testing.TB, ic in.TestContext) {

diff --git a/backend/controller/dal/fsm_integration_test.go b/backend/controller/dal/fsm_integration_test.go
@@ -8,8 +8,9 @@ import (
 	"testing"
 	"time"
 
-	in "github.com/TBD54566975/ftl/integration"
 	"github.com/alecthomas/assert/v2"
+
+	in "github.com/TBD54566975/ftl/integration"
 )
 
 func TestFSM(t *testing.T) {
@@ -22,7 +23,7 @@ func TestFSM(t *testing.T) {
 			WHERE fsm = 'fsm.fsm' AND key = '%s'
 		`, instance), status, state)
 	}
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("fsm"),
 		in.Deploy("fsm"),
 
@@ -81,7 +82,7 @@ func TestFSMRetry(t *testing.T) {
 		}
 	}
 
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("fsmretry"),
 		in.Build("fsmretry"),
 		in.Deploy("fsmretry"),
@@ -127,7 +128,7 @@ func TestFSMRetry(t *testing.T) {
 func TestFSMGoTests(t *testing.T) {
 	logFilePath := filepath.Join(t.TempDir(), "fsm.log")
 	t.Setenv("FSM_LOG_FILE", logFilePath)
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("fsm"),
 		in.Build("fsm"),
 		in.ExecModuleTest("fsm"),

diff --git a/backend/controller/ingress/ingress_integration_test.go b/backend/controller/ingress/ingress_integration_test.go
@@ -14,7 +14,7 @@ import (
 )
 
 func TestHttpIngress(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("httpingress"),
 		in.Deploy("httpingress"),
 		in.HttpCall(http.MethodGet, "/users/123/posts/456", nil, in.JsonData(t, in.Obj{}), func(t testing.TB, resp *in.HTTPResponse) {
@@ -152,7 +152,7 @@ func TestHttpIngress(t *testing.T) {
 func TestHttpIngressWithCors(t *testing.T) {
 	os.Setenv("FTL_CONTROLLER_ALLOW_ORIGIN", "http://localhost:8892")
 	os.Setenv("FTL_CONTROLLER_ALLOW_HEADERS", "x-forwarded-capabilities")
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("httpingress"),
 		in.Deploy("httpingress"),
 		// A correct CORS preflight request

diff --git a/backend/controller/leases/lease_integration_test.go b/backend/controller/leases/lease_integration_test.go
@@ -18,7 +18,7 @@ import (
 )
 
 func TestLease(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("leases"),
 		in.Build("leases"),
 		// checks if leases work in a unit test environment

diff --git a/backend/controller/pubsub/integration_test.go b/backend/controller/pubsub/integration_test.go
@@ -15,7 +15,7 @@ import (
 func TestPubSub(t *testing.T) {
 	calls := 20
 	events := calls * 10
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("publisher"),
 		in.CopyModule("subscriber"),
 		in.Deploy("publisher"),
@@ -40,7 +40,7 @@ func TestPubSub(t *testing.T) {
 }
 
 func TestConsumptionDelay(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("publisher"),
 		in.CopyModule("subscriber"),
 		in.Deploy("publisher"),
@@ -83,7 +83,7 @@ func TestConsumptionDelay(t *testing.T) {
 
 func TestRetry(t *testing.T) {
 	retriesPerCall := 2
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("publisher"),
 		in.CopyModule("subscriber"),
 		in.Deploy("publisher"),
@@ -135,7 +135,7 @@ func TestRetry(t *testing.T) {
 }
 
 func TestExternalPublishRuntimeCheck(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("publisher"),
 		in.CopyModule("subscriber"),
 		in.Deploy("publisher"),

diff --git a/backend/controller/sql/database_integration_test.go b/backend/controller/sql/database_integration_test.go
@@ -10,7 +10,8 @@ import (
 )
 
 func TestDatabase(t *testing.T) {
-	in.Run(t, "database/ftl-project.toml",
+	in.Run(t,
+		in.WithFTLConfig("database/ftl-project.toml"),
 		// deploy real module against "testdb"
 		in.CopyModule("database"),
 		in.CreateDBAction("database", "testdb", false),
@@ -33,7 +34,8 @@ func TestMigrate(t *testing.T) {
 		return in.QueryRow(dbName, "SELECT version FROM schema_migrations WHERE version = '20240704103403'", "20240704103403")
 	}
 
-	in.RunWithoutController(t, "",
+	in.Run(t,
+		in.WithoutController(),
 		in.DropDBAction(t, dbName),
 		in.Fail(q(), "Should fail because the database does not exist."),
 		in.Exec("ftl", "migrate", "--dsn", dbUri),

diff --git a/cmd/ftl/integration_test.go b/cmd/ftl/integration_test.go
@@ -24,7 +24,8 @@ func TestBox(t *testing.T) {
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	err := exec.Command(ctx, log.Debug, "../..", "docker", "build", "-t", "ftl0/ftl-box:latest", "--progress=plain", "--platform=linux/amd64", "-f", "Dockerfile.box", ".").Run()
 	assert.NoError(t, err)
-	RunWithoutController(t, "",
+	Run(t,
+		WithoutController(),
 		CopyModule("time"),
 		CopyModule("echo"),
 		Exec("ftl", "box", "echo", "--compose=echo-compose.yml"),
@@ -35,17 +36,17 @@ func TestBox(t *testing.T) {
 }
 
 func TestConfigsWithController(t *testing.T) {
-	Run(t, "", configActions(t)...)
+	Run(t, configActions(t)...)
 }
 
 func TestConfigsWithoutController(t *testing.T) {
-	RunWithoutController(t, "", configActions(t)...)
+	Run(t, configActions(t, WithoutController())...)
 }
 
-func configActions(t *testing.T) []Action {
+func configActions(t *testing.T, prepend ...ActionOrOption) []ActionOrOption {
 	t.Helper()
 
-	return []Action{
+	return append(prepend,
 		// test setting value without --json flag
 		Exec("ftl", "config", "set", "test.one", "hello world", "--inline"),
 		ExecWithExpectedOutput("\"hello world\"\n", "ftl", "config", "get", "test.one"),
@@ -58,18 +59,18 @@ func configActions(t *testing.T) []Action {
 			ExecWithOutput("ftl", []string{"config", "get", "test.one"}, func(output string) {}),
 			"failed to get from config manager: not found",
 		),
-	}
+	)
 }
 
 func TestSecretsWithController(t *testing.T) {
-	Run(t, "", secretActions(t)...)
+	Run(t, secretActions(t)...)
 }
 
 func TestSecretsWithoutController(t *testing.T) {
-	RunWithoutController(t, "", secretActions(t)...)
+	Run(t, secretActions(t, WithoutController())...)
 }
 
-func secretActions(t *testing.T) []Action {
+func secretActions(t *testing.T, prepend ...ActionOrOption) []ActionOrOption {
 	t.Helper()
 
 	// can not easily use Exec() to enter secure text, using secret import instead
@@ -78,7 +79,7 @@ func secretActions(t *testing.T) []Action {
 	secretsPath2, err := filepath.Abs("testdata/secrets2.json")
 	assert.NoError(t, err)
 
-	return []Action{
+	return append(prepend,
 		// test setting secret without --json flag
 		Exec("ftl", "secret", "import", "--inline", secretsPath1),
 		ExecWithExpectedOutput("\"hello world\"\n", "ftl", "secret", "get", "test.one"),
@@ -91,7 +92,7 @@ func secretActions(t *testing.T) []Action {
 			ExecWithOutput("ftl", []string{"secret", "get", "test.one"}, func(output string) {}),
 			"failed to get from secret manager: not found",
 		),
-	}
+	)
 }
 
 func TestSecretImportExport(t *testing.T) {
@@ -116,7 +117,8 @@ func testImportExport(t *testing.T, object string) {
 	blank := ""
 	exported := &blank
 
-	RunWithoutController(t, "",
+	Run(t,
+		WithoutController(),
 		// duplicate project file in the temp directory
 		Exec("cp", firstProjFile, secondProjFile),
 		// import into first project file
@@ -152,7 +154,7 @@ func NewFunction(ctx context.Context, req TimeRequest) (TimeResponse, error) {
 	return TimeResponse{Time: time.Now()}, nil
 }
 `
-	Run(t, "",
+	Run(t,
 		CopyModule("time"),
 		Deploy("time"),
 		ExecWithOutput("ftl", []string{"schema", "diff"}, func(output string) {
@@ -199,7 +201,7 @@ func TestResetSubscription(t *testing.T) {
 		`, module, subscription), cursor)
 	}
 
-	Run(t, "",
+	Run(t,
 		CopyModule("time"),
 		CopyModule("echo"),
 		Deploy("time"),

diff --git a/common/projectconfig/integration_test.go b/common/projectconfig/integration_test.go
@@ -14,7 +14,8 @@ import (
 )
 
 func TestDefaultToRootWhenModuleDirsMissing(t *testing.T) {
-	in.Run(t, "no-module-dirs-ftl-project.toml",
+	in.Run(t,
+		in.WithFTLConfig("no-module-dirs-ftl-project.toml"),
 		in.CopyModule("echo"),
 		in.Exec("ftl", "build"), // Needs to be `ftl build`, not `ftl build echo`
 		in.Deploy("echo"),
@@ -25,7 +26,9 @@ func TestDefaultToRootWhenModuleDirsMissing(t *testing.T) {
 }
 
 func TestConfigCmdWithoutController(t *testing.T) {
-	in.RunWithoutController(t, "configs-ftl-project.toml",
+	in.Run(t,
+		in.WithFTLConfig("configs-ftl-project.toml"),
+		in.WithoutController(),
 		in.ExecWithExpectedOutput("\"value\"\n", "ftl", "config", "get", "key"),
 	)
 }
@@ -49,7 +52,8 @@ func TestFindConfig(t *testing.T) {
 			assert.Equal(t, "test = \"test\"\n", string(output))
 		}
 	}
-	in.RunWithoutController(t, "",
+	in.Run(t,
+		in.WithoutController(),
 		in.CopyModule("findconfig"),
 		checkConfig("findconfig"),
 		checkConfig("findconfig/subdir"),
@@ -61,7 +65,8 @@ func TestFindConfig(t *testing.T) {
 }
 
 func TestConfigValidation(t *testing.T) {
-	in.Run(t, "./validateconfig/ftl-project.toml",
+	in.Run(t,
+		in.WithFTLConfig("./validateconfig/ftl-project.toml"),
 		in.CopyModule("validateconfig"),
 
 		// Global sets never error.

diff --git a/go-runtime/compile/compile_integration_test.go b/go-runtime/compile/compile_integration_test.go
@@ -12,7 +12,7 @@ import (
 )
 
 func TestNonExportedDecls(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("time"),
 		in.Deploy("time"),
 		in.CopyModule("echo"),
@@ -26,7 +26,7 @@ func TestNonExportedDecls(t *testing.T) {
 }
 
 func TestUndefinedExportedDecls(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("time"),
 		in.Deploy("time"),
 		in.CopyModule("echo"),
@@ -40,7 +40,7 @@ func TestUndefinedExportedDecls(t *testing.T) {
 }
 
 func TestNonFTLTypes(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("external"),
 		in.Deploy("external"),
 		in.Call("external", "echo", in.Obj{"message": "hello"}, func(t testing.TB, response in.Obj) {
@@ -50,7 +50,7 @@ func TestNonFTLTypes(t *testing.T) {
 }
 
 func TestNonStructRequestResponse(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("two"),
 		in.Deploy("two"),
 		in.CopyModule("one"),

diff --git a/go-runtime/encoding/encoding_integration_test.go b/go-runtime/encoding/encoding_integration_test.go
@@ -6,12 +6,13 @@ import (
 	"net/http"
 	"testing"
 
-	in "github.com/TBD54566975/ftl/integration"
 	"github.com/alecthomas/assert/v2"
+
+	in "github.com/TBD54566975/ftl/integration"
 )
 
 func TestHttpEncodeOmitempty(t *testing.T) {
-	in.Run(t, "",
+	in.Run(t,
 		in.CopyModule("omitempty"),
 		in.Deploy("omitempty"),
 		in.HttpCall(http.MethodGet, "/get", nil, in.JsonData(t, in.Obj{}), func(t testing.TB, resp *in.HTTPResponse) {