refactor: move configuration manager, routers and providers into thei…

…r own packages

@matt2e the ASM tests use the internals of the manager et al, and I couldn't
figure out an easy way to resolve that. I'll file a ticket for us to followup
on this later.

.github/workflows/ci.yml

@@ -254,6 +254,7 @@ jobs:
       - name: Extract test cases
         id: extract-tests
         run: |
+          set -euo pipefail
           # shellcheck disable=SC2046
           echo "matrix={\"test\":$(jq -c -n '$ARGS.positional' --args $(git grep -l '^//go:build integration' | xargs grep '^func Test' | awk '{print $2}' | cut -d'(' -f1))}" >> "$GITHUB_OUTPUT"
   integration-run:
@@ -281,8 +282,9 @@ jobs:
         run: go mod download
       - name: Run ${{ matrix.test }}
         run: |
+          set -euo pipefail
           # shellcheck disable=SC2046
-          go test -v -race -tags integration -run ${{ matrix.test }} $(git grep -l '^//go:build integration' | xargs grep -l '^func ${{ matrix.test }}' | xargs -I {} dirname ./{})
+          go test -v -race -tags integration -run '^${{ matrix.test }}$' $(git grep -l '^//go:build integration' | xargs grep -l '^func ${{ matrix.test }}' | xargs -I {} dirname ./{})
   integration-success:
     name: Integration Success
     needs: [integration-run]

backend/controller/admin/admin.go

@@ -11,14 +11,15 @@ import (
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/go-runtime/encoding"
-	cf "github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
 type AdminService struct {
 	schr SchemaRetriever
-	cm   *cf.Manager[cf.Configuration]
-	sm   *cf.Manager[cf.Secrets]
+	cm   *manager.Manager[configuration.Configuration]
+	sm   *manager.Manager[configuration.Secrets]
 }
 
 var _ ftlv1connect.AdminServiceHandler = (*AdminService)(nil)
@@ -27,7 +28,7 @@ type SchemaRetriever interface {
 	GetActiveSchema(ctx context.Context) (*schema.Schema, error)
 }
 
-func NewAdminService(cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets], schr SchemaRetriever) *AdminService {
+func NewAdminService(cm *manager.Manager[configuration.Configuration], sm *manager.Manager[configuration.Secrets], schr SchemaRetriever) *AdminService {
 	return &AdminService{
 		schr: schr,
 		cm:   cm,
@@ -141,7 +142,7 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl
 	}
 	secrets := []*ftlv1.ListSecretsResponse_Secret{}
 	for _, secret := range listing {
-		if req.Msg.Provider != nil && cf.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) {
+		if req.Msg.Provider != nil && manager.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) {
 			// Skip secrets that don't match the provider in the request
 			continue
 		}
@@ -231,11 +232,11 @@ func (s *AdminService) SecretUnset(ctx context.Context, req *connect.Request[ftl
 	return connect.NewResponse(&ftlv1.UnsetSecretResponse{}), nil
 }
 
-func refFromConfigRef(cr *ftlv1.ConfigRef) cf.Ref {
-	return cf.NewRef(cr.GetModule(), cr.GetName())
+func refFromConfigRef(cr *ftlv1.ConfigRef) configuration.Ref {
+	return configuration.NewRef(cr.GetModule(), cr.GetName())
 }
 
-func (s *AdminService) validateAgainstSchema(ctx context.Context, isSecret bool, ref cf.Ref, value json.RawMessage) error {
+func (s *AdminService) validateAgainstSchema(ctx context.Context, isSecret bool, ref configuration.Ref, value json.RawMessage) error {
 	logger := log.FromContext(ctx)
 
 	// Globals aren't in the module schemas, so we have nothing to validate against.

backend/controller/admin/admin_test.go

@@ -14,22 +14,25 @@ import (
 
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 	"github.com/TBD54566975/ftl/backend/schema"
-	cf "github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
+	"github.com/TBD54566975/ftl/internal/configuration/providers"
+	"github.com/TBD54566975/ftl/internal/configuration/routers"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
 func TestAdminService(t *testing.T) {
 	config := tempConfigPath(t, "testdata/ftl-project.toml", "admin")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	cm, err := cf.NewConfigurationManager(ctx, cf.ProjectConfigResolver[cf.Configuration]{Config: config})
+	cm, err := manager.NewConfigurationManager(ctx, routers.ProjectConfig[configuration.Configuration]{Config: config})
 	assert.NoError(t, err)
 
-	sm, err := cf.New(ctx,
-		cf.ProjectConfigResolver[cf.Secrets]{Config: config},
-		[]cf.Provider[cf.Secrets]{
-			cf.EnvarProvider[cf.Secrets]{},
-			cf.InlineProvider[cf.Secrets]{},
+	sm, err := manager.New(ctx,
+		routers.ProjectConfig[configuration.Secrets]{Config: config},
+		[]configuration.Provider[configuration.Secrets]{
+			providers.Envar[configuration.Secrets]{},
+			providers.Inline[configuration.Secrets]{},
 		})
 	assert.NoError(t, err)
 	admin := NewAdminService(cm, sm, &diskSchemaRetriever{})
@@ -39,20 +42,20 @@ func TestAdminService(t *testing.T) {
 	assert.NoError(t, err)
 
 	testAdminConfigs(t, ctx, "FTL_CONFIG_YmFy", admin, []expectedEntry{
-		{Ref: cf.Ref{Name: "bar"}, Value: string(expectedEnvarValue)},
-		{Ref: cf.Ref{Name: "foo"}, Value: `"foobar"`},
-		{Ref: cf.Ref{Name: "mutable"}, Value: `"helloworld"`},
-		{Ref: cf.Ref{Module: optional.Some[string]("echo"), Name: "default"}, Value: `"anonymous"`},
+		{Ref: configuration.Ref{Name: "bar"}, Value: string(expectedEnvarValue)},
+		{Ref: configuration.Ref{Name: "foo"}, Value: `"foobar"`},
+		{Ref: configuration.Ref{Name: "mutable"}, Value: `"helloworld"`},
+		{Ref: configuration.Ref{Module: optional.Some[string]("echo"), Name: "default"}, Value: `"anonymous"`},
 	})
 
 	testAdminSecrets(t, ctx, "FTL_SECRET_YmFy", admin, []expectedEntry{
-		{Ref: cf.Ref{Name: "bar"}, Value: string(expectedEnvarValue)},
-		{Ref: cf.Ref{Name: "foo"}, Value: `"foobarsecret"`},
+		{Ref: configuration.Ref{Name: "bar"}, Value: string(expectedEnvarValue)},
+		{Ref: configuration.Ref{Name: "foo"}, Value: `"foobarsecret"`},
 	})
 }
 
 type expectedEntry struct {
-	Ref   cf.Ref
+	Ref   configuration.Ref
 	Value string
 }
 
@@ -204,14 +207,14 @@ func TestAdminValidation(t *testing.T) {
 	config := tempConfigPath(t, "testdata/ftl-project.toml", "admin")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	cm, err := cf.NewConfigurationManager(ctx, cf.ProjectConfigResolver[cf.Configuration]{Config: config})
+	cm, err := manager.NewConfigurationManager(ctx, routers.ProjectConfig[configuration.Configuration]{Config: config})
 	assert.NoError(t, err)
 
-	sm, err := cf.New(ctx,
-		cf.ProjectConfigResolver[cf.Secrets]{Config: config},
-		[]cf.Provider[cf.Secrets]{
-			cf.EnvarProvider[cf.Secrets]{},
-			cf.InlineProvider[cf.Secrets]{},
+	sm, err := manager.New(ctx,
+		routers.ProjectConfig[configuration.Secrets]{Config: config},
+		[]configuration.Provider[configuration.Secrets]{
+			providers.Envar[configuration.Secrets]{},
+			providers.Inline[configuration.Secrets]{},
 		})
 	assert.NoError(t, err)
 	admin := NewAdminService(cm, sm, &mockSchemaRetriever{})

backend/controller/admin/local_client.go

@@ -10,6 +10,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/internal/buildengine"
 	cf "github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
 	"github.com/TBD54566975/ftl/internal/projectconfig"
 )
 
@@ -26,7 +27,7 @@ type diskSchemaRetriever struct {
 }
 
 // NewLocalClient creates a admin client that reads and writes from the provided config and secret managers
-func NewLocalClient(cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets]) Client {
+func NewLocalClient(cm *manager.Manager[cf.Configuration], sm *manager.Manager[cf.Secrets]) Client {
 	return &localClient{NewAdminService(cm, sm, &diskSchemaRetriever{})}
 }
 

backend/controller/admin/local_client_test.go

@@ -10,6 +10,9 @@ import (
 	"github.com/alecthomas/types/optional"
 
 	cf "github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
+	"github.com/TBD54566975/ftl/internal/configuration/providers"
+	"github.com/TBD54566975/ftl/internal/configuration/routers"
 	in "github.com/TBD54566975/ftl/internal/integration"
 	"github.com/TBD54566975/ftl/internal/log"
 )
@@ -49,14 +52,14 @@ func TestAdminNoValidationWithNoSchema(t *testing.T) {
 	config := tempConfigPath(t, "testdata/ftl-project.toml", "admin")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	cm, err := cf.NewConfigurationManager(ctx, cf.ProjectConfigResolver[cf.Configuration]{Config: config})
+	cm, err := manager.NewConfigurationManager(ctx, routers.ProjectConfig[cf.Configuration]{Config: config})
 	assert.NoError(t, err)
 
-	sm, err := cf.New(ctx,
-		cf.ProjectConfigResolver[cf.Secrets]{Config: config},
+	sm, err := manager.New(ctx,
+		routers.ProjectConfig[cf.Secrets]{Config: config},
 		[]cf.Provider[cf.Secrets]{
-			cf.EnvarProvider[cf.Secrets]{},
-			cf.InlineProvider[cf.Secrets]{},
+			providers.Envar[cf.Secrets]{},
+			providers.Inline[cf.Secrets]{},
 		})
 	assert.NoError(t, err)
 

backend/controller/controller.go

@@ -54,7 +54,7 @@ import (
 	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/frontend"
-	cf "github.com/TBD54566975/ftl/internal/configuration"
+	cf "github.com/TBD54566975/ftl/internal/configuration/manager"
 	"github.com/TBD54566975/ftl/internal/cors"
 	"github.com/TBD54566975/ftl/internal/encryption"
 	ftlhttp "github.com/TBD54566975/ftl/internal/http"

cmd/ftl-controller/main.go

@@ -18,6 +18,9 @@ import (
 	_ "github.com/TBD54566975/ftl/internal/automaxprocs" // Set GOMAXPROCS to match Linux container CPU quota.
 	cf "github.com/TBD54566975/ftl/internal/configuration"
 	cfdal "github.com/TBD54566975/ftl/internal/configuration/dal"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
+	"github.com/TBD54566975/ftl/internal/configuration/providers"
+	"github.com/TBD54566975/ftl/internal/configuration/routers"
 	"github.com/TBD54566975/ftl/internal/log"
 	"github.com/TBD54566975/ftl/internal/observability"
 )
@@ -27,7 +30,7 @@ var cli struct {
 	ObservabilityConfig observability.Config `embed:"" prefix:"o11y-"`
 	LogConfig           log.Config           `embed:"" prefix:"log-"`
 	ControllerConfig    controller.Config    `embed:""`
-	ConfigFlag          string               `name:"config" short:"C" help:"Path to FTL project configuration file." env:"FTL_CONFIG" placeholder:"FILE"`
+	ConfigFlag          string               `name:"config" short:"C" help:"Path to FTL project cf file." env:"FTL_CONFIG" placeholder:"FILE"`
 }
 
 func main() {
@@ -50,7 +53,7 @@ func main() {
 	err = observability.Init(ctx, false, "", "ftl-controller", ftl.Version, cli.ObservabilityConfig)
 	kctx.FatalIfErrorf(err, "failed to initialize observability")
 
-	// The FTL controller currently only supports DB as a configuration provider/resolver.
+	// The FTL controller currently only supports DB as a cf provider/resolver.
 	conn, err := observability.OpenDBAndInstrument(cli.ControllerConfig.DSN)
 	kctx.FatalIfErrorf(err)
 
@@ -59,21 +62,21 @@ func main() {
 
 	configDal := cfdal.New(conn)
 	kctx.FatalIfErrorf(err)
-	configProviders := []cf.Provider[cf.Configuration]{cf.NewDBConfigProvider(configDal)}
-	configResolver := cf.NewDBConfigResolver(configDal)
-	cm, err := cf.New(ctx, configResolver, configProviders)
+	configProviders := []cf.Provider[cf.Configuration]{providers.NewDatabaseConfig(configDal)}
+	configResolver := routers.NewDatabaseConfig(configDal)
+	cm, err := manager.New(ctx, configResolver, configProviders)
 	kctx.FatalIfErrorf(err)
 
-	ctx = cf.ContextWithConfig(ctx, cm)
+	ctx = manager.ContextWithConfig(ctx, cm)
 
 	// The FTL controller currently only supports AWS Secrets Manager as a secrets provider.
 	awsConfig, err := config.LoadDefaultConfig(ctx)
 	kctx.FatalIfErrorf(err)
-	asmSecretProvider := cf.NewASM(ctx, secretsmanager.NewFromConfig(awsConfig), cli.ControllerConfig.Advertise, dal)
-	dbSecretResolver := cf.NewDBSecretResolver(configDal)
-	sm, err := cf.New[cf.Secrets](ctx, dbSecretResolver, []cf.Provider[cf.Secrets]{asmSecretProvider})
+	asmSecretProvider := providers.NewASM(ctx, secretsmanager.NewFromConfig(awsConfig), cli.ControllerConfig.Advertise, dal)
+	dbSecretResolver := routers.NewDatabaseSecrets(configDal)
+	sm, err := manager.New[cf.Secrets](ctx, dbSecretResolver, []cf.Provider[cf.Secrets]{asmSecretProvider})
 	kctx.FatalIfErrorf(err)
-	ctx = cf.ContextWithSecrets(ctx, sm)
+	ctx = manager.ContextWithSecrets(ctx, sm)
 
 	err = controller.Start(ctx, cli.ControllerConfig, scaling.NewK8sScaling(), conn)
 	kctx.FatalIfErrorf(err)

cmd/ftl/cmd_config.go

@@ -13,7 +13,9 @@ import (
 	"github.com/TBD54566975/ftl/backend/controller/admin"
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
-	cf "github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
+	"github.com/TBD54566975/ftl/internal/configuration/routers"
 	"github.com/TBD54566975/ftl/internal/log"
 	"github.com/TBD54566975/ftl/internal/projectconfig"
 	"github.com/TBD54566975/ftl/internal/rpc"
@@ -39,7 +41,7 @@ etc.
 `
 }
 
-func configRefFromRef(ref cf.Ref) *ftlv1.ConfigRef {
+func configRefFromRef(ref configuration.Ref) *ftlv1.ConfigRef {
 	module := ref.Module.Default("")
 	return &ftlv1.ConfigRef{
 		Module: &module,
@@ -71,19 +73,19 @@ func setUpAdminClient(ctx context.Context, config projectconfig.Config) (ctxOut
 	}
 	if shouldUseLocalClient {
 		// create config and secret managers
-		cr := cf.ProjectConfigResolver[cf.Configuration]{Config: config.Path}
-		cm, err := cf.NewConfigurationManager(ctx, cr)
+		cr := routers.ProjectConfig[configuration.Configuration]{Config: config.Path}
+		cm, err := manager.NewConfigurationManager(ctx, cr)
 		if err != nil {
 			return ctx, client, fmt.Errorf("could not create config manager: %w", err)
 		}
-		ctx = cf.ContextWithConfig(ctx, cm)
+		ctx = manager.ContextWithConfig(ctx, cm)
 
-		sr := cf.ProjectConfigResolver[cf.Secrets]{Config: config.Path}
-		sm, err := cf.NewSecretsManager(ctx, sr, cli.Vault, config.Path)
+		sr := routers.ProjectConfig[configuration.Secrets]{Config: config.Path}
+		sm, err := manager.NewSecretsManager(ctx, sr, cli.Vault, config.Path)
 		if err != nil {
 			return ctx, client, fmt.Errorf("could not create secrets manager: %w", err)
 		}
-		ctx = cf.ContextWithSecrets(ctx, sm)
+		ctx = manager.ContextWithSecrets(ctx, sm)
 
 		return ctx, admin.NewLocalClient(cm, sm), nil
 	}
@@ -115,7 +117,7 @@ func (s *configListCmd) Run(ctx context.Context, projConfig projectconfig.Config
 }
 
 type configGetCmd struct {
-	Ref cf.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 }
 
 func (s *configGetCmd) Help() string {
@@ -140,9 +142,9 @@ func (s *configGetCmd) Run(ctx context.Context, projConfig projectconfig.Config)
 }
 
 type configSetCmd struct {
-	JSON  bool    `help:"Assume input value is JSON."`
-	Ref   cf.Ref  `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
-	Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
+	JSON  bool              `help:"Assume input value is JSON."`
+	Ref   configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Value *string           `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
 }
 
 func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, projConfig projectconfig.Config) error {
@@ -189,7 +191,7 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, projConfig proj
 }
 
 type configUnsetCmd struct {
-	Ref cf.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 }
 
 func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, projConfig projectconfig.Config) error {
@@ -235,7 +237,7 @@ func (s *configImportCmd) Run(ctx context.Context, cmd *configCmd, projConfig pr
 		return fmt.Errorf("could not parse JSON: %w", err)
 	}
 	for refPath, value := range entries {
-		ref, err := cf.ParseRef(refPath)
+		ref, err := configuration.ParseRef(refPath)
 		if err != nil {
 			return fmt.Errorf("could not parse ref %q: %w", refPath, err)
 		}

cmd/ftl/cmd_serve.go

@@ -25,7 +25,9 @@ import (
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/internal/bind"
-	cf "github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration"
+	"github.com/TBD54566975/ftl/internal/configuration/manager"
+	"github.com/TBD54566975/ftl/internal/configuration/routers"
 	"github.com/TBD54566975/ftl/internal/container"
 	"github.com/TBD54566975/ftl/internal/exec"
 	"github.com/TBD54566975/ftl/internal/log"
@@ -131,20 +133,20 @@ func (s *serveCmd) run(ctx context.Context, projConfig projectconfig.Config, ini
 		controllerCtx := log.ContextWithLogger(ctx, logger.Scope(scope))
 
 		// create config manager for controller
-		cr := cf.ProjectConfigResolver[cf.Configuration]{Config: projConfig.Path}
-		cm, err := cf.NewConfigurationManager(controllerCtx, cr)
+		cr := routers.ProjectConfig[configuration.Configuration]{Config: projConfig.Path}
+		cm, err := manager.NewConfigurationManager(controllerCtx, cr)
 		if err != nil {
 			return fmt.Errorf("could not create config manager: %w", err)
 		}
-		controllerCtx = cf.ContextWithConfig(controllerCtx, cm)
+		controllerCtx = manager.ContextWithConfig(controllerCtx, cm)
 
 		// create secrets manager for controller
-		sr := cf.ProjectConfigResolver[cf.Secrets]{Config: projConfig.Path}
-		sm, err := cf.NewSecretsManager(controllerCtx, sr, cli.Vault, projConfig.Path)
+		sr := routers.ProjectConfig[configuration.Secrets]{Config: projConfig.Path}
+		sm, err := manager.NewSecretsManager(controllerCtx, sr, cli.Vault, projConfig.Path)
 		if err != nil {
 			return fmt.Errorf("could not create secrets manager: %w", err)
 		}
-		controllerCtx = cf.ContextWithSecrets(controllerCtx, sm)
+		controllerCtx = manager.ContextWithSecrets(controllerCtx, sm)
 
 		// Bring up the DB connection and DAL.
 		conn, err := observability.OpenDBAndInstrument(config.DSN)