feat: support configuration overrides

This necessitated splitting the construction of the config resolver from
the providers so that the configuration files can be specified at the
global level.

backend/controller/scaling/localscaling/devel.go

@@ -17,7 +17,7 @@ var templateDirOnce sync.Once
 func templateDir(ctx context.Context) string {
 	templateDirOnce.Do(func() {
 		// TODO: Figure out how to make maven build offline
-		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "--level=trace", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx)
+		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx)
 		if err != nil {
 			panic(err)
 		}

backend/runner/runner.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -27,7 +28,6 @@ import (
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/plugin"
-	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/download"
 	"github.com/TBD54566975/ftl/internal/log"
 	"github.com/TBD54566975/ftl/internal/model"
@@ -37,6 +37,7 @@ import (
 )
 
 type Config struct {
+	Config             []string        `name:"config" short:"C" help:"Paths to FTL project configuration files." env:"FTL_CONFIG" placeholder:"FILE[,FILE,...]"`
 	Bind               *url.URL        `help:"Endpoint the Runner should bind to and advertise." default:"http://localhost:8893" env:"FTL_RUNNER_BIND"`
 	Advertise          *url.URL        `help:"Endpoint the Runner should advertise (use --bind if omitted)." default:"" env:"FTL_RUNNER_ADVERTISE"`
 	Key                model.RunnerKey `help:"Runner key (auto)." placeholder:"R<ULID>" default:"R00000000000000000000000000"`
@@ -219,7 +220,7 @@ func (s *Service) Deploy(ctx context.Context, req *connect.Request[ftlv1.DeployR
 		ftlv1connect.NewVerbServiceClient,
 		plugin.WithEnvars(
 			"FTL_ENDPOINT="+s.config.ControllerEndpoint.String(),
-			"FTL_CONFIG="+filepath.Join(internal.GitRoot(""), "ftl-project.toml"),
+			"FTL_CONFIG="+strings.Join(s.config.Config, ","),
 			"FTL_OBSERVABILITY_ENDPOINT="+s.config.ControllerEndpoint.String(),
 		),
 	)

cmd/ftl/cmd_config.go

@@ -7,11 +7,11 @@ import (
 	"io"
 	"os"
 
-	"github.com/TBD54566975/ftl/common/configuration"
+	cf "github.com/TBD54566975/ftl/common/configuration"
 )
 
 type configCmd struct {
-	configuration.DefaultConfigMixin
+	cf.DefaultConfigMixin
 
 	List  configListCmd  `cmd:"" help:"List configuration."`
 	Get   configGetCmd   `cmd:"" help:"Get a configuration value."`
@@ -31,8 +31,8 @@ type configListCmd struct {
 	Module string `optional:"" arg:"" placeholder:"MODULE" help:"List configuration only in this module."`
 }
 
-func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := scmd.NewConfigurationManager(ctx)
+func (s *configListCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error {
+	sm, err := scmd.NewConfigurationManager(ctx, cr)
 	if err != nil {
 		return err
 	}
@@ -68,7 +68,7 @@ func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error {
 }
 
 type configGetCmd struct {
-	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Ref cf.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 }
 
 func (s *configGetCmd) Help() string {
@@ -77,8 +77,8 @@ Returns a JSON-encoded configuration value.
 `
 }
 
-func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := scmd.NewConfigurationManager(ctx)
+func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error {
+	sm, err := scmd.NewConfigurationManager(ctx, cr)
 	if err != nil {
 		return err
 	}
@@ -98,13 +98,13 @@ func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error {
 }
 
 type configSetCmd struct {
-	JSON  bool              `help:"Assume input value is JSON."`
-	Ref   configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
-	Value *string           `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
+	JSON  bool    `help:"Assume input value is JSON."`
+	Ref   cf.Ref  `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
 }
 
-func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := scmd.NewConfigurationManager(ctx)
+func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error {
+	sm, err := scmd.NewConfigurationManager(ctx, cr)
 	if err != nil {
 		return err
 	}
@@ -135,11 +135,11 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error {
 }
 
 type configUnsetCmd struct {
-	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Ref cf.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 }
 
-func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := scmd.NewConfigurationManager(ctx)
+func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error {
+	sm, err := scmd.NewConfigurationManager(ctx, cr)
 	if err != nil {
 		return err
 	}

cmd/ftl/cmd_secret.go

@@ -10,11 +10,11 @@ import (
 	"github.com/mattn/go-isatty"
 	"golang.org/x/term"
 
-	"github.com/TBD54566975/ftl/common/configuration"
+	cf "github.com/TBD54566975/ftl/common/configuration"
 )
 
 type secretCmd struct {
-	configuration.DefaultSecretsMixin
+	cf.DefaultSecretsMixin
 
 	List  secretListCmd  `cmd:"" help:"List secrets."`
 	Get   secretGetCmd   `cmd:"" help:"Get a secret."`
@@ -37,8 +37,8 @@ type secretListCmd struct {
 	Module string `optional:"" arg:"" placeholder:"MODULE" help:"List secrets only in this module."`
 }
 
-func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := scmd.NewSecretsManager(ctx)
+func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error {
+	sm, err := scmd.NewSecretsManager(ctx, sr)
 	if err != nil {
 		return err
 	}
@@ -74,7 +74,7 @@ func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error {
 }
 
 type secretGetCmd struct {
-	Ref configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
+	Ref cf.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
 func (s *secretGetCmd) Help() string {
@@ -83,8 +83,8 @@ Returns a JSON-encoded secret value.
 `
 }
 
-func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := scmd.NewSecretsManager(ctx)
+func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error {
+	sm, err := scmd.NewSecretsManager(ctx, sr)
 	if err != nil {
 		return err
 	}
@@ -104,12 +104,12 @@ func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error {
 }
 
 type secretSetCmd struct {
-	JSON bool              `help:"Assume input value is JSON."`
-	Ref  configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
+	JSON bool   `help:"Assume input value is JSON."`
+	Ref  cf.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
-func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := scmd.NewSecretsManager(ctx)
+func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error {
+	sm, err := scmd.NewSecretsManager(ctx, sr)
 	if err != nil {
 		return err
 	}
@@ -146,11 +146,11 @@ func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error {
 }
 
 type secretUnsetCmd struct {
-	Ref configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
+	Ref cf.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
-func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := scmd.NewSecretsManager(ctx)
+func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error {
+	sm, err := scmd.NewSecretsManager(ctx, sr)
 	if err != nil {
 		return err
 	}

cmd/ftl/main.go

@@ -6,22 +6,25 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
+	"strings"
 	"syscall"
 
 	"github.com/alecthomas/kong"
 	kongtoml "github.com/alecthomas/kong-toml"
 
 	"github.com/TBD54566975/ftl"
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
+	cf "github.com/TBD54566975/ftl/common/configuration"
 	_ "github.com/TBD54566975/ftl/internal/automaxprocs" // Set GOMAXPROCS to match Linux container CPU quota.
 	"github.com/TBD54566975/ftl/internal/log"
 	"github.com/TBD54566975/ftl/internal/rpc"
 )
 
 type CLI struct {
-	Version   kong.VersionFlag `help:"Show version."`
-	LogConfig log.Config       `embed:"" prefix:"log-" group:"Logging:"`
-	Endpoint  *url.URL         `default:"http://127.0.0.1:8892" help:"FTL endpoint to bind/connect to." env:"FTL_ENDPOINT"`
+	Version    kong.VersionFlag `help:"Show version."`
+	LogConfig  log.Config       `embed:"" prefix:"log-" group:"Logging:"`
+	Endpoint   *url.URL         `default:"http://127.0.0.1:8892" help:"FTL endpoint to bind/connect to." env:"FTL_ENDPOINT"`
+	ConfigFlag []string         `name:"config" short:"C" help:"Paths to FTL project configuration files." env:"FTL_CONFIG" placeholder:"FILE[,FILE,...]" type:"existingfile"`
 
 	Authenticators map[string]string `help:"Authenticators to use for FTL endpoints." mapsep:"," env:"FTL_AUTHENTICATORS" placeholder:"HOST=EXE,…"`
 
@@ -65,14 +68,22 @@ func main() {
 
 	rpc.InitialiseClients(cli.Authenticators)
 
-	// Set the log level for child processes.
+	// Set some envars for child processes.
 	os.Setenv("LOG_LEVEL", cli.LogConfig.Level.String())
+	if len(cli.ConfigFlag) > 0 {
+		os.Setenv("FTL_CONFIG", strings.Join(cli.ConfigFlag, ","))
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 
 	logger := log.Configure(os.Stderr, cli.LogConfig)
 	ctx = log.ContextWithLogger(ctx, logger)
 
+	sr := cf.ProjectConfigResolver[cf.Secrets]{Config: cli.ConfigFlag}
+	cr := cf.ProjectConfigResolver[cf.Configuration]{Config: cli.ConfigFlag}
+	kctx.BindTo(sr, (*cf.Resolver[cf.Secrets])(nil))
+	kctx.BindTo(cr, (*cf.Resolver[cf.Configuration])(nil))
+
 	// Handle signals.
 	sigch := make(chan os.Signal, 1)
 	signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)

common/configuration/1password.go → common/configuration/1password_provider.go

@@ -21,7 +21,8 @@ type OnePasswordProvider struct {
 
 var _ MutableProvider[Secrets] = OnePasswordProvider{}
 
-func (o OnePasswordProvider) Key() Secrets                              { return "op" }
+func (OnePasswordProvider) Role() Secrets                               { return Secrets{} }
+func (o OnePasswordProvider) Key() string                               { return "op" }
 func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil }
 
 func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {

common/configuration/api.go

@@ -26,6 +26,7 @@ import (
 // ErrNotFound is returned when a configuration entry is not found or cannot be resolved.
 var ErrNotFound = errors.New("not found")
 
+// Entry in the configuration store.
 type Entry struct {
 	Ref
 	Accessor *url.URL
@@ -75,7 +76,8 @@ func (k *Ref) UnmarshalText(text []byte) error {
 // abstracted from the configuration itself. For example, the ftl-project.toml
 // file contains per-module and global configuration maps, but the secrets
 // themselves may be stored in a separate secret store such as a system keychain.
-type Resolver interface {
+type Resolver[R Role] interface {
+	Role() R
 	Get(ctx context.Context, ref Ref) (key *url.URL, err error)
 	Set(ctx context.Context, ref Ref, key *url.URL) error
 	Unset(ctx context.Context, ref Ref) error
@@ -84,7 +86,8 @@ type Resolver interface {
 
 // Provider is a generic interface for storing and retrieving configuration and secrets.
 type Provider[R Role] interface {
-	Key() R
+	Role() R
+	Key() string
 	Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error)
 }
 

common/configuration/defaults.go

@@ -7,54 +7,44 @@ import (
 )
 
 // NewConfigurationManager constructs a new [Manager] with the default providers for configuration.
-func NewConfigurationManager(ctx context.Context, configPath string) (*Manager[Configuration], error) {
-	conf := DefaultConfigMixin{
-		ProjectConfigResolver: ProjectConfigResolver[Configuration]{
-			Config: configPath,
-		},
-	}
+func NewConfigurationManager(ctx context.Context, resolver Resolver[Configuration]) (*Manager[Configuration], error) {
+	conf := DefaultConfigMixin{}
 	_ = kong.ApplyDefaults(&conf)
-	return conf.NewConfigurationManager(ctx)
+	return conf.NewConfigurationManager(ctx, resolver)
 }
 
 // DefaultConfigMixin is a Kong mixin that provides the default configuration manager.
 type DefaultConfigMixin struct {
-	ProjectConfigResolver[Configuration]
 	InlineProvider[Configuration]
 	EnvarProvider[Configuration]
 }
 
 // NewConfigurationManager creates a new configuration manager with the default configuration providers.
-func (d DefaultConfigMixin) NewConfigurationManager(ctx context.Context) (*Manager[Configuration], error) {
-	return New(ctx, &d.ProjectConfigResolver, []Provider[Configuration]{
+func (d DefaultConfigMixin) NewConfigurationManager(ctx context.Context, resolver Resolver[Configuration]) (*Manager[Configuration], error) {
+	return New(ctx, resolver, []Provider[Configuration]{
 		d.InlineProvider,
 		d.EnvarProvider,
 	})
 }
 
 // NewSecretsManager constructs a new [Manager] with the default providers for secrets.
-func NewSecretsManager(ctx context.Context, configPath string) (*Manager[Secrets], error) {
-	conf := DefaultSecretsMixin{
-		ProjectConfigResolver: ProjectConfigResolver[Secrets]{
-			Config: configPath,
-		},
-	}
+func NewSecretsManager(ctx context.Context, resolver Resolver[Secrets]) (*Manager[Secrets], error) {
+	conf := DefaultSecretsMixin{}
 	_ = kong.ApplyDefaults(&conf)
-	return conf.NewSecretsManager(ctx)
+	return conf.NewSecretsManager(ctx, resolver)
 }
 
 // DefaultSecretsMixin is a Kong mixin that provides the default secrets manager.
 type DefaultSecretsMixin struct {
-	ProjectConfigResolver[Secrets]
 	InlineProvider[Secrets]
 	EnvarProvider[Secrets]
 	KeychainProvider
 	OnePasswordProvider
 }
 
 // NewSecretsManager creates a new secrets manager with the default secret providers.
-func (d DefaultSecretsMixin) NewSecretsManager(ctx context.Context) (*Manager[Secrets], error) {
-	return New(ctx, &d.ProjectConfigResolver, []Provider[Secrets]{
+func (d DefaultSecretsMixin) NewSecretsManager(ctx context.Context, resolver Resolver[Secrets]) (*Manager[Secrets], error) {
+	return New(ctx, resolver, []Provider[Secrets]{
 		d.InlineProvider,
 		d.EnvarProvider,
 		d.KeychainProvider,

common/configuration/envar.go → common/configuration/envar_provider.go

@@ -16,7 +16,8 @@ type EnvarProvider[R Role] struct {
 
 var _ MutableProvider[Configuration] = EnvarProvider[Configuration]{}
 
-func (EnvarProvider[R]) Key() R { return "envar" }
+func (EnvarProvider[R]) Role() R     { var r R; return r }
+func (EnvarProvider[R]) Key() string { return "envar" }
 
 func (e EnvarProvider[R]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {
 	// FTL_<type>_[<module>]_<name> where <module> and <name> are base64 encoded.

common/configuration/inline.go → common/configuration/inline_provider.go

@@ -14,7 +14,8 @@ type InlineProvider[R Role] struct {
 
 var _ MutableProvider[Configuration] = InlineProvider[Configuration]{}
 
-func (InlineProvider[R]) Key() R { return "inline" }
+func (InlineProvider[R]) Role() R     { var r R; return r }
+func (InlineProvider[R]) Key() string { return "inline" }
 
 func (i InlineProvider[R]) Writer() bool { return i.Inline }
 

common/configuration/keychain.go → common/configuration/keychain_provider.go

@@ -16,7 +16,8 @@ type KeychainProvider struct {
 
 var _ MutableProvider[Secrets] = KeychainProvider{}
 
-func (k KeychainProvider) Key() Secrets { return "keychain" }
+func (KeychainProvider) Role() Secrets { return Secrets{} }
+func (k KeychainProvider) Key() string { return "keychain" }
 
 func (k KeychainProvider) Writer() bool { return k.Keychain }