From 2facf948480fd41e233f623cb43b02798f07c90a Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Fri, 8 Mar 2024 11:04:03 +1100 Subject: [PATCH] feat: support configuration overrides This necessitated splitting the construction of the config resolver from the providers so that the configuration files can be specified at the global level. --- .../controller/scaling/localscaling/devel.go | 2 +- backend/runner/runner.go | 5 +- cmd/ftl/cmd_config.go | 30 +++---- cmd/ftl/cmd_secret.go | 28 +++---- cmd/ftl/main.go | 19 ++++- .../{1password.go => 1password_provider.go} | 3 +- common/configuration/api.go | 7 +- common/configuration/defaults.go | 30 +++---- .../{envar.go => envar_provider.go} | 3 +- .../{inline.go => inline_provider.go} | 3 +- .../{keychain.go => keychain_provider.go} | 3 +- common/configuration/manager.go | 20 +++-- common/configuration/manager_test.go | 4 +- ...ectconfig.go => projectconfig_resolver.go} | 34 ++++---- common/projectconfig/merge.go | 43 ++++++++++ common/projectconfig/merge_test.go | 82 +++++++++++++++++++ examples/go/echo/go.mod | 2 +- examples/go/echo/go.sum | 4 +- go-runtime/ftl/config_test.go | 3 +- go-runtime/ftl/ftltest/ftltest.go | 18 ++-- go-runtime/ftl/secrets_test.go | 3 +- go-runtime/server/server.go | 12 +-- go.mod | 2 +- go.sum | 4 +- 24 files changed, 260 insertions(+), 104 deletions(-) rename common/configuration/{1password.go => 1password_provider.go} (92%) rename common/configuration/{envar.go => envar_provider.go} (96%) rename common/configuration/{inline.go => inline_provider.go} (89%) rename common/configuration/{keychain.go => keychain_provider.go} (91%) rename common/configuration/{projectconfig.go => projectconfig_resolver.go} (82%) create mode 100644 common/projectconfig/merge.go create mode 100644 common/projectconfig/merge_test.go diff --git a/backend/controller/scaling/localscaling/devel.go b/backend/controller/scaling/localscaling/devel.go index 948c4afc36..5e197cdee1 100644 --- a/backend/controller/scaling/localscaling/devel.go +++ b/backend/controller/scaling/localscaling/devel.go @@ -17,7 +17,7 @@ var templateDirOnce sync.Once func templateDir(ctx context.Context) string { templateDirOnce.Do(func() { // TODO: Figure out how to make maven build offline - err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "--level=trace", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx) + err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx) if err != nil { panic(err) } diff --git a/backend/runner/runner.go b/backend/runner/runner.go index 9bb44d4d18..9b5050b1a0 100644 --- a/backend/runner/runner.go +++ b/backend/runner/runner.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "sync" "syscall" "time" @@ -27,7 +28,6 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/common/plugin" - "github.com/TBD54566975/ftl/internal" "github.com/TBD54566975/ftl/internal/download" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" @@ -37,6 +37,7 @@ import ( ) type Config struct { + Config []string `name:"config" short:"C" help:"Paths to FTL project configuration files." env:"FTL_CONFIG" placeholder:"FILE[,FILE,...]"` Bind *url.URL `help:"Endpoint the Runner should bind to and advertise." default:"http://localhost:8893" env:"FTL_RUNNER_BIND"` Advertise *url.URL `help:"Endpoint the Runner should advertise (use --bind if omitted)." default:"" env:"FTL_RUNNER_ADVERTISE"` Key model.RunnerKey `help:"Runner key (auto)." placeholder:"R" default:"R00000000000000000000000000"` @@ -219,7 +220,7 @@ func (s *Service) Deploy(ctx context.Context, req *connect.Request[ftlv1.DeployR ftlv1connect.NewVerbServiceClient, plugin.WithEnvars( "FTL_ENDPOINT="+s.config.ControllerEndpoint.String(), - "FTL_CONFIG="+filepath.Join(internal.GitRoot(""), "ftl-project.toml"), + "FTL_CONFIG="+strings.Join(s.config.Config, ","), "FTL_OBSERVABILITY_ENDPOINT="+s.config.ControllerEndpoint.String(), ), ) diff --git a/cmd/ftl/cmd_config.go b/cmd/ftl/cmd_config.go index ecd6877c8e..84ec9965a5 100644 --- a/cmd/ftl/cmd_config.go +++ b/cmd/ftl/cmd_config.go @@ -7,11 +7,11 @@ import ( "io" "os" - "github.com/TBD54566975/ftl/common/configuration" + cf "github.com/TBD54566975/ftl/common/configuration" ) type configCmd struct { - configuration.DefaultConfigMixin + cf.DefaultConfigMixin List configListCmd `cmd:"" help:"List configuration."` Get configGetCmd `cmd:"" help:"Get a configuration value."` @@ -31,8 +31,8 @@ type configListCmd struct { Module string `optional:"" arg:"" placeholder:"MODULE" help:"List configuration only in this module."` } -func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := scmd.NewConfigurationManager(ctx) +func (s *configListCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error { + sm, err := scmd.NewConfigurationManager(ctx, cr) if err != nil { return err } @@ -68,7 +68,7 @@ func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error { } type configGetCmd struct { - Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` + Ref cf.Ref `arg:"" help:"Configuration reference in the form [.]."` } func (s *configGetCmd) Help() string { @@ -77,8 +77,8 @@ Returns a JSON-encoded configuration value. ` } -func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := scmd.NewConfigurationManager(ctx) +func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error { + sm, err := scmd.NewConfigurationManager(ctx, cr) if err != nil { return err } @@ -98,13 +98,13 @@ func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error { } type configSetCmd struct { - JSON bool `help:"Assume input value is JSON."` - Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` - Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""` + JSON bool `help:"Assume input value is JSON."` + Ref cf.Ref `arg:"" help:"Configuration reference in the form [.]."` + Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""` } -func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := scmd.NewConfigurationManager(ctx) +func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error { + sm, err := scmd.NewConfigurationManager(ctx, cr) if err != nil { return err } @@ -135,11 +135,11 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error { } type configUnsetCmd struct { - Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` + Ref cf.Ref `arg:"" help:"Configuration reference in the form [.]."` } -func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := scmd.NewConfigurationManager(ctx) +func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, cr cf.Resolver[cf.Configuration]) error { + sm, err := scmd.NewConfigurationManager(ctx, cr) if err != nil { return err } diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go index 5ae5db1929..0d8a03c83b 100644 --- a/cmd/ftl/cmd_secret.go +++ b/cmd/ftl/cmd_secret.go @@ -10,11 +10,11 @@ import ( "github.com/mattn/go-isatty" "golang.org/x/term" - "github.com/TBD54566975/ftl/common/configuration" + cf "github.com/TBD54566975/ftl/common/configuration" ) type secretCmd struct { - configuration.DefaultSecretsMixin + cf.DefaultSecretsMixin List secretListCmd `cmd:"" help:"List secrets."` Get secretGetCmd `cmd:"" help:"Get a secret."` @@ -37,8 +37,8 @@ type secretListCmd struct { Module string `optional:"" arg:"" placeholder:"MODULE" help:"List secrets only in this module."` } -func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := scmd.NewSecretsManager(ctx) +func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error { + sm, err := scmd.NewSecretsManager(ctx, sr) if err != nil { return err } @@ -74,7 +74,7 @@ func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error { } type secretGetCmd struct { - Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` + Ref cf.Ref `arg:"" help:"Secret reference in the form [.]."` } func (s *secretGetCmd) Help() string { @@ -83,8 +83,8 @@ Returns a JSON-encoded secret value. ` } -func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := scmd.NewSecretsManager(ctx) +func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error { + sm, err := scmd.NewSecretsManager(ctx, sr) if err != nil { return err } @@ -104,12 +104,12 @@ func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error { } type secretSetCmd struct { - JSON bool `help:"Assume input value is JSON."` - Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` + JSON bool `help:"Assume input value is JSON."` + Ref cf.Ref `arg:"" help:"Secret reference in the form [.]."` } -func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := scmd.NewSecretsManager(ctx) +func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error { + sm, err := scmd.NewSecretsManager(ctx, sr) if err != nil { return err } @@ -146,11 +146,11 @@ func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error { } type secretUnsetCmd struct { - Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` + Ref cf.Ref `arg:"" help:"Secret reference in the form [.]."` } -func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := scmd.NewSecretsManager(ctx) +func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, sr cf.Resolver[cf.Secrets]) error { + sm, err := scmd.NewSecretsManager(ctx, sr) if err != nil { return err } diff --git a/cmd/ftl/main.go b/cmd/ftl/main.go index 39ccf0e73a..0c23a51b23 100644 --- a/cmd/ftl/main.go +++ b/cmd/ftl/main.go @@ -6,6 +6,7 @@ import ( "os" "os/signal" "runtime" + "strings" "syscall" "github.com/alecthomas/kong" @@ -13,15 +14,17 @@ import ( "github.com/TBD54566975/ftl" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" + cf "github.com/TBD54566975/ftl/common/configuration" _ "github.com/TBD54566975/ftl/internal/automaxprocs" // Set GOMAXPROCS to match Linux container CPU quota. "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/rpc" ) type CLI struct { - Version kong.VersionFlag `help:"Show version."` - LogConfig log.Config `embed:"" prefix:"log-" group:"Logging:"` - Endpoint *url.URL `default:"http://127.0.0.1:8892" help:"FTL endpoint to bind/connect to." env:"FTL_ENDPOINT"` + Version kong.VersionFlag `help:"Show version."` + LogConfig log.Config `embed:"" prefix:"log-" group:"Logging:"` + Endpoint *url.URL `default:"http://127.0.0.1:8892" help:"FTL endpoint to bind/connect to." env:"FTL_ENDPOINT"` + ConfigFlag []string `name:"config" short:"C" help:"Paths to FTL project configuration files." env:"FTL_CONFIG" placeholder:"FILE[,FILE,...]" type:"existingfile"` Authenticators map[string]string `help:"Authenticators to use for FTL endpoints." mapsep:"," env:"FTL_AUTHENTICATORS" placeholder:"HOST=EXE,…"` @@ -65,14 +68,22 @@ func main() { rpc.InitialiseClients(cli.Authenticators) - // Set the log level for child processes. + // Set some envars for child processes. os.Setenv("LOG_LEVEL", cli.LogConfig.Level.String()) + if len(cli.ConfigFlag) > 0 { + os.Setenv("FTL_CONFIG", strings.Join(cli.ConfigFlag, ",")) + } ctx, cancel := context.WithCancel(context.Background()) logger := log.Configure(os.Stderr, cli.LogConfig) ctx = log.ContextWithLogger(ctx, logger) + sr := cf.ProjectConfigResolver[cf.Secrets]{Config: cli.ConfigFlag} + cr := cf.ProjectConfigResolver[cf.Configuration]{Config: cli.ConfigFlag} + kctx.BindTo(sr, (*cf.Resolver[cf.Secrets])(nil)) + kctx.BindTo(cr, (*cf.Resolver[cf.Configuration])(nil)) + // Handle signals. sigch := make(chan os.Signal, 1) signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM) diff --git a/common/configuration/1password.go b/common/configuration/1password_provider.go similarity index 92% rename from common/configuration/1password.go rename to common/configuration/1password_provider.go index 15a0526865..e4817a8dd4 100644 --- a/common/configuration/1password.go +++ b/common/configuration/1password_provider.go @@ -21,7 +21,8 @@ type OnePasswordProvider struct { var _ MutableProvider[Secrets] = OnePasswordProvider{} -func (o OnePasswordProvider) Key() Secrets { return "op" } +func (OnePasswordProvider) Role() Secrets { return Secrets{} } +func (o OnePasswordProvider) Key() string { return "op" } func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { diff --git a/common/configuration/api.go b/common/configuration/api.go index d43a04d43e..e0717547d4 100644 --- a/common/configuration/api.go +++ b/common/configuration/api.go @@ -26,6 +26,7 @@ import ( // ErrNotFound is returned when a configuration entry is not found or cannot be resolved. var ErrNotFound = errors.New("not found") +// Entry in the configuration store. type Entry struct { Ref Accessor *url.URL @@ -75,7 +76,8 @@ func (k *Ref) UnmarshalText(text []byte) error { // abstracted from the configuration itself. For example, the ftl-project.toml // file contains per-module and global configuration maps, but the secrets // themselves may be stored in a separate secret store such as a system keychain. -type Resolver interface { +type Resolver[R Role] interface { + Role() R Get(ctx context.Context, ref Ref) (key *url.URL, err error) Set(ctx context.Context, ref Ref, key *url.URL) error Unset(ctx context.Context, ref Ref) error @@ -84,7 +86,8 @@ type Resolver interface { // Provider is a generic interface for storing and retrieving configuration and secrets. type Provider[R Role] interface { - Key() R + Role() R + Key() string Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) } diff --git a/common/configuration/defaults.go b/common/configuration/defaults.go index e2454481b2..990ace3d4e 100644 --- a/common/configuration/defaults.go +++ b/common/configuration/defaults.go @@ -7,45 +7,35 @@ import ( ) // NewConfigurationManager constructs a new [Manager] with the default providers for configuration. -func NewConfigurationManager(ctx context.Context, configPath string) (*Manager[Configuration], error) { - conf := DefaultConfigMixin{ - ProjectConfigResolver: ProjectConfigResolver[Configuration]{ - Config: configPath, - }, - } +func NewConfigurationManager(ctx context.Context, resolver Resolver[Configuration]) (*Manager[Configuration], error) { + conf := DefaultConfigMixin{} _ = kong.ApplyDefaults(&conf) - return conf.NewConfigurationManager(ctx) + return conf.NewConfigurationManager(ctx, resolver) } // DefaultConfigMixin is a Kong mixin that provides the default configuration manager. type DefaultConfigMixin struct { - ProjectConfigResolver[Configuration] InlineProvider[Configuration] EnvarProvider[Configuration] } // NewConfigurationManager creates a new configuration manager with the default configuration providers. -func (d DefaultConfigMixin) NewConfigurationManager(ctx context.Context) (*Manager[Configuration], error) { - return New(ctx, &d.ProjectConfigResolver, []Provider[Configuration]{ +func (d DefaultConfigMixin) NewConfigurationManager(ctx context.Context, resolver Resolver[Configuration]) (*Manager[Configuration], error) { + return New(ctx, resolver, []Provider[Configuration]{ d.InlineProvider, d.EnvarProvider, }) } // NewSecretsManager constructs a new [Manager] with the default providers for secrets. -func NewSecretsManager(ctx context.Context, configPath string) (*Manager[Secrets], error) { - conf := DefaultSecretsMixin{ - ProjectConfigResolver: ProjectConfigResolver[Secrets]{ - Config: configPath, - }, - } +func NewSecretsManager(ctx context.Context, resolver Resolver[Secrets]) (*Manager[Secrets], error) { + conf := DefaultSecretsMixin{} _ = kong.ApplyDefaults(&conf) - return conf.NewSecretsManager(ctx) + return conf.NewSecretsManager(ctx, resolver) } // DefaultSecretsMixin is a Kong mixin that provides the default secrets manager. type DefaultSecretsMixin struct { - ProjectConfigResolver[Secrets] InlineProvider[Secrets] EnvarProvider[Secrets] KeychainProvider @@ -53,8 +43,8 @@ type DefaultSecretsMixin struct { } // NewSecretsManager creates a new secrets manager with the default secret providers. -func (d DefaultSecretsMixin) NewSecretsManager(ctx context.Context) (*Manager[Secrets], error) { - return New(ctx, &d.ProjectConfigResolver, []Provider[Secrets]{ +func (d DefaultSecretsMixin) NewSecretsManager(ctx context.Context, resolver Resolver[Secrets]) (*Manager[Secrets], error) { + return New(ctx, resolver, []Provider[Secrets]{ d.InlineProvider, d.EnvarProvider, d.KeychainProvider, diff --git a/common/configuration/envar.go b/common/configuration/envar_provider.go similarity index 96% rename from common/configuration/envar.go rename to common/configuration/envar_provider.go index f360199507..94f83c7ac6 100644 --- a/common/configuration/envar.go +++ b/common/configuration/envar_provider.go @@ -16,7 +16,8 @@ type EnvarProvider[R Role] struct { var _ MutableProvider[Configuration] = EnvarProvider[Configuration]{} -func (EnvarProvider[R]) Key() R { return "envar" } +func (EnvarProvider[R]) Role() R { var r R; return r } +func (EnvarProvider[R]) Key() string { return "envar" } func (e EnvarProvider[R]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { // FTL__[]_ where and are base64 encoded. diff --git a/common/configuration/inline.go b/common/configuration/inline_provider.go similarity index 89% rename from common/configuration/inline.go rename to common/configuration/inline_provider.go index 9e4478ff2f..fec78639f0 100644 --- a/common/configuration/inline.go +++ b/common/configuration/inline_provider.go @@ -14,7 +14,8 @@ type InlineProvider[R Role] struct { var _ MutableProvider[Configuration] = InlineProvider[Configuration]{} -func (InlineProvider[R]) Key() R { return "inline" } +func (InlineProvider[R]) Role() R { var r R; return r } +func (InlineProvider[R]) Key() string { return "inline" } func (i InlineProvider[R]) Writer() bool { return i.Inline } diff --git a/common/configuration/keychain.go b/common/configuration/keychain_provider.go similarity index 91% rename from common/configuration/keychain.go rename to common/configuration/keychain_provider.go index 5fe100e662..9af59899e8 100644 --- a/common/configuration/keychain.go +++ b/common/configuration/keychain_provider.go @@ -16,7 +16,8 @@ type KeychainProvider struct { var _ MutableProvider[Secrets] = KeychainProvider{} -func (k KeychainProvider) Key() Secrets { return "keychain" } +func (KeychainProvider) Role() Secrets { return Secrets{} } +func (k KeychainProvider) Key() string { return "keychain" } func (k KeychainProvider) Writer() bool { return k.Keychain } diff --git a/common/configuration/manager.go b/common/configuration/manager.go index d29ad2272b..5718e54718 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -15,22 +15,26 @@ type Role interface { Secrets | Configuration } -type Secrets string +type Secrets struct{} -type Configuration string +func (Secrets) String() string { return "secrets" } + +type Configuration struct{} + +func (Configuration) String() string { return "configuration" } // Manager is a high-level configuration manager that abstracts the details of // the Resolver and Provider interfaces. type Manager[R Role] struct { - providers map[R]Provider[R] + providers map[string]Provider[R] writer MutableProvider[R] - resolver Resolver + resolver Resolver[R] } // New configuration manager. -func New[R Role](ctx context.Context, resolver Resolver, providers []Provider[R]) (*Manager[R], error) { +func New[R Role](ctx context.Context, resolver Resolver[R], providers []Provider[R]) (*Manager[R], error) { m := &Manager[R]{ - providers: map[R]Provider[R]{}, + providers: map[string]Provider[R]{}, } for _, p := range providers { m.providers[p.Key()] = p @@ -54,7 +58,7 @@ func (m *Manager[R]) Mutable() error { writers := []string{} for _, p := range m.providers { if mutable, ok := p.(MutableProvider[R]); ok { - writers = append(writers, "--"+string(mutable.Key())) + writers = append(writers, "--"+mutable.Key()) } } return fmt.Errorf("no writeable configuration provider available, specify one of %s", strings.Join(writers, ", ")) @@ -76,7 +80,7 @@ func (m *Manager[R]) Get(ctx context.Context, ref Ref, value any) error { } else if err != nil { return err } - provider, ok := m.providers[R(key.Scheme)] + provider, ok := m.providers[key.Scheme] if !ok { return fmt.Errorf("no provider for scheme %q", key.Scheme) } diff --git a/common/configuration/manager_test.go b/common/configuration/manager_test.go index fffab30c68..56c620ad5f 100644 --- a/common/configuration/manager_test.go +++ b/common/configuration/manager_test.go @@ -29,7 +29,7 @@ func TestManager(t *testing.T) { _, err := kcp.Store(ctx, Ref{Name: "mutable"}, []byte("hello")) assert.NoError(t, err) cf, err := New(ctx, - ProjectConfigResolver[Secrets]{Config: config}, + ProjectConfigResolver[Secrets]{Config: []string{config}}, []Provider[Secrets]{ EnvarProvider[Secrets]{}, InlineProvider[Secrets]{}, @@ -44,7 +44,7 @@ func TestManager(t *testing.T) { }) t.Run("Configuration", func(t *testing.T) { cf, err := New(ctx, - ProjectConfigResolver[Configuration]{Config: config}, + ProjectConfigResolver[Configuration]{Config: []string{config}}, []Provider[Configuration]{ EnvarProvider[Configuration]{}, InlineProvider[Configuration]{Inline: true}, // Writer diff --git a/common/configuration/projectconfig.go b/common/configuration/projectconfig_resolver.go similarity index 82% rename from common/configuration/projectconfig.go rename to common/configuration/projectconfig_resolver.go index f7ee689e80..7a8cffabf0 100644 --- a/common/configuration/projectconfig.go +++ b/common/configuration/projectconfig_resolver.go @@ -2,12 +2,12 @@ package configuration import ( "context" - "errors" "fmt" "net/url" "os" "path/filepath" "sort" + "strings" "github.com/alecthomas/types/optional" "golang.org/x/exp/maps" @@ -22,11 +22,13 @@ import ( // // See the [projectconfig] package for details on the configuration file format. type ProjectConfigResolver[R Role] struct { - Config string `help:"Load project configuration file." placeholder:"FILE" type:"existingfile" env:"FTL_CONFIG"` + Config []string `help:"Path to project configuration file." placeholder:"FILE" type:"existingfile" env:"FTL_CONFIG"` } -var _ Resolver = ProjectConfigResolver[Configuration]{} -var _ Resolver = ProjectConfigResolver[Secrets]{} +var _ Resolver[Configuration] = ProjectConfigResolver[Configuration]{} +var _ Resolver[Secrets] = ProjectConfigResolver[Secrets]{} + +func (p ProjectConfigResolver[R]) Role() R { var r R; return r } func (p ProjectConfigResolver[R]) Get(ctx context.Context, ref Ref) (*url.URL, error) { mapping, err := p.getMapping(ctx, ref.Module) @@ -87,21 +89,24 @@ func (p ProjectConfigResolver[From]) Unset(ctx context.Context, ref Ref) error { return p.setMapping(ctx, ref.Module, mapping) } -func (p ProjectConfigResolver[R]) configPath() string { - if p.Config != "" { +func (p ProjectConfigResolver[R]) configPaths() []string { + if len(p.Config) > 0 { return p.Config } - return filepath.Join(internal.GitRoot(""), "ftl-project.toml") + path := filepath.Join(internal.GitRoot(""), "ftl-project.toml") + _, err := os.Stat(path) + if err == nil { + return []string{path} + } + return []string{} } func (p ProjectConfigResolver[R]) loadConfig(ctx context.Context) (pc.Config, error) { logger := log.FromContext(ctx) - configPath := p.configPath() - logger.Tracef("Loading config from %s", configPath) - config, err := pc.Load(configPath) - if errors.Is(err, os.ErrNotExist) { - return pc.Config{}, nil - } else if err != nil { + configPaths := p.configPaths() + logger.Debugf("Loading config from %s", strings.Join(configPaths, " ")) + config, err := pc.Merge(configPaths...) + if err != nil { return pc.Config{}, err } return config, nil @@ -166,5 +171,6 @@ func (p ProjectConfigResolver[R]) setMapping(ctx context.Context, module optiona } else { set(&config.Global, mapping) } - return pc.Save(p.configPath(), config) + configPaths := p.configPaths() + return pc.Save(configPaths[len(configPaths)-1], config) } diff --git a/common/projectconfig/merge.go b/common/projectconfig/merge.go new file mode 100644 index 0000000000..647fcf592e --- /dev/null +++ b/common/projectconfig/merge.go @@ -0,0 +1,43 @@ +package projectconfig + +// Merge configuration files. +// +// Config is merged left to right, with later files taking precedence over earlier files. +func Merge(paths ...string) (Config, error) { + config := Config{} + for _, path := range paths { + partial, err := Load(path) + if err != nil { + return config, err + } + config = merge(config, partial) + } + return config, nil +} + +func merge(a, b Config) Config { + a.Global = mergeConfigAndSecrets(a.Global, b.Global) + for k, v := range b.Modules { + if a.Modules == nil { + a.Modules = map[string]ConfigAndSecrets{} + } + a.Modules[k] = mergeConfigAndSecrets(a.Modules[k], v) + } + return a +} + +func mergeConfigAndSecrets(a, b ConfigAndSecrets) ConfigAndSecrets { + for k, v := range b.Config { + if a.Config == nil { + a.Config = map[string]*URL{} + } + a.Config[k] = v + } + for k, v := range b.Secrets { + if a.Secrets == nil { + a.Secrets = map[string]*URL{} + } + a.Secrets[k] = v + } + return a +} diff --git a/common/projectconfig/merge_test.go b/common/projectconfig/merge_test.go new file mode 100644 index 0000000000..5e26ecf328 --- /dev/null +++ b/common/projectconfig/merge_test.go @@ -0,0 +1,82 @@ +package projectconfig + +import ( + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestMerge(t *testing.T) { + a := Config{ + Global: ConfigAndSecrets{ + Config: map[string]*URL{ + "key1": MustParseURL("inline://foo"), + }, + Secrets: map[string]*URL{ + "key2": MustParseURL("inline://bar"), + }, + }, + Modules: map[string]ConfigAndSecrets{ + "test": { + Config: map[string]*URL{ + "key3": MustParseURL("inline://baz"), + }, + Secrets: map[string]*URL{ + "key4": MustParseURL("inline://qux"), + "key10": MustParseURL("inline://qux10"), + }, + }, + }, + } + b := Config{ + Global: ConfigAndSecrets{ + Config: map[string]*URL{ + "key1": MustParseURL("inline://foo2"), + "key5": MustParseURL("inline://foo5"), + }, + Secrets: map[string]*URL{ + "key2": MustParseURL("inline://bar2"), + "key6": MustParseURL("inline://bar6"), + }, + }, + Modules: map[string]ConfigAndSecrets{ + "test": { + Config: map[string]*URL{ + "key3": MustParseURL("inline://baz2"), + "key7": MustParseURL("inline://baz7"), + }, + Secrets: map[string]*URL{ + "key4": MustParseURL("inline://qux2"), + "key8": MustParseURL("inline://qux8"), + }, + }, + }, + } + a = merge(a, b) + expected := Config{ + Global: ConfigAndSecrets{ + Config: map[string]*URL{ + "key1": MustParseURL("inline://foo2"), + "key5": MustParseURL("inline://foo5"), + }, + Secrets: map[string]*URL{ + "key2": MustParseURL("inline://bar2"), + "key6": MustParseURL("inline://bar6"), + }, + }, + Modules: map[string]ConfigAndSecrets{ + "test": { + Config: map[string]*URL{ + "key3": MustParseURL("inline://baz2"), + "key7": MustParseURL("inline://baz7"), + }, + Secrets: map[string]*URL{ + "key4": MustParseURL("inline://qux2"), + "key8": MustParseURL("inline://qux8"), + "key10": MustParseURL("inline://qux10"), + }, + }, + }, + } + assert.Equal(t, expected, a) +} diff --git a/examples/go/echo/go.mod b/examples/go/echo/go.mod index bee24467b1..62fed4c898 100644 --- a/examples/go/echo/go.mod +++ b/examples/go/echo/go.mod @@ -13,7 +13,7 @@ require ( github.com/BurntSushi/toml v1.3.2 // indirect github.com/TBD54566975/scaffolder v0.8.0 // indirect github.com/alecthomas/concurrency v0.0.2 // indirect - github.com/alecthomas/kong v0.8.1 // indirect + github.com/alecthomas/kong v0.9.0 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect github.com/alecthomas/types v0.13.0 // indirect github.com/alessio/shellescape v1.4.2 // indirect diff --git a/examples/go/echo/go.sum b/examples/go/echo/go.sum index f0a092c5ac..42527a30c9 100644 --- a/examples/go/echo/go.sum +++ b/examples/go/echo/go.sum @@ -12,8 +12,8 @@ github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/concurrency v0.0.2 h1:Q3kGPtLbleMbH9lHX5OBFvJygfyFw29bXZKBg+IEVuo= github.com/alecthomas/concurrency v0.0.2/go.mod h1:GmuQb/iHX7mbNtPlC/WDzEFxDMB0HYFer2Qda9QTs7w= -github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= -github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA= +github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os= github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8= github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= diff --git a/go-runtime/ftl/config_test.go b/go-runtime/ftl/config_test.go index bffbf56303..bad71287ba 100644 --- a/go-runtime/ftl/config_test.go +++ b/go-runtime/ftl/config_test.go @@ -12,7 +12,8 @@ import ( func TestConfig(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - cm, err := configuration.NewConfigurationManager(ctx, "testdata/ftl-project.toml") + cr := configuration.ProjectConfigResolver[configuration.Configuration]{Config: []string{"testdata/ftl-project.toml"}} + cm, err := configuration.NewConfigurationManager(ctx, cr) assert.NoError(t, err) ctx = configuration.ContextWithConfig(ctx, cm) type C struct { diff --git a/go-runtime/ftl/ftltest/ftltest.go b/go-runtime/ftl/ftltest/ftltest.go index ea8ef2d464..f08aa296b9 100644 --- a/go-runtime/ftl/ftltest/ftltest.go +++ b/go-runtime/ftl/ftltest/ftltest.go @@ -4,22 +4,30 @@ package ftltest import ( "context" - "github.com/TBD54566975/ftl/common/configuration" + "github.com/alecthomas/kong" + "github.com/alecthomas/repr" + + cf "github.com/TBD54566975/ftl/common/configuration" "github.com/TBD54566975/ftl/internal/log" ) // Context suitable for use in testing FTL verbs. func Context() context.Context { ctx := log.ContextWithNewDefaultLogger(context.Background()) - cm, err := configuration.DefaultConfigMixin{}.NewConfigurationManager(ctx) + cr := &cf.ProjectConfigResolver[cf.Configuration]{Config: []string{}} + _ = kong.ApplyDefaults(cr) + repr.Println(cr) + cm, err := cf.NewConfigurationManager(ctx, cr) if err != nil { panic(err) } - ctx = configuration.ContextWithConfig(ctx, cm) - sm, err := configuration.DefaultSecretsMixin{}.NewSecretsManager(ctx) + ctx = cf.ContextWithConfig(ctx, cm) + sr := &cf.ProjectConfigResolver[cf.Secrets]{Config: []string{}} + _ = kong.ApplyDefaults(sr) + sm, err := cf.NewSecretsManager(ctx, sr) if err != nil { panic(err) } - ctx = configuration.ContextWithSecrets(ctx, sm) + ctx = cf.ContextWithSecrets(ctx, sm) return ctx } diff --git a/go-runtime/ftl/secrets_test.go b/go-runtime/ftl/secrets_test.go index 4874f48657..47ed08d0e0 100644 --- a/go-runtime/ftl/secrets_test.go +++ b/go-runtime/ftl/secrets_test.go @@ -12,7 +12,8 @@ import ( func TestSecret(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - sm, err := configuration.NewSecretsManager(ctx, "testdata/ftl-project.toml") + sr := configuration.ProjectConfigResolver[configuration.Secrets]{Config: []string{"testdata/ftl-project.toml"}} + sm, err := configuration.NewSecretsManager(ctx, sr) assert.NoError(t, err) ctx = configuration.ContextWithSecrets(ctx, sm) type C struct { diff --git a/go-runtime/server/server.go b/go-runtime/server/server.go index 868e34f6a4..29b858c2fd 100644 --- a/go-runtime/server/server.go +++ b/go-runtime/server/server.go @@ -23,8 +23,7 @@ import ( type UserVerbConfig struct { FTLEndpoint *url.URL `help:"FTL endpoint." env:"FTL_ENDPOINT" required:""` ObservabilityConfig observability.Config `embed:"" prefix:"o11y-"` - - Config string `help:"Load project configuration file." placeholder:"FILE" type:"existingfile" env:"FTL_CONFIG"` + Config []string `name:"config" short:"C" help:"Paths to FTL project configuration files." env:"FTL_CONFIG" placeholder:"FILE[,FILE,...]" type:"existingfile"` } // NewUserVerbServer starts a new code-generated drive for user Verbs. @@ -33,16 +32,19 @@ type UserVerbConfig struct { func NewUserVerbServer(moduleName string, handlers ...Handler) plugin.Constructor[ftlv1connect.VerbServiceHandler, UserVerbConfig] { return func(ctx context.Context, uc UserVerbConfig) (context.Context, ftlv1connect.VerbServiceHandler, error) { verbServiceClient := rpc.Dial(ftlv1connect.NewVerbServiceClient, uc.FTLEndpoint.String(), log.Error) - // Add config manager to context. ctx = rpc.ContextWithClient(ctx, verbServiceClient) - cm, err := cf.NewConfigurationManager(ctx, uc.Config) + + // Add config manager to context. + cr := &cf.ProjectConfigResolver[cf.Configuration]{Config: uc.Config} + cm, err := cf.NewConfigurationManager(ctx, cr) if err != nil { return nil, nil, err } ctx = cf.ContextWithConfig(ctx, cm) // Add secrets manager to context. - sm, err := cf.NewSecretsManager(ctx, uc.Config) + sr := &cf.ProjectConfigResolver[cf.Secrets]{Config: uc.Config} + sm, err := cf.NewSecretsManager(ctx, sr) if err != nil { return nil, nil, err } diff --git a/go.mod b/go.mod index 1398d494f9..14c7dd7f6b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/alecthomas/assert/v2 v2.6.0 github.com/alecthomas/atomic v0.1.0-alpha2 github.com/alecthomas/concurrency v0.0.2 - github.com/alecthomas/kong v0.8.1 + github.com/alecthomas/kong v0.9.0 github.com/alecthomas/kong-toml v0.1.0 github.com/alecthomas/participle/v2 v2.1.1 github.com/alecthomas/types v0.13.0 diff --git a/go.sum b/go.sum index 1567859668..4da5a3d6a8 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELk github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI= github.com/alecthomas/concurrency v0.0.2 h1:Q3kGPtLbleMbH9lHX5OBFvJygfyFw29bXZKBg+IEVuo= github.com/alecthomas/concurrency v0.0.2/go.mod h1:GmuQb/iHX7mbNtPlC/WDzEFxDMB0HYFer2Qda9QTs7w= -github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= -github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/kong v0.9.0 h1:G5diXxc85KvoV2f0ZRVuMsi45IrBgx9zDNGNj165aPA= +github.com/alecthomas/kong v0.9.0/go.mod h1:Y47y5gKfHp1hDc7CH7OeXgLIpp+Q2m1Ni0L5s3bI8Os= github.com/alecthomas/kong-toml v0.1.0 h1:jKrdGj/G0mkHGbOV5a+Cok3cTDZ+Qxa5CBq177gPKUA= github.com/alecthomas/kong-toml v0.1.0/go.mod h1:aDIxp+T6kJZY9zLThZX6qI9xxUlQgYlvqmw7PI//7/Y= github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=