feat: partial implemention of the secrets/config design (#1003)

Design is [here](https://hackmd.io/@ftl/S1e6YVEuq6).

Specifically, this doesn't implement any of the backend changes, only
the layered reference storage approach. References are currently only
stored in an `ftl-project.toml` file stored at the root of the
repository.

It supports the following configuration providers:

- Inlined.
- Environment variables.
- 1Password [secret
references](https://developer.1password.com/docs/cli/secret-references/)
- The system keychain.

Note also that this is not hooked up to the Go runtime yet. Will do that
in a followup. Kotlin will have to wait until the full backend
implementation is available.

Fixes #1001

cmd/ftl/cmd_config.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/alecthomas/kong"
+
+	"github.com/TBD54566975/ftl/common/configuration"
+)
+
+type mutableConfigProviderMixin struct {
+	configuration.InlineProvider
+	configuration.EnvarProvider[configuration.EnvarTypeConfig]
+}
+
+func (s *mutableConfigProviderMixin) newConfigManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) {
+	return configuration.New(ctx, resolver, []configuration.Provider{s.InlineProvider, s.EnvarProvider})
+}
+
+type configCmd struct {
+	configuration.ProjectConfigResolver[configuration.FromConfig]
+
+	List  configListCmd  `cmd:"" help:"List configuration."`
+	Get   configGetCmd   `cmd:"" help:"Get a configuration value."`
+	Set   configSetCmd   `cmd:"" help:"Set a configuration value."`
+	Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."`
+}
+
+func (s *configCmd) newConfigManager(ctx context.Context) (*configuration.Manager, error) {
+	mp := mutableConfigProviderMixin{}
+	_ = kong.ApplyDefaults(&mp)
+	return mp.newConfigManager(ctx, s.ProjectConfigResolver)
+}
+
+func (s *configCmd) Help() string {
+	return `
+Configuration values are used to store non-sensitive information such as URLs,
+etc.
+`
+}
+
+type configListCmd struct {
+	Values bool   `help:"List configuration values."`
+	Module string `optional:"" arg:"" placeholder:"MODULE" help:"List configuration only in this module."`
+}
+
+func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error {
+	sm, err := scmd.newConfigManager(ctx)
+	if err != nil {
+		return err
+	}
+	listing, err := sm.List(ctx)
+	if err != nil {
+		return err
+	}
+	for _, config := range listing {
+		module, ok := config.Module.Get()
+		if s.Module != "" && module != s.Module {
+			continue
+		}
+		if ok {
+			fmt.Printf("%s.%s", module, config.Name)
+		} else {
+			fmt.Print(config.Name)
+		}
+		if s.Values {
+			var value any
+			err := sm.Get(ctx, config.Ref, &value)
+			if err != nil {
+				fmt.Printf(" (error: %s)\n", err)
+			} else {
+				data, _ := json.Marshal(value)
+				fmt.Printf(" = %s\n", data)
+			}
+		} else {
+			fmt.Println()
+		}
+	}
+	return nil
+
+}
+
+type configGetCmd struct {
+	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+}
+
+func (s *configGetCmd) Help() string {
+	return `
+Returns a JSON-encoded configuration value.
+`
+}
+
+func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error {
+	sm, err := scmd.newConfigManager(ctx)
+	if err != nil {
+		return err
+	}
+	var value any
+	err = sm.Get(ctx, s.Ref, &value)
+	if err != nil {
+		return err
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	err = enc.Encode(value)
+	if err != nil {
+		return fmt.Errorf("%s: %w", s.Ref, err)
+	}
+	return nil
+}
+
+type configSetCmd struct {
+	mutableConfigProviderMixin
+
+	JSON  bool              `help:"Assume input value is JSON."`
+	Ref   configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+	Value *string           `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
+}
+
+func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error {
+	sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver)
+	if err != nil {
+		return err
+	}
+
+	if err := sm.Mutable(); err != nil {
+		return err
+	}
+
+	var config []byte
+	if s.Value != nil {
+		config = []byte(*s.Value)
+	} else {
+		config, err = io.ReadAll(os.Stdin)
+		if err != nil {
+			return fmt.Errorf("failed to read config from stdin: %w", err)
+		}
+	}
+
+	var configValue any
+	if s.JSON {
+		if err := json.Unmarshal(config, &configValue); err != nil {
+			return fmt.Errorf("config is not valid JSON: %w", err)
+		}
+	} else {
+		configValue = string(config)
+	}
+	return sm.Set(ctx, s.Ref, configValue)
+}
+
+type configUnsetCmd struct {
+	mutableConfigProviderMixin
+
+	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
+}
+
+func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd) error {
+	sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver)
+	if err != nil {
+		return err
+	}
+	return sm.Unset(ctx, s.Ref)
+}

cmd/ftl/cmd_secret.go

@@ -0,0 +1,182 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/alecthomas/kong"
+	"github.com/mattn/go-isatty"
+	"golang.org/x/term"
+
+	"github.com/TBD54566975/ftl/common/configuration"
+)
+
+type mutableSecretProviderMixin struct {
+	configuration.InlineProvider
+	configuration.KeychainProvider
+	configuration.EnvarProvider[configuration.EnvarTypeSecrets]
+	configuration.OnePasswordProvider
+}
+
+func (s *mutableSecretProviderMixin) newSecretsManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) {
+	return configuration.New(ctx, resolver, []configuration.Provider{
+		s.InlineProvider, s.KeychainProvider, s.EnvarProvider, s.OnePasswordProvider,
+	})
+}
+
+type secretCmd struct {
+	configuration.ProjectConfigResolver[configuration.FromSecrets]
+
+	List  secretListCmd  `cmd:"" help:"List secrets."`
+	Get   secretGetCmd   `cmd:"" help:"Get a secret."`
+	Set   secretSetCmd   `cmd:"" help:"Set a secret."`
+	Unset secretUnsetCmd `cmd:"" help:"Unset a secret."`
+}
+
+func (s *secretCmd) newSecretsManager(ctx context.Context) (*configuration.Manager, error) {
+	mp := mutableSecretProviderMixin{}
+	_ = kong.ApplyDefaults(&mp)
+	return mp.newSecretsManager(ctx, s.ProjectConfigResolver)
+}
+
+func (s *secretCmd) Help() string {
+	return `
+Secrets are used to store sensitive information such as passwords, tokens, and
+keys. When setting a secret, the value is read from a password prompt if stdin
+is a terminal, otherwise it is read from stdin directly. Secrets can be stored
+in the project's configuration file, in the system keychain, in environment
+variables, and so on.
+`
+}
+
+type secretListCmd struct {
+	Values bool   `help:"List secret values."`
+	Module string `optional:"" arg:"" placeholder:"MODULE" help:"List secrets only in this module."`
+}
+
+func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error {
+	sm, err := scmd.newSecretsManager(ctx)
+	if err != nil {
+		return err
+	}
+	listing, err := sm.List(ctx)
+	if err != nil {
+		return err
+	}
+	for _, secret := range listing {
+		module, ok := secret.Module.Get()
+		if s.Module != "" && module != s.Module {
+			continue
+		}
+		if ok {
+			fmt.Printf("%s.%s", module, secret.Name)
+		} else {
+			fmt.Print(secret.Name)
+		}
+		if s.Values {
+			var value any
+			err := sm.Get(ctx, secret.Ref, &value)
+			if err != nil {
+				fmt.Printf(" (error: %s)\n", err)
+			} else {
+				data, _ := json.Marshal(value)
+				fmt.Printf(" = %s\n", data)
+			}
+		} else {
+			fmt.Println()
+		}
+	}
+	return nil
+
+}
+
+type secretGetCmd struct {
+	Ref configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
+}
+
+func (s *secretGetCmd) Help() string {
+	return `
+Returns a JSON-encoded secret value.
+`
+}
+
+func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error {
+	sm, err := scmd.newSecretsManager(ctx)
+	if err != nil {
+		return err
+	}
+	var value any
+	err = sm.Get(ctx, s.Ref, &value)
+	if err != nil {
+		return err
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	err = enc.Encode(value)
+	if err != nil {
+		return fmt.Errorf("%s: %w", s.Ref, err)
+	}
+	return nil
+}
+
+type secretSetCmd struct {
+	mutableSecretProviderMixin
+
+	JSON bool              `help:"Assume input value is JSON."`
+	Ref  configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
+}
+
+func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error {
+	sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver)
+	if err != nil {
+		return err
+	}
+
+	if err := sm.Mutable(); err != nil {
+		return err
+	}
+
+	// Prompt for a secret if stdin is a terminal, otherwise read from stdin.
+	var secret []byte
+	if isatty.IsTerminal(0) {
+		fmt.Print("Secret: ")
+		secret, err = term.ReadPassword(0)
+		fmt.Println()
+		if err != nil {
+			return err
+		}
+	} else {
+		secret, err = io.ReadAll(os.Stdin)
+		if err != nil {
+			return fmt.Errorf("failed to read secret from stdin: %w", err)
+		}
+	}
+
+	var secretValue any
+	if s.JSON {
+		if err := json.Unmarshal(secret, &secretValue); err != nil {
+			return fmt.Errorf("secret is not valid JSON: %w", err)
+		}
+	} else {
+		secretValue = string(secret)
+	}
+	return sm.Set(ctx, s.Ref, secretValue)
+}
+
+type secretUnsetCmd struct {
+	mutableSecretProviderMixin
+
+	Ref configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
+}
+
+func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd) error {
+	sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver)
+	if err != nil {
+		return err
+	}
+	return sm.Unset(ctx, s.Ref)
+}

cmd/ftl/main.go

@@ -20,7 +20,6 @@ import (
 
 type CLI struct {
 	Version   kong.VersionFlag `help:"Show version."`
-	Config    kong.ConfigFlag  `help:"Load configuration from TOML file." placeholder:"FILE"`
 	LogConfig log.Config       `embed:"" prefix:"log-" group:"Logging:"`
 	Endpoint  *url.URL         `default:"http://127.0.0.1:8892" help:"FTL endpoint to bind/connect to." env:"FTL_ENDPOINT"`
 
@@ -38,6 +37,8 @@ type CLI struct {
 	Build    buildCmd    `cmd:"" help:"Build an FTL module."`
 	Deploy   deployCmd   `cmd:"" help:"Create a new deployment."`
 	Download downloadCmd `cmd:"" help:"Download a deployment."`
+	Secret   secretCmd   `cmd:"" help:"Manage secrets."`
+	Config   configCmd   `cmd:"" help:"Manage configuration."`
 }
 
 var cli CLI
@@ -46,6 +47,8 @@ func main() {
 	kctx := kong.Parse(&cli,
 		kong.Description(`FTL - Towards a 𝝺-calculus for large-scale systems`),
 		kong.Configuration(kongtoml.Loader, ".ftl.toml", "~/.ftl.toml"),
+		kong.ShortUsageOnError(),
+		kong.HelpOptions{Compact: true, WrapUpperBound: 80},
 		kong.AutoGroup(func(parent kong.Visitable, flag *kong.Flag) *kong.Group {
 			node, ok := parent.(*kong.Command)
 			if !ok {