feat: import/export of config and secrets (#1982)

closes #1948

**Usage:**

**Exporting**
Prints json of all secrets to stdout:
`ftl secret export`

Exports all inline secrets to a json file
`ftl secret export --inline > exported.json`

**Importing**
Imports json from a file:
`ftl secret import --inline exported.json`

**Migrating**
Migrates inline secrets to 1Password:
`ftl secret export --inline | ftl secret import --op --opvault <vauld
id>`

**JSON Format**
```
{
    "<module>.<name>": <secret>,
    ...
}
```

backend/controller/admin/admin.go

@@ -40,7 +40,7 @@ func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv
 	configs := []*ftlv1.ListConfigResponse_Config{}
 	for _, config := range listing {
 		module, ok := config.Module.Get()
-		if *req.Msg.Module != "" && module != *req.Msg.Module {
+		if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
 			continue
 		}
 
@@ -127,8 +127,12 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl
 	}
 	secrets := []*ftlv1.ListSecretsResponse_Secret{}
 	for _, secret := range listing {
+		if req.Msg.Provider != nil && cf.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) {
+			// Skip secrets that don't match the provider in the request
+			continue
+		}
 		module, ok := secret.Module.Get()
-		if *req.Msg.Module != "" && module != *req.Msg.Module {
+		if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
 			continue
 		}
 		ref := secret.Name

backend/protos/xyz/block/ftl/v1/ftl.proto

@@ -414,6 +414,7 @@ enum ConfigProvider {
 message ListConfigRequest {
   optional string module = 1;
   optional bool include_values = 2;
+  optional ConfigProvider provider = 3;
 }
 message ListConfigResponse {
   message Config {
@@ -463,6 +464,7 @@ enum SecretProvider {
 message ListSecretsRequest {
   optional string module = 1;
   optional bool include_values = 2;
+  optional SecretProvider provider = 3;
 }
 message ListSecretsResponse {
   message Secret {

cmd/ftl/cmd_config.go

@@ -17,10 +17,12 @@ import (
 )
 
 type configCmd struct {
-	List  configListCmd  `cmd:"" help:"List configuration."`
-	Get   configGetCmd   `cmd:"" help:"Get a configuration value."`
-	Set   configSetCmd   `cmd:"" help:"Set a configuration value."`
-	Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."`
+	List   configListCmd   `cmd:"" help:"List configuration."`
+	Get    configGetCmd    `cmd:"" help:"Get a configuration value."`
+	Set    configSetCmd    `cmd:"" help:"Set a configuration value."`
+	Unset  configUnsetCmd  `cmd:"" help:"Unset a configuration value."`
+	Import configImportCmd `cmd:"" help:"Import configuration values."`
+	Export configExportCmd `cmd:"" help:"Export configuration values."`
 
 	Envar  bool `help:"Print configuration as environment variables." group:"Provider:" xor:"configwriter"`
 	Inline bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"configwriter"`
@@ -162,3 +164,85 @@ func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, adminClient a
 	}
 	return nil
 }
+
+type configImportCmd struct {
+	Input *os.File `arg:"" placeholder:"JSON" help:"JSON to import as configuration values (read from stdin if omitted). Format: {\"<module>.<name>\": <value>, ... }" optional:"" default:"-"`
+}
+
+func (s *configImportCmd) Help() string {
+	return `
+Imports configuration values from a JSON object.
+`
+}
+
+func (s *configImportCmd) Run(ctx context.Context, cmd *configCmd, adminClient admin.Client) error {
+	input, err := io.ReadAll(s.Input)
+	if err != nil {
+		return fmt.Errorf("failed to read input: %w", err)
+	}
+	var entries map[string]json.RawMessage
+	err = json.Unmarshal(input, &entries)
+	if err != nil {
+		return fmt.Errorf("could not parse JSON: %w", err)
+	}
+	for refPath, value := range entries {
+		ref, err := cf.ParseRef(refPath)
+		if err != nil {
+			return fmt.Errorf("could not parse ref %q: %w", refPath, err)
+		}
+		bytes, err := json.Marshal(value)
+		if err != nil {
+			return fmt.Errorf("could not marshal value for %q: %w", refPath, err)
+		}
+		req := &ftlv1.SetConfigRequest{
+			Ref:   configRefFromRef(ref),
+			Value: bytes,
+		}
+		if provider, ok := cmd.provider().Get(); ok {
+			req.Provider = &provider
+		}
+		_, err = adminClient.ConfigSet(ctx, connect.NewRequest(req))
+		if err != nil {
+			return fmt.Errorf("could not import config for %q: %w", refPath, err)
+		}
+	}
+	return nil
+}
+
+type configExportCmd struct {
+}
+
+func (s *configExportCmd) Help() string {
+	return `
+Outputs configuration values in a JSON object. A provider can be used to filter which values are included.
+`
+}
+
+func (s *configExportCmd) Run(ctx context.Context, cmd *configCmd, adminClient admin.Client) error {
+	req := &ftlv1.ListConfigRequest{
+		IncludeValues: optional.Some(true).Ptr(),
+	}
+	if provider, ok := cmd.provider().Get(); ok {
+		req.Provider = &provider
+	}
+	listResponse, err := adminClient.ConfigList(ctx, connect.NewRequest(req))
+	if err != nil {
+		return fmt.Errorf("could not retrieve configs: %w", err)
+	}
+	entries := make(map[string]json.RawMessage, 0)
+	for _, config := range listResponse.Msg.Configs {
+		var value json.RawMessage
+		err = json.Unmarshal(config.Value, &value)
+		if err != nil {
+			return fmt.Errorf("could not export %q: %w", config.RefPath, err)
+		}
+		entries[config.RefPath] = value
+	}
+
+	output, err := json.Marshal(entries)
+	if err != nil {
+		return fmt.Errorf("could not build output: %w", err)
+	}
+	fmt.Println(string(output))
+	return nil
+}

cmd/ftl/cmd_secret.go

@@ -19,10 +19,12 @@ import (
 )
 
 type secretCmd struct {
-	List  secretListCmd  `cmd:"" help:"List secrets."`
-	Get   secretGetCmd   `cmd:"" help:"Get a secret."`
-	Set   secretSetCmd   `cmd:"" help:"Set a secret."`
-	Unset secretUnsetCmd `cmd:"" help:"Unset a secret."`
+	List   secretListCmd   `cmd:"" help:"List secrets."`
+	Get    secretGetCmd    `cmd:"" help:"Get a secret."`
+	Set    secretSetCmd    `cmd:"" help:"Set a secret."`
+	Unset  secretUnsetCmd  `cmd:"" help:"Unset a secret."`
+	Import secretImportCmd `cmd:"" help:"Import secrets."`
+	Export secretExportCmd `cmd:"" help:"Export secrets."`
 
 	Envar    bool `help:"Write configuration as environment variables." group:"Provider:" xor:"secretwriter"`
 	Inline   bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"secretwriter"`
@@ -169,3 +171,85 @@ func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient a
 	}
 	return nil
 }
+
+type secretImportCmd struct {
+	Input *os.File `arg:"" placeholder:"JSON" help:"JSON to import as secrets (read from stdin if omitted). Format: {\"<module>.<name>\": <secret>, ... }" optional:"" default:"-"`
+}
+
+func (s *secretImportCmd) Help() string {
+	return `
+Imports secrets from a JSON object.
+`
+}
+
+func (s *secretImportCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error {
+	input, err := io.ReadAll(s.Input)
+	if err != nil {
+		return fmt.Errorf("failed to read input: %w", err)
+	}
+	var entries map[string]json.RawMessage
+	err = json.Unmarshal(input, &entries)
+	if err != nil {
+		return fmt.Errorf("could not parse JSON: %w", err)
+	}
+	for refPath, value := range entries {
+		ref, err := cf.ParseRef(refPath)
+		if err != nil {
+			return fmt.Errorf("could not parse ref %q: %w", refPath, err)
+		}
+		bytes, err := json.Marshal(value)
+		if err != nil {
+			return fmt.Errorf("could not marshal value for %q: %w", refPath, err)
+		}
+		req := &ftlv1.SetSecretRequest{
+			Ref:   configRefFromRef(ref),
+			Value: bytes,
+		}
+		if provider, ok := scmd.provider().Get(); ok {
+			req.Provider = &provider
+		}
+		_, err = adminClient.SecretSet(ctx, connect.NewRequest(req))
+		if err != nil {
+			return fmt.Errorf("could not import secret for %q: %w", refPath, err)
+		}
+	}
+	return nil
+}
+
+type secretExportCmd struct {
+}
+
+func (s *secretExportCmd) Help() string {
+	return `
+Outputs secrets in a JSON object. A provider can be used to filter which secrets are included.
+`
+}
+
+func (s *secretExportCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error {
+	req := &ftlv1.ListSecretsRequest{
+		IncludeValues: optional.Some(true).Ptr(),
+	}
+	if provider, ok := scmd.provider().Get(); ok {
+		req.Provider = &provider
+	}
+	listResponse, err := adminClient.SecretsList(ctx, connect.NewRequest(req))
+	if err != nil {
+		return fmt.Errorf("could not retrieve secrets: %w", err)
+	}
+	entries := make(map[string]json.RawMessage, 0)
+	for _, secret := range listResponse.Msg.Secrets {
+		var value json.RawMessage
+		err = json.Unmarshal(secret.Value, &value)
+		if err != nil {
+			return fmt.Errorf("could not export %q: %w", secret.RefPath, err)
+		}
+		entries[secret.RefPath] = value
+	}
+
+	output, err := json.Marshal(entries)
+	if err != nil {
+		return fmt.Errorf("could not build output: %w", err)
+	}
+	fmt.Println(string(output))
+	return nil
+}

cmd/ftl/integration_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"context"
+	"path/filepath"
 	"testing"
 
 	"github.com/alecthomas/assert/v2"
@@ -31,3 +32,54 @@ func TestBox(t *testing.T) {
 		Exec("docker", "compose", "-f", "echo-compose.yml", "down", "--rmi", "local"),
 	)
 }
+
+func TestSecretImportExport(t *testing.T) {
+	testImportExport(t, "secret")
+}
+
+func TestConfigImportExport(t *testing.T) {
+	testImportExport(t, "config")
+}
+
+func testImportExport(t *testing.T, object string) {
+	t.Helper()
+
+	firstProjFile := "ftl-project.toml"
+	secondProjFile := "ftl-project-2.toml"
+	destinationFile := "exported.json"
+
+	importPath, err := filepath.Abs("testdata/import.json")
+	assert.NoError(t, err)
+
+	// use a pointer to keep track of the exported json so that i can be modified from within actions
+	blank := ""
+	exported := &blank
+
+	RunWithoutController(t, "",
+		// duplicate project file in the temp directory
+		Exec("cp", firstProjFile, secondProjFile),
+		// import into first project file
+		Exec("ftl", object, "import", "--inline", "--config", firstProjFile, importPath),
+
+		// export from first project file
+		ExecWithOutput("ftl", []string{object, "export", "--config", firstProjFile}, func(output string) {
+			*exported = output
+
+			// make sure the exported json contains a value (otherwise the test could pass with the first import doing nothing)
+			assert.Contains(t, output, "test.one")
+		}),
+
+		// import into second project file
+		// wrapped in a func to avoid capturing the initial valye of *exported
+		func(t testing.TB, ic TestContext) {
+			WriteFile(destinationFile, []byte(*exported))(t, ic)
+			Exec("ftl", object, "import", destinationFile, "--inline", "--config", secondProjFile)(t, ic)
+		},
+
+		// export from second project file
+		ExecWithOutput("ftl", []string{object, "export", "--config", secondProjFile}, func(output string) {
+			// check that both exported the same json
+			assert.Equal(t, *exported, output)
+		}),
+	)
+}

cmd/ftl/testdata/import.json

@@ -0,0 +1,5 @@
+{
+    "test.one": 1,
+    "test.two": "a string",
+    "test2.three": {"key":"value"}
+}

common/configuration/manager.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 
@@ -75,6 +76,10 @@ func New[R Role](ctx context.Context, router Router[R], providers []Provider[R])
 	return m, nil
 }
 
+func ProviderKeyForAccessor(accessor *url.URL) string {
+	return accessor.Scheme
+}
+
 // getData returns a data value for a configuration from the active providers.
 // The data can be unmarshalled from JSON.
 func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) {
@@ -89,7 +94,7 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) {
 	} else if err != nil {
 		return nil, err
 	}
-	provider, ok := m.providers[key.Scheme]
+	provider, ok := m.providers[ProviderKeyForAccessor(key)]
 	if !ok {
 		return nil, fmt.Errorf("no provider for scheme %q", key.Scheme)
 	}