fix: database-backed secret resolver (#1901)

Fixes #1887

backend/controller/sql/schema/001_init.sql

@@ -518,4 +518,14 @@ CREATE TABLE module_configuration
     UNIQUE (module, name)
 );
 
+CREATE TABLE module_secrets
+(
+    id         BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'),
+    module     TEXT,  -- If NULL, configuration is global.
+    name       TEXT   NOT NULL,
+    url        TEXT   NOT NULL,
+    UNIQUE (module, name)
+);
+
 -- migrate:down

cmd/ftl-controller/main.go

@@ -63,9 +63,9 @@ func main() {
 	// The FTL controller currently only supports AWS Secrets Manager as a secrets provider.
 	awsConfig, err := config.LoadDefaultConfig(ctx)
 	kctx.FatalIfErrorf(err)
-	secretsResolver := cf.NewASM(ctx, secretsmanager.NewFromConfig(awsConfig), cli.ControllerConfig.Advertise, dal)
-	secretsProviders := []cf.Provider[cf.Secrets]{secretsResolver}
-	sm, err := cf.New[cf.Secrets](ctx, secretsResolver, secretsProviders)
+	asmSecretProvider := cf.NewASM(ctx, secretsmanager.NewFromConfig(awsConfig), cli.ControllerConfig.Advertise, dal)
+	dbSecretResolver := cf.NewDBSecretResolver(configDal)
+	sm, err := cf.New[cf.Secrets](ctx, dbSecretResolver, []cf.Provider[cf.Secrets]{asmSecretProvider})
 	kctx.FatalIfErrorf(err)
 	ctx = cf.ContextWithSecrets(ctx, sm)
 

common/configuration/asm.go

@@ -2,7 +2,6 @@ package configuration
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 	"time"
 
@@ -24,18 +23,15 @@ type asmClient interface {
 	delete(ctx context.Context, ref Ref) error
 }
 
-// ASM implements Router and Provider for AWS Secrets Manager (ASM).
+// ASM implements a Provider for AWS Secrets Manager (ASM).
 // Only supports loading "string" secrets, not binary secrets.
 //
-// The router does a direct/proxy map from a Ref to a URL, module.name <-> asm://module.name and does not access ASM at all.
-//
 // One controller is elected as the leader and is responsible for syncing the cache of secrets from ASM (see asmLeader).
 // Others get secrets from the leader via AdminService (see asmFollower).
 type ASM struct {
 	coordinator *leader.Coordinator[asmClient]
 }
 
-var _ Router[Secrets] = &ASM{}
 var _ Provider[Secrets] = &ASM{}
 
 func NewASM(ctx context.Context, secretsClient *secretsmanager.Client, advertise *url.URL, leaser leases.Leaser) *ASM {
@@ -78,33 +74,6 @@ func (ASM) Key() string {
 	return "asm"
 }
 
-func (ASM) Get(ctx context.Context, ref Ref) (*url.URL, error) {
-	return asmURLForRef(ref), nil
-}
-
-func (ASM) Set(ctx context.Context, ref Ref, key *url.URL) error {
-	expectedKey := asmURLForRef(ref)
-	if key.String() != expectedKey.String() {
-		return fmt.Errorf("key does not match expected key for ref: %s", expectedKey)
-	}
-
-	return nil
-}
-
-func (ASM) Unset(ctx context.Context, ref Ref) error {
-	// removing a secret is handled in Delete()
-	return nil
-}
-
-// List all secrets in the account
-func (a *ASM) List(ctx context.Context) ([]Entry, error) {
-	client, err := a.coordinator.Get()
-	if err != nil {
-		return nil, err
-	}
-	return client.list(ctx)
-}
-
 func (a *ASM) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {
 	client, err := a.coordinator.Get()
 	if err != nil {

common/configuration/asm_test.go

@@ -54,7 +54,8 @@ func TestASMWorkflow(t *testing.T) {
 	asm, leader, _, _ := localstack(ctx, t)
 	ref := Ref{Module: Some("foo"), Name: "bar"}
 	var mySecret = jsonBytes(t, "my secret")
-	manager, err := New(ctx, asm, []Provider[Secrets]{asm})
+	sr := NewDBSecretResolver(&mockDBSecretResolverDAL{})
+	manager, err := New(ctx, sr, []Provider[Secrets]{asm})
 	assert.NoError(t, err)
 
 	var gotSecret []byte
@@ -102,7 +103,8 @@ func TestASMWorkflow(t *testing.T) {
 func TestASMPagination(t *testing.T) {
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	asm, leader, _, _ := localstack(ctx, t)
-	manager, err := New(ctx, asm, []Provider[Secrets]{asm})
+	sr := NewDBSecretResolver(&mockDBSecretResolverDAL{})
+	manager, err := New(ctx, sr, []Provider[Secrets]{asm})
 	assert.NoError(t, err)
 
 	// Create 210 secrets, so we paginate at least twice.
@@ -341,7 +343,11 @@ func (c *fakeAdminClient) ConfigUnset(ctx context.Context, req *connect.Request[
 }
 
 func (c *fakeAdminClient) SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) {
-	listing, err := c.asm.List(ctx)
+	client, err := c.asm.coordinator.Get()
+	if err != nil {
+		return nil, err
+	}
+	listing, err := client.list(ctx)
 	if err != nil {
 		return nil, err
 	}

common/configuration/dal/dal.go

@@ -3,6 +3,7 @@ package dal
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/alecthomas/types/optional"
 	"github.com/jackc/pgx/v5/pgxpool"
@@ -45,3 +46,44 @@ func (d *DAL) ListModuleConfiguration(ctx context.Context) ([]sql.ModuleConfigur
 	}
 	return l, nil
 }
+
+func (d *DAL) GetModuleSecretURL(ctx context.Context, module optional.Option[string], name string) (string, error) {
+	b, err := d.db.GetModuleSecretURL(ctx, module, name)
+	if err != nil {
+		return "", fmt.Errorf("could not get secret URL: %w", dalerrs.TranslatePGError(err))
+	}
+	return b, nil
+}
+
+func (d *DAL) SetModuleSecretURL(ctx context.Context, module optional.Option[string], name string, url string) error {
+	err := d.db.SetModuleSecretURL(ctx, module, name, url)
+	if err != nil {
+		return fmt.Errorf("could not set secret URL: %w", dalerrs.TranslatePGError(err))
+	}
+	return nil
+}
+
+func (d *DAL) UnsetModuleSecret(ctx context.Context, module optional.Option[string], name string) error {
+	err := d.db.UnsetModuleSecret(ctx, module, name)
+	if err != nil {
+		return fmt.Errorf("could not unset secret: %w", dalerrs.TranslatePGError(err))
+	}
+	return nil
+}
+
+type ModuleSecret sql.ModuleSecret
+
+func (d *DAL) ListModuleSecrets(ctx context.Context) ([]ModuleSecret, error) {
+	l, err := d.db.ListModuleSecrets(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("could not list secrets: %w", dalerrs.TranslatePGError(err))
+	}
+
+	// Convert []sql.ModuleSecret to []ModuleSecret
+	ms := make([]ModuleSecret, len(l))
+	for i, secret := range l {
+		ms[i] = ModuleSecret(secret)
+	}
+
+	return ms, nil
+}

common/configuration/db_secret_resolver.go

@@ -0,0 +1,81 @@
+package configuration
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/TBD54566975/ftl/common/configuration/dal"
+	"github.com/alecthomas/types/optional"
+)
+
+// DBSecretResolver loads values a project's secrets from the given database.
+type DBSecretResolver struct {
+	dal DBSecretResolverDAL
+}
+
+type DBSecretResolverDAL interface {
+	GetModuleSecretURL(ctx context.Context, module optional.Option[string], name string) (string, error)
+	ListModuleSecrets(ctx context.Context) ([]dal.ModuleSecret, error)
+	SetModuleSecretURL(ctx context.Context, module optional.Option[string], name string, url string) error
+	UnsetModuleSecret(ctx context.Context, module optional.Option[string], name string) error
+}
+
+// DBSecretResolver should only be used for secrets
+var _ Router[Secrets] = DBSecretResolver{}
+
+func NewDBSecretResolver(db DBSecretResolverDAL) DBSecretResolver {
+	return DBSecretResolver{dal: db}
+}
+
+func (d DBSecretResolver) Role() Secrets { return Secrets{} }
+
+func (d DBSecretResolver) Get(ctx context.Context, ref Ref) (*url.URL, error) {
+	u, err := d.dal.GetModuleSecretURL(ctx, ref.Module, ref.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get secret URL: %w", err)
+	}
+	url, err := url.Parse(u)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse secret URL: %w", err)
+	}
+	return url, nil
+}
+
+func (d DBSecretResolver) List(ctx context.Context) ([]Entry, error) {
+	secrets, err := d.dal.ListModuleSecrets(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("could not list module secrets: %w", err)
+	}
+	entries := make([]Entry, len(secrets))
+	for i, s := range secrets {
+		url, err := url.Parse(s.Url)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse secret URL: %w", err)
+		}
+		entries[i] = Entry{
+			Ref: Ref{
+				Module: s.Module,
+				Name:   s.Name,
+			},
+			Accessor: url,
+		}
+	}
+	return entries, nil
+}
+
+func (d DBSecretResolver) Set(ctx context.Context, ref Ref, key *url.URL) error {
+	err := d.dal.SetModuleSecretURL(ctx, ref.Module, ref.Name, key.String())
+	if err != nil {
+		return fmt.Errorf("failed to set secret URL: %w", err)
+	}
+	return nil
+}
+
+func (d DBSecretResolver) Unset(ctx context.Context, ref Ref) error {
+	err := d.dal.UnsetModuleSecret(ctx, ref.Module, ref.Name)
+	if err != nil {
+		return fmt.Errorf("failed to unset secret: %w", err)
+	}
+	return nil
+}

common/configuration/db_secret_resolver_test.go

@@ -0,0 +1,82 @@
+package configuration
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"testing"
+
+	"github.com/TBD54566975/ftl/common/configuration/dal"
+	"github.com/alecthomas/assert/v2"
+	. "github.com/alecthomas/types/optional"
+)
+
+type mockDBSecretResolverDAL struct {
+	entries []dal.ModuleSecret
+}
+
+func (d *mockDBSecretResolverDAL) findEntry(module Option[string], name string) (Option[dal.ModuleSecret], int) {
+	for i := range d.entries {
+		if d.entries[i].Module.Default("") == module.Default("") && d.entries[i].Name == name {
+			return Some(d.entries[i]), i
+		}
+	}
+	return None[dal.ModuleSecret](), -1
+}
+
+func (d *mockDBSecretResolverDAL) GetModuleSecretURL(ctx context.Context, module Option[string], name string) (string, error) {
+	entry, _ := d.findEntry(module, name)
+	if e, ok := entry.Get(); ok {
+		return e.Url, nil
+	}
+	return "", fmt.Errorf("secret not found")
+}
+
+func (d *mockDBSecretResolverDAL) ListModuleSecrets(ctx context.Context) ([]dal.ModuleSecret, error) {
+	return d.entries, nil
+}
+
+func (d *mockDBSecretResolverDAL) SetModuleSecretURL(ctx context.Context, module Option[string], name string, url string) error {
+	err := d.UnsetModuleSecret(ctx, module, name)
+	if err != nil {
+		return fmt.Errorf("could not unset secret %w", err)
+	}
+	d.entries = append(d.entries, dal.ModuleSecret{Module: module, Name: name, Url: url})
+	return nil
+}
+
+func (d *mockDBSecretResolverDAL) UnsetModuleSecret(ctx context.Context, module Option[string], name string) error {
+	entry, i := d.findEntry(module, name)
+	if _, ok := entry.Get(); ok {
+		d.entries = append(d.entries[:i], d.entries[i+1:]...)
+	}
+	return nil
+}
+
+func TestDBSecretResolverList(t *testing.T) {
+	ctx := context.Background()
+	resolver := NewDBSecretResolver(&mockDBSecretResolverDAL{})
+
+	rone := Ref{Module: Some("foo"), Name: "one"}
+	err := resolver.Set(ctx, rone, &url.URL{Scheme: "asm", Host: rone.String()})
+	assert.NoError(t, err)
+
+	rtwo := Ref{Module: Some("foo"), Name: "two"}
+	err = resolver.Set(ctx, rtwo, &url.URL{Scheme: "asm", Host: rtwo.String()})
+	assert.NoError(t, err)
+
+	entries, err := resolver.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, len(entries), 2)
+
+	err = resolver.Unset(ctx, rone)
+	assert.NoError(t, err)
+
+	entries, err = resolver.List(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, len(entries), 1)
+
+	url, err := resolver.Get(ctx, rtwo)
+	assert.NoError(t, err)
+	assert.Equal(t, url.String(), "asm://foo.two")
+}