diff --git a/common/configuration/asm_leader.go b/common/configuration/asm_leader.go index 141b47a8b..286560ce0 100644 --- a/common/configuration/asm_leader.go +++ b/common/configuration/asm_leader.go @@ -123,7 +123,7 @@ func (l *asmLeader) sync(ctx context.Context, secrets *xsync.MapOf[Ref, cachedSe if s.SecretBinary != nil { return fmt.Errorf("secret for %s in ASM is not a string", ref) } - data := []byte(*s.SecretString) + data := unwrapComments([]byte(*s.SecretString)) secrets.Store(ref, cachedSecret{ value: data, versionToken: optional.Some[any](refsToLoad[ref]), @@ -155,9 +155,10 @@ func (l *asmLeader) load(ctx context.Context, ref Ref, key *url.URL) ([]byte, er // store and if the secret already exists, update it. func (l *asmLeader) store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + valueWithComments := aws.String(string(wrapWithComments(value, defaultSecretModificationWarning))) _, err := l.client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ Name: aws.String(ref.String()), - SecretString: aws.String(string(value)), + SecretString: valueWithComments, Tags: []types.Tag{ {Key: aws.String(asmTagKey), Value: aws.String(ref.Module.Default(""))}, }, @@ -168,7 +169,7 @@ func (l *asmLeader) store(ctx context.Context, ref Ref, value []byte) (*url.URL, if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceExistsException" { _, err = l.client.UpdateSecret(ctx, &secretsmanager.UpdateSecretInput{ SecretId: aws.String(ref.String()), - SecretString: aws.String(string(value)), + SecretString: valueWithComments, }) if err != nil { return nil, fmt.Errorf("unable to update secret in ASM: %w", err) diff --git a/common/configuration/comments.go b/common/configuration/comments.go new file mode 100644 index 000000000..0fe163865 --- /dev/null +++ b/common/configuration/comments.go @@ -0,0 +1,32 @@ +package configuration + +import ( + "strings" + + "github.com/TBD54566975/ftl/internal/slices" +) + +const defaultSecretModificationWarning = `This secret is managed by "ftl secret set", DO NOT MODIFY` + +// wrapWithComments wraps the secret with a comment to indicate that it is managed by FTL. +// +// This is used by providers that want to include a warning to avoid manual modification. +// The provider must support multiline secrets. +// Comment lines are prefixed with '# ' in the result. +func wrapWithComments(secret []byte, comments string) []byte { + lines := []string{} + for _, line := range strings.Split(comments, "\n") { + lines = append(lines, "# "+line) + } + lines = append(lines, string(secret)) + return []byte(strings.Join(lines, "\n")) +} + +// unwrapComments removes comments if they exist by looking for the lines starting with '#' +func unwrapComments(secret []byte) []byte { + lines := strings.Split(string(secret), "\n") + lines = slices.Filter(lines, func(line string) bool { + return !strings.HasPrefix(line, "#") + }) + return []byte(strings.Join(lines, "\n")) +} diff --git a/common/configuration/comments_test.go b/common/configuration/comments_test.go new file mode 100644 index 000000000..384780e50 --- /dev/null +++ b/common/configuration/comments_test.go @@ -0,0 +1,34 @@ +package configuration + +import ( + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestComments(t *testing.T) { + for _, tt := range []struct { + input string + comment string + output string + }{ + { + input: "test input can be anything", + comment: "This is a test", + output: "# This is a test\ntest input can be anything", + }, + { + input: "{\n \"key\": \"value\"\n}", + comment: "This is a multi\nline\ncomment", + output: "# This is a multi\n# line\n# comment\n{\n \"key\": \"value\"\n}", + }, + } { + t.Run(tt.input, func(t *testing.T) { + wrapped := wrapWithComments([]byte(tt.input), tt.comment) + assert.Equal(t, tt.output, string(wrapped)) + + unwrapped := unwrapComments(wrapped) + assert.Equal(t, tt.input, string(unwrapped)) + }) + } +} diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 60ca5414c..7490581f7 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -21,6 +21,12 @@ type Secrets struct{} func (Secrets) String() string { return "secrets" } +func (Secrets) obfuscator() Obfuscator { + return Obfuscator{ + key: []byte("obfuscatesecrets"), // 16 characters (AES-128), not meant to provide security + } +} + type Configuration struct{} func (Configuration) String() string { return "configuration" } @@ -28,8 +34,9 @@ func (Configuration) String() string { return "configuration" } // Manager is a high-level configuration manager that abstracts the details of // the Router and Provider interfaces. type Manager[R Role] struct { - providers map[string]Provider[R] - router Router[R] + providers map[string]Provider[R] + router Router[R] + obfuscator optional.Option[Obfuscator] } func ConfigFromEnvironment() []string { @@ -61,6 +68,9 @@ func New[R Role](ctx context.Context, router Router[R], providers []Provider[R]) for _, p := range providers { m.providers[p.Key()] = p } + if provider, ok := any(new(R)).(ObfuscatorProvider); ok { + m.obfuscator = optional.Some(provider.obfuscator()) + } m.router = router return m, nil } @@ -87,6 +97,12 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { if err != nil { return nil, fmt.Errorf("%s: %w", ref, err) } + if obfuscator, ok := m.obfuscator.Get(); ok { + data, err = obfuscator.Reveal(data) + if err != nil { + return nil, fmt.Errorf("could not reveal obfuscated value: %w", err) + } + } return data, nil } @@ -127,12 +143,23 @@ func (m *Manager[R]) SetJSON(ctx context.Context, pkey string, ref Ref, value js if err := checkJSON(value); err != nil { return fmt.Errorf("invalid value for %s, must be JSON: %w", m.router.Role(), err) } + var bytes []byte + if obfuscator, ok := m.obfuscator.Get(); ok { + var err error + bytes, err = obfuscator.Obfuscate(value) + if err != nil { + return fmt.Errorf("could not obfuscate: %w", err) + } + } else { + bytes = value + } + provider, ok := m.providers[pkey] if !ok { pkeys := strings.Join(m.availableProviderKeys(), ", ") return fmt.Errorf("no provider for key %q, specify one of: %s", pkey, pkeys) } - key, err := provider.Store(ctx, ref, value) + key, err := provider.Store(ctx, ref, bytes) if err != nil { return err } diff --git a/common/configuration/obfuscator.go b/common/configuration/obfuscator.go new file mode 100644 index 000000000..8d940131f --- /dev/null +++ b/common/configuration/obfuscator.go @@ -0,0 +1,69 @@ +package configuration + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "io" + "strings" +) + +type ObfuscatorProvider interface { + obfuscator() Obfuscator +} + +// Obfuscator hides and reveals a value, but does not provide real security +// instead the aim of this Obfuscator is to make values not easily human readable +// +// Obfuscation is done by XOR-ing the input with the AES key. Length of key must be 16, 24 or 32 bytes (corresponding to AES-128, AES-192 or AES-256 keys). +type Obfuscator struct { + key []byte +} + +// Obfuscate takes a value and returns an obfuscated value (encoded in base64) +func (o Obfuscator) Obfuscate(input []byte) ([]byte, error) { + block, err := aes.NewCipher(o.key) + if err != nil { + return nil, fmt.Errorf("could not create cypher for obfuscation: %w", err) + } + ciphertext := make([]byte, aes.BlockSize+len(input)) + iv := ciphertext[:aes.BlockSize] + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return nil, fmt.Errorf("could not generate IV for obfuscation: %w", err) + } + cfb := cipher.NewCFBEncrypter(block, iv) + cfb.XORKeyStream(ciphertext[aes.BlockSize:], input) + return []byte(base64.StdEncoding.EncodeToString(ciphertext)), nil +} + +// Reveal takes an obfuscated value and de-obfuscates the base64 encoded value +func (o Obfuscator) Reveal(input []byte) ([]byte, error) { + // check if the input looks like it was obfuscated + if !strings.ContainsRune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/=", rune(input[0])) { + // known issue: an unobfuscated value which is just a number will be considered obfuscated + return input, nil + } + + obfuscated, err := base64.StdEncoding.DecodeString(string(input)) + if err != nil { + return nil, fmt.Errorf("expected hexadecimal string: %w", err) + } + block, err := aes.NewCipher(o.key) + if err != nil { + return nil, fmt.Errorf("could not create cypher for decoding obfuscation: %w", err) + } + if len(obfuscated) < aes.BlockSize { + return nil, errors.New("obfuscated value too short to decode") + } + iv := obfuscated[:aes.BlockSize] + obfuscated = obfuscated[aes.BlockSize:] + cfb := cipher.NewCFBDecrypter(block, iv) + + var output = make([]byte, len(obfuscated)) + cfb.XORKeyStream(output, obfuscated) + + return output, nil +} diff --git a/common/configuration/obfuscator_test.go b/common/configuration/obfuscator_test.go new file mode 100644 index 000000000..7d33584a0 --- /dev/null +++ b/common/configuration/obfuscator_test.go @@ -0,0 +1,72 @@ +package configuration + +import ( + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/alecthomas/types/optional" +) + +func TestObfuscator(t *testing.T) { + defaultKey := []byte("1234567890123456") // 32 characters + for _, tt := range []struct { + input string + key []byte + expectedError optional.Option[string] + backwardsCompatible bool + }{ + { + input: "test input can be anything", + key: defaultKey, + backwardsCompatible: false, + }, + { + input: `"test input can be anything"`, + key: defaultKey, + backwardsCompatible: true, + }, + { + input: `"{\n "key": "value"\n}`, + key: defaultKey, + backwardsCompatible: true, + }, + { + input: `1.2345`, + key: defaultKey, + backwardsCompatible: false, + }, + { + input: "key is too short", + key: []byte("too-short"), + expectedError: optional.Some("could not create cypher for obfuscation: crypto/aes: invalid key size 9"), + }, + } { + t.Run(tt.input, func(t *testing.T) { + o := Obfuscator{ + key: tt.key, + } + // obfuscate + obfuscated, err := o.Obfuscate([]byte(tt.input)) + if expectedError, ok := tt.expectedError.Get(); ok { + assert.EqualError(t, err, expectedError) + return + } + assert.NoError(t, err) + + // reveal obfuscated value + revealed, err := o.Reveal(obfuscated) + assert.NoError(t, err) + assert.Equal(t, tt.input, string(revealed)) + + // obfuscated value should not include the input we are trying to obfuscate + assert.NotContains(t, string(obfuscated), tt.input) + + // reveal unobfuscated value to check backwards compatibility + if tt.backwardsCompatible { + revealed, err = o.Reveal([]byte(tt.input)) + assert.NoError(t, err) + assert.Equal(t, tt.input, string(revealed)) + } + }) + } +}