Ability to Use Postgres as a Backing Store for VSecM Safe (#1165)

Maintains feature parity with the former version.

Unit and integration tests pass.

I'm merging this. Will do follow-up PRs to address `TODO:` comments in the code, and also clean-up the code.

---

* wip

* add polling to main to test config setting for safe itself

* temporarily disable relay client

* testing persistence to postgres

* SQL statement change

* lastworking

* add entity.go

* mostly-working version

* fixed store response

* add guard clauses

* more debugging

* exception for initial persistence to db

* changes

app/safe/cmd/entity.go

@@ -0,0 +1,47 @@
+/*
+| Protect your secrets, protect your sensitive data.
+: Explore VMware Secrets Manager docs at https://vsecm.com/
+</
+<>/ keep your secrets... secret
+>/
+<>/' Copyright 2023-present VMware Secrets Manager contributors.
+>/' SPDX-License-Identifier: BSD-2-Clause
+*/
+
+package main
+
+// TODO: obviously there is a need for cleanup; once things start to work
+// as expected, move the codes to where they should belong.
+
+// TODO: move me to a proper place.
+
+// TODO: by design, VSecM Safe will not use more than one backing store
+// (create an ADR for that).
+// This means, there is a chicken-and-the-egg problem for persisting the
+// internal VSecM Safe configuration.
+//
+// For postgres backing store, VSecM Safe should keep its initial config
+// in memory until the database is there; and then it should save it to
+// the database, too.
+
+// TODO: we should check for the existence of the table in postgres and
+// log an error if it's not there.
+
+// TODO: when postgres mode vsecm safe shall be read-only (except for config update)
+// until it is initialized. once initialized, it should save its config to postgres too
+// and then it should be readwrite.
+
+// TODO: we need documentation for this postgres store feature. (and also a demo recording)
+
+// TODO: it's best block requests when the db is not ready yet (in postgres mode)
+// because otherwise, the initCommand will retry in exponential backoff and
+// eventually give up.
+// or the keystone secret will not be persisted although keystone will
+// be informed that safe is ready.
+
+type SafeConfig struct {
+ Config struct {
+ BackingStore string `json:"backingStore"`
+ DataSourceName string `json:"dataSourceName"`
+ } `json:"config"`
+}

app/safe/cmd/main.go

@@ -12,18 +12,48 @@ package main
 
 import (
  "context"
+ "encoding/json"
+ "time"
 
  "github.com/spiffe/go-spiffe/v2/workloadapi"
 
  "github.com/vmware-tanzu/secrets-manager/app/safe/internal/bootstrap"
  server "github.com/vmware-tanzu/secrets-manager/app/safe/internal/server/engine"
+ "github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/io"
+ "github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/secret/collection"
  "github.com/vmware-tanzu/secrets-manager/core/constants/env"
  "github.com/vmware-tanzu/secrets-manager/core/constants/key"
  "github.com/vmware-tanzu/secrets-manager/core/crypto"
+ entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+ cEnv "github.com/vmware-tanzu/secrets-manager/core/env"
  log "github.com/vmware-tanzu/secrets-manager/core/log/std"
  "github.com/vmware-tanzu/secrets-manager/core/probe"
 )
 
+func pollForConfig(ctx context.Context, id string) (*SafeConfig, error) {
+ for {
+ log.InfoLn(&id, "Polling for VSecM Safe internal configuration")
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ default:
+ vSecMSafeInternalConfig, err := collection.ReadSecret(id, "vsecm-safe")
+ if err != nil {
+ log.InfoLn(&id, "Failed to load VSecM Safe internal configuration", err.Error())
+ } else if vSecMSafeInternalConfig != nil && len(vSecMSafeInternalConfig.Values) > 0 {
+ var safeConfig SafeConfig
+ err := json.Unmarshal([]byte(vSecMSafeInternalConfig.Values[0]), &safeConfig)
+ if err != nil {
+ log.InfoLn(&id, "Failed to parse VSecM Safe internal configuration", err.Error())
+ } else {
+ return &safeConfig, nil
+ }
+ }
+ time.Sleep(5 * time.Second)
+ }
+ }
+}
+
 func main() {
  id := crypto.Id()
 
@@ -38,6 +68,28 @@ func main() {
  )
  defer cancel()
 
+ if cEnv.BackingStoreForSafe() == entity.Postgres {
+ go func() {
+ log.InfoLn(&id, "Backing store is postgres.")
+ log.InfoLn(&id, "VSecM Safe will remain read-only until the internal configuration is loaded.")
+
+ safeConfig, err := pollForConfig(ctx, id)
+ if err != nil {
+ log.FatalLn(&id, "Failed to retrieve VSecM Safe internal configuration", err.Error())
+ }
+
+ log.InfoLn(&id, "VSecM Safe internal configuration loaded. Initializing database.")
+
+ err = io.InitDB(safeConfig.Config.DataSourceName)
+ if err != nil {
+ log.FatalLn(&id, "Failed to initialize database:", err)
+ return
+ }
+
+ log.InfoLn(&id, "Database connection initialized.")
+ }()
+ }
+
  log.InfoLn(&id, "Acquiring identity...")
 
  // Channel to notify when the bootstrap timeout has been reached.

app/safe/internal/server/route/base/extract/extract.go

@@ -19,7 +19,7 @@ import (
  log "github.com/vmware-tanzu/secrets-manager/core/log/std"
 )
 
-// WorkloadIDAndParts extracts the workload identifier and its constituent parts
+// WorkloadIdAndParts extracts the workload identifier and its constituent parts
 // from a SPIFFE ID string, based on a predefined prefix that is removed from
 // the SPIFFE ID.
 //
@@ -32,7 +32,7 @@ import (
 // which is essentially the first part of the SPIFFE ID after removing the
 // prefix. The second return value is a slice of strings representing all
 // parts of the SPIFFE ID after the prefix removal.
-func WorkloadIDAndParts(spiffeid string) (string, []string) {
+func WorkloadIdAndParts(spiffeid string) (string, []string) {
  re := env.NameRegExpForWorkload()
  if re == "" {
  return "", nil

app/safe/internal/server/route/fetch/fetch.go

@@ -88,7 +88,7 @@ func Fetch(
 
  log.DebugLn(&cid, "Fetch: preparing request")
 
- workloadId, parts := extract.WorkloadIDAndParts(spiffeid)
+ workloadId, parts := extract.WorkloadIdAndParts(spiffeid)
  if len(parts) == 0 {
  handle.BadPeerSvidResponse(cid, w, spiffeid, j)
  return

app/safe/internal/state/io/disk.go

@@ -34,6 +34,10 @@ import (
 // channel allows the function to operate asynchronously, notifying the
 // caller of any issues in the process of persisting the secret.
 func PersistToDisk(secret entity.SecretStored, errChan chan<- error) {
+ if env.BackingStoreForSafe() != entity.File {
+ panic("Attempted to save to disk when backing store is not file")
+ }
+
  backupCount := env.SecretBackupCountForSafe()
 
  // Save the secret

app/safe/internal/state/io/postgres.go

@@ -0,0 +1,90 @@
+package io
+
+import (
+ "database/sql"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+
+ _ "github.com/lib/pq"
+
+ "github.com/vmware-tanzu/secrets-manager/core/crypto"
+ entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+ "github.com/vmware-tanzu/secrets-manager/core/env"
+ log "github.com/vmware-tanzu/secrets-manager/core/log/std"
+ "github.com/vmware-tanzu/secrets-manager/lib/backoff"
+)
+
+var db *sql.DB
+
+// InitDB initializes the database connection
+func InitDB(dataSourceName string) error {
+ var err error
+ db, err = sql.Open("postgres", dataSourceName)
+ if err != nil {
+ return err
+ }
+ return db.Ping()
+}
+
+// PersistToPostgres saves a given secret to the Postgres database
+func PersistToPostgres(secret entity.SecretStored, errChan chan<- error) {
+ cid := secret.Meta.CorrelationId
+
+ log.TraceLn(&cid, "PersistToPostgres: Persisting secret to database")
+
+ // Serialize the secret to JSON
+ jsonData, err := json.Marshal(secret)
+ if err != nil {
+ errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to marshal secret"))
+ log.ErrorLn(&cid, "PersistToPostgres: Error marshaling secret:", err.Error())
+ return
+ }
+
+ // Encrypt the JSON data
+ var encryptedData string
+ fipsMode := env.FipsCompliantModeForSafe()
+
+ if fipsMode {
+ encryptedBytes, err := crypto.EncryptBytesAes(jsonData)
+ if err != nil {
+ errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to encrypt secret with AES"))
+ log.ErrorLn(&cid, "PersistToPostgres: Error encrypting secret with AES:", err.Error())
+ return
+ }
+ encryptedData = base64.StdEncoding.EncodeToString(encryptedBytes)
+ } else {
+ encryptedBytes, err := crypto.EncryptBytesAge(jsonData)
+ if err != nil {
+ errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to encrypt secret with Age"))
+ log.ErrorLn(&cid, "PersistToPostgres: Error encrypting secret with Age:", err.Error())
+ return
+ }
+ encryptedData = base64.StdEncoding.EncodeToString(encryptedBytes)
+ }
+
+ err = backoff.RetryExponential("PersistToPostgres", func() error {
+ if db == nil {
+ if secret.Name == "vsecm-safe" {
+ // TODO: implement me.
+ log.InfoLn(&cid, "PersistToPostgres: vsecm-safe secret will be persisted after db connection is initialized")
+ return nil
+ }
+ return errors.New("PersistToPostgres: Database connection is nil")
+ }
+
+ // TODO: get table name from env var.
+ _, err := db.Exec(
+ `INSERT INTO "vsecm-secrets" (name, data) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET data = $2`,
+ secret.Name, encryptedData)
+ return err
+ })
+
+ if err != nil {
+ errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to persist secret to database"))
+ log.ErrorLn(&cid, "PersistToPostgres: Error persisting secret to database:", err.Error())
+ return
+ }
+
+ log.TraceLn(&cid, "PersistToPostgres: Secret persisted to database successfully")
+}

app/safe/internal/state/io/read.go

@@ -13,6 +13,7 @@ package io
 import (
  "encoding/json"
  "errors"
+ "github.com/vmware-tanzu/secrets-manager/core/env"
 
  "github.com/vmware-tanzu/secrets-manager/core/crypto"
  entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
@@ -36,6 +37,10 @@ import (
 // returned. The error provides context about the nature of the failure,
 // such as issues with decryption or data deserialization.
 func ReadFromDisk(key string) (*entity.SecretStored, error) {
+ if env.BackingStoreForSafe() != entity.File {
+ panic("Attempted to read from disk when backing store is not file")
+ }
+
  contents, err := crypto.DecryptDataFromDisk(key)
  if err != nil {
  return nil, errors.Join(

app/safe/internal/state/secret/collection/populate.go

@@ -72,6 +72,9 @@ func PopulateSecrets(cid string) error {
  if err != nil {
  log.ErrorLn(&cid, "populateSecrets:error", err.Error())
  }
+ case data.Postgres:
+ // TODO: implement me.
+ log.WarnLn(&cid, "populateSecrets: postgres initial secrets population is not implemented yet.")
  case data.Kubernetes:
  panic("implement kubernetes store")
  case data.AwsSecretStore:

app/safe/internal/state/secret/collection/read.go

@@ -15,6 +15,7 @@ import (
  "github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/stats"
  "github.com/vmware-tanzu/secrets-manager/core/crypto"
  entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+ "github.com/vmware-tanzu/secrets-manager/core/env"
  log "github.com/vmware-tanzu/secrets-manager/core/log/std"
  data "github.com/vmware-tanzu/secrets-manager/lib/entity"
 )
@@ -199,6 +200,16 @@ func ReadSecret(cid string, key string) (*entity.SecretStored, error) {
  return &s, nil
  }
 
+ store := env.BackingStoreForSafe()
+
+ switch store {
+ case entity.File:
+ log.TraceLn(&cid, "will read from file store.")
+ case entity.Postgres:
+ log.WarnLn(&cid, "TODO: fetch from postgres store")
+ return nil, nil
+ }
+
  stored, err := io.ReadFromDisk(key)
 
  if err != nil {

app/safe/internal/state/secret/queue/deletion/disk.go

@@ -69,7 +69,7 @@ func ProcessSecretBackingStoreQueue() {
  log.TraceLn(&cid, "ProcessSecretQueue: using in-memory store.")
  return
  case entity.File:
- log.TraceLn(&cid, "ProcessSecretQueue: Will persist to disk.")
+ log.TraceLn(&cid, "ProcessSecretQueue: Will delete secret from disk.")
  case entity.Kubernetes:
  panic("implement kubernetes store")
  case entity.AwsSecretStore:
@@ -78,6 +78,9 @@ func ProcessSecretBackingStoreQueue() {
  panic("implement azure secret store")
  case entity.GcpSecretStore:
  panic("implement gcp secret store")
+ case entity.Postgres:
+ log.WarnLn(&cid, "Delete operation has not been implemented for postgres backing store yet.")
+ return
  }
 
  if secret.Name == "" {

app/safe/internal/state/secret/queue/insertion/disk.go

@@ -73,8 +73,12 @@ func ProcessSecretBackingStoreQueue() {
  panic("implement azure secret store")
  case entity.GcpSecretStore:
  panic("implement gcp secret store")
+ case entity.Postgres:
+ log.TraceLn(&cid, "ProcessSecretQueue: Will persist to Postgres.")
  }
 
+ // TODO: will definitely need cleanup.
+
  // Get a secret to be persisted to the disk.
  secret := <-SecretUpsertQueue
 
@@ -93,7 +97,14 @@ func ProcessSecretBackingStoreQueue() {
  //
  // Do not call this function elsewhere.
  // It is meant to be called inside this `processSecretQueue` goroutine.
- io.PersistToDisk(secret, errChan)
+ if store == entity.Postgres {
+
+ // TODO: for debugging; delete values before merging.
+ log.TraceLn(&cid, "Persisting to Postgres.", secret.Name)
+ io.PersistToPostgres(secret, errChan)
+ } else {
+ io.PersistToDisk(secret, errChan)
+ }
 
  log.TraceLn(&cid,
  "processSecretQueue: should have persisted the secret.")

core/constants/env/env.go

@@ -37,6 +37,7 @@ const VSecMRootKeyInputModeManual VarName = "VSECM_ROOT_KEY_INPUT_MODE_MANUAL"
 const VSecMRootKeyName VarName = "VSECM_ROOT_KEY_NAME"
 const VSecMRootKeyPath VarName = "VSECM_ROOT_KEY_PATH"
 const VSecMSafeBackingStore VarName = "VSECM_SAFE_BACKING_STORE"
+const VSecMSafePostgresDataSourceName VarName = "VSECM_SAFE_POSTGRES_DATASOURCE_NAME"
 const VSecMSafeBootstrapTimeout VarName = "VSECM_SAFE_BOOTSTRAP_TIMEOUT"
 const VSecMSafeDataPath VarName = "VSECM_SAFE_DATA_PATH"
 const VSecMSafeEndpointUrl VarName = "VSECM_SAFE_ENDPOINT_URL"
@@ -125,6 +126,7 @@ const VSecMSpiffeIdPrefixSafeDefault VarValue = "^spiffe://vsecm.com/workload/vs
 const VSecMSpiffeIdPrefixSentinelDefault VarValue = "^spiffe://vsecm.com/workload/vsecm-sentinel/ns/vsecm-system/sa/vsecm-sentinel/n/[^/]+$"
 const VSecMSpiffeIdPrefixWorkloadDefault VarValue = "^spiffe://vsecm.com/workload/[^/]+/ns/[^/]+/sa/[^/]+/n/[^/]+$"
 const VSecMNameRegExpForWorkloadDefault VarValue = "^spiffe://vsecm.com/workload/([^/]+)/ns/[^/]+/sa/[^/]+/n/[^/]+$"
+const VSecMSafePostgresDataSourceNameDefault VarValue = "user=postgres dbname=postgres sslmode=disable"
 
 const VSecMRelayServerUrlDefault VarValue = "https://vsecm-relay.vsecm-system.svc.cluster.local:443/"