[DERCBOT-1159] Immediate deletion of secrets in the secret manager (#…

…1762)
theopenconversationkit · Nov 20, 2024 · 14cd2fc · 14cd2fc
1 parent e2b4134
commit 14cd2fc
Show file tree

Hide file tree

Showing 13 changed files with 122 additions and 24 deletions.
diff --git a/bot/admin/server/src/main/kotlin/BotAdminService.kt b/bot/admin/server/src/main/kotlin/BotAdminService.kt
@@ -38,6 +38,8 @@ import ai.tock.bot.admin.dialog.*
 import ai.tock.bot.admin.kotlin.compiler.KotlinFile
 import ai.tock.bot.admin.kotlin.compiler.client.KotlinCompilerClient
 import ai.tock.bot.admin.model.*
+import ai.tock.bot.admin.service.ObservabilityService
+import ai.tock.bot.admin.service.RAGService
 import ai.tock.bot.admin.story.*
 import ai.tock.bot.admin.story.dump.*
 import ai.tock.bot.admin.user.UserReportDAO
@@ -49,6 +51,8 @@ import ai.tock.bot.engine.dialog.DialogFlowDAO
 import ai.tock.bot.engine.feature.FeatureDAO
 import ai.tock.bot.engine.feature.FeatureState
 import ai.tock.bot.engine.user.PlayerType
+import ai.tock.genai.orchestratorcore.models.observability.LangfuseObservabilitySetting
+import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import ai.tock.nlp.admin.AdminService
 import ai.tock.nlp.core.Intent
 import ai.tock.nlp.front.client.FrontClient
@@ -58,6 +62,8 @@ import ai.tock.nlp.front.shared.config.ClassifiedSentenceStatus.model
 import ai.tock.nlp.front.shared.config.ClassifiedSentenceStatus.validated
 import ai.tock.shared.*
 import ai.tock.shared.security.UserLogin
+import ai.tock.shared.security.key.HasSecretKey
+import ai.tock.shared.security.key.SecretKey
 import ai.tock.shared.vertx.WebVerticle.Companion.badRequest
 import ai.tock.translator.*
 import com.github.salomonbrys.kodein.instance
@@ -1141,23 +1147,30 @@ object BotAdminService {
         }
 
         // delete the RAG configuration
-        ragConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let {
-            ragConfigurationDAO.delete(it._id)
+        ragConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let { config ->
+            ragConfigurationDAO.delete(config._id)
+            config.llmSetting.apiKey?.let { SecurityUtils.deleteSecret(it) }
+            config.emSetting.apiKey?.let { SecurityUtils.deleteSecret(it) }
         }
 
         // delete the Sentence Generation configuration
-        sentenceGenerationConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let {
-            sentenceGenerationConfigurationDAO.delete(it._id)
+        sentenceGenerationConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let { config ->
+            sentenceGenerationConfigurationDAO.delete(config._id)
+            config.llmSetting.apiKey?.let { SecurityUtils.deleteSecret(it) }
         }
 
         // delete the Observability configuration
-        observabilityConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let {
-            observabilityConfigurationDAO.delete(it._id)
+        observabilityConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let { config ->
+            observabilityConfigurationDAO.delete(config._id)
+            (config.setting as? HasSecretKey<SecretKey>)?.secretKey?.let { secret ->
+                SecurityUtils.deleteSecret(secret)
+            }
         }
 
         // delete the Vector Store configuration
-        vectorStoreConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let {
-            vectorStoreConfigurationDAO.delete(it._id)
+        vectorStoreConfigurationDAO.findByNamespaceAndBotId(app.namespace, app.name)?.let { config ->
+            vectorStoreConfigurationDAO.delete(config._id)
+            SecurityUtils.deleteSecret(config.setting.password)
         }
     }
 

diff --git a/bot/admin/server/src/main/kotlin/service/ObservabilityService.kt b/bot/admin/server/src/main/kotlin/service/ObservabilityService.kt
@@ -20,6 +20,8 @@ import ai.tock.bot.admin.BotAdminService
 import ai.tock.bot.admin.bot.observability.BotObservabilityConfiguration
 import ai.tock.bot.admin.bot.observability.BotObservabilityConfigurationDAO
 import ai.tock.bot.admin.model.BotObservabilityConfigurationDTO
+import ai.tock.genai.orchestratorcore.models.observability.LangfuseObservabilitySetting
+import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import ai.tock.shared.exception.rest.BadRequestException
 import ai.tock.shared.injector
 import ai.tock.shared.provide
@@ -62,8 +64,15 @@ object ObservabilityService {
     fun deleteConfig(namespace: String, botId: String) {
         val observabilityConfig = observabilityConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
             ?: WebVerticle.badRequest("No Observability configuration is defined yet [namespace: $namespace, botId: $botId]")
+
         logger.info { "Deleting the Observability Configuration [namespace: $namespace, botId: $botId]" }
-        return observabilityConfigurationDAO.delete(observabilityConfig._id)
+        observabilityConfigurationDAO.delete(observabilityConfig._id)
+
+        val setting = observabilityConfig.setting
+        setting.takeIf { it is LangfuseObservabilitySetting }?.let {
+            logger.info { "Deleting the Observability secret ..." }
+            SecurityUtils.deleteSecret((setting as LangfuseObservabilitySetting).secretKey)
+        } ?: logger.info { "No secret to delete for the current setting." }
     }
 
     /**

diff --git a/bot/admin/server/src/main/kotlin/service/RAGService.kt b/bot/admin/server/src/main/kotlin/service/RAGService.kt
@@ -23,6 +23,7 @@ import ai.tock.bot.admin.model.BotRAGConfigurationDTO
 import ai.tock.bot.admin.story.StoryDefinitionConfiguration
 import ai.tock.bot.admin.story.StoryDefinitionConfigurationDAO
 import ai.tock.bot.admin.story.StoryDefinitionConfigurationFeature
+import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import ai.tock.nlp.core.Intent
 import ai.tock.shared.exception.rest.BadRequestException
 import ai.tock.shared.injector
@@ -59,8 +60,14 @@ object RAGService {
     fun deleteConfig(namespace: String, botId: String) {
         val ragConfig = ragConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
             ?: WebVerticle.badRequest("No RAG configuration is defined yet [namespace: $namespace, botId: $botId]")
+
         logger.info { "Deleting the RAG Configuration [namespace: $namespace, botId: $botId]" }
-        return ragConfigurationDAO.delete(ragConfig._id)
+        ragConfigurationDAO.delete(ragConfig._id)
+
+        logger.info { "Deleting the LLM secret ..." }
+        ragConfig.llmSetting.apiKey?.let { SecurityUtils.deleteSecret(it) }
+        logger.info { "Deleting the Embedding secret ..." }
+        ragConfig.emSetting.apiKey?.let { SecurityUtils.deleteSecret(it) }
     }
 
     /**

diff --git a/bot/admin/server/src/main/kotlin/service/SentenceGenerationService.kt b/bot/admin/server/src/main/kotlin/service/SentenceGenerationService.kt
@@ -17,9 +17,10 @@
 package ai.tock.bot.admin.service
 
 import ai.tock.bot.admin.BotAdminService
-import ai.tock.bot.admin.bot.sentencegeneration.BotSentenceGenerationConfigurationDAO
 import ai.tock.bot.admin.bot.sentencegeneration.BotSentenceGenerationConfiguration
+import ai.tock.bot.admin.bot.sentencegeneration.BotSentenceGenerationConfigurationDAO
 import ai.tock.bot.admin.model.BotSentenceGenerationConfigurationDTO
+import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import ai.tock.shared.exception.rest.BadRequestException
 import ai.tock.shared.injector
 import ai.tock.shared.provide
@@ -52,8 +53,12 @@ object SentenceGenerationService {
     fun deleteConfig(namespace: String, botId: String) {
         val sentenceGenerationConfiguration = sentenceGenerationConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
             ?: WebVerticle.badRequest("No LLM Sentence Generation configuration is defined yet [namespace: $namespace, botId: $botId]")
-        logger.info { "Deleting the LLM Sentence Generation Configuration [namespace: $namespace, botId: $botId]" }
-        return sentenceGenerationConfigurationDAO.delete(sentenceGenerationConfiguration._id)
+
+        logger.info { "Deleting the LLM Sentence Generation Configuration [namespace: $namespace, botId: $botId] ..." }
+        sentenceGenerationConfigurationDAO.delete(sentenceGenerationConfiguration._id)
+
+        logger.info { "Deleting the LLM secret ..." }
+        sentenceGenerationConfiguration.llmSetting.apiKey?.let { SecurityUtils.deleteSecret(it) }
     }
 
     /**

diff --git a/bot/admin/server/src/main/kotlin/service/VectorStoreService.kt b/bot/admin/server/src/main/kotlin/service/VectorStoreService.kt
@@ -20,6 +20,7 @@ import ai.tock.bot.admin.BotAdminService
 import ai.tock.bot.admin.bot.vectorstore.BotVectorStoreConfiguration
 import ai.tock.bot.admin.bot.vectorstore.BotVectorStoreConfigurationDAO
 import ai.tock.bot.admin.model.BotVectorStoreConfigurationDTO
+import ai.tock.genai.orchestratorcore.utils.SecurityUtils
 import ai.tock.shared.exception.rest.BadRequestException
 import ai.tock.shared.injector
 import ai.tock.shared.provide
@@ -63,8 +64,12 @@ object VectorStoreService {
     fun deleteConfig(namespace: String, botId: String) {
         val vectorStoreConfig = vectorStoreConfigurationDAO.findByNamespaceAndBotId(namespace, botId)
             ?: WebVerticle.badRequest("No Vector Store configuration is defined yet [namespace: $namespace, botId: $botId]")
+
         logger.info { "Deleting the Vector Store Configuration [namespace: $namespace, botId: $botId]" }
-        return vectorStoreConfigurationDAO.delete(vectorStoreConfig._id)
+        vectorStoreConfigurationDAO.delete(vectorStoreConfig._id)
+
+        logger.info { "Deleting the database secret ..." }
+        SecurityUtils.deleteSecret(vectorStoreConfig.setting.password)
     }
 
     /**

diff --git a/...otlin/ai/tock/genai/orchestratorcore/models/observability/LangfuseObservabilitySetting.kt b/...otlin/ai/tock/genai/orchestratorcore/models/observability/LangfuseObservabilitySetting.kt
@@ -16,8 +16,10 @@
 
 package ai.tock.genai.orchestratorcore.models.observability
 
+import ai.tock.shared.security.key.HasSecretKey
+
 data class LangfuseObservabilitySetting<T>(
-    val secretKey: T,
+    override val secretKey: T,
     val publicKey: String,
     val url: String
-) : ObservabilitySettingBase<T>(ObservabilityProvider.Langfuse)
+) : ObservabilitySettingBase<T>(ObservabilityProvider.Langfuse), HasSecretKey<T>
diff --git a/...c/main/kotlin/ai/tock/genai/orchestratorcore/models/vectorstore/VectorStoreSettingBase.kt b/...c/main/kotlin/ai/tock/genai/orchestratorcore/models/vectorstore/VectorStoreSettingBase.kt
@@ -17,7 +17,6 @@
 package ai.tock.genai.orchestratorcore.models.vectorstore
 
 
-import ai.tock.genai.orchestratorcore.mappers.LLMSettingMapper
 import ai.tock.genai.orchestratorcore.mappers.VectorStoreSettingMapper
 import ai.tock.genai.orchestratorcore.models.Constants
 import ai.tock.shared.security.key.SecretKey

diff --git a/...i/orchestrator-core/src/main/kotlin/ai/tock/genai/orchestratorcore/utils/SecurityUtils.kt b/...i/orchestrator-core/src/main/kotlin/ai/tock/genai/orchestratorcore/utils/SecurityUtils.kt
@@ -18,11 +18,11 @@ package ai.tock.genai.orchestratorcore.utils
 
 import ai.tock.shared.injector
 import ai.tock.shared.provide
-import ai.tock.shared.security.SecretManagerProviderType
 import ai.tock.shared.security.SecretManagerService
-import ai.tock.shared.security.credentials.AIProviderSecret
 import ai.tock.shared.security.genAISecretManagerProvider
-import ai.tock.shared.security.key.*
+import ai.tock.shared.security.key.NamedSecretKey
+import ai.tock.shared.security.key.RawSecretKey
+import ai.tock.shared.security.key.SecretKey
 import mu.KLogger
 import mu.KotlinLogging
 
@@ -83,5 +83,31 @@ object SecurityUtils {
             secretMangerService.createOrUpdateSecretKey(namespace, botId, feature, secretValue)
         } ?: RawSecretKey(secretValue)
 
+    /**
+     * Delete a secret
+     * @param secret the secret key
+     */
+    fun deleteSecret(secret: SecretKey) {
+        try {
+            // If the secret is a raw value, there's nothing to be done
+            if(secret is RawSecretKey) return
+
+            // Check SecretManagerProvider if it is defined
+            if (genAISecretManagerProvider == null) {
+                throw IllegalArgumentException("No Gen AI secret manager provider has been defined to delete the secret. Type=${secret.type}")
+            }
+
+            // Check whether the SecretManagerProvider supports secret
+            if(secretMangerService.isSecretTypeSupported(secret)) {
+                return secretMangerService.deleteSecret((secret as NamedSecretKey).secretName)
+            }else{
+                throw IllegalArgumentException("The secret manager provider type '${secret.type}' is not supported by " +
+                        "the instantiated service ${secretMangerService::class.simpleName}.")
+            }
+        } catch (e: Exception) {
+            logger.warn("The secret has not been deleted.", e)
+            // Do not block treatment if it fails.
+        }
+    }
 
 }
diff --git a/shared/src/main/kotlin/security/SecretManagerService.kt b/shared/src/main/kotlin/security/SecretManagerService.kt
@@ -20,7 +20,6 @@ import ai.tock.shared.property
 import ai.tock.shared.propertyOrNull
 import ai.tock.shared.security.credentials.AIProviderSecret
 import ai.tock.shared.security.credentials.Credentials
-import ai.tock.shared.security.key.AwsSecretKey
 import ai.tock.shared.security.key.SecretKey
 
 // The expected values correspond to the names of the SecretManagerProviderType elements
@@ -99,6 +98,11 @@ interface SecretManagerService {
      */
     fun isSecretTypeSupported(secret : SecretKey): Boolean
 
+    /**
+     * Delete a secret
+     */
+    fun deleteSecret(secretName: String)
+
 }
 
 enum class SecretManagerProviderType {

diff --git a/shared/src/main/kotlin/security/key/SecretKey.kt b/shared/src/main/kotlin/security/key/SecretKey.kt
@@ -37,4 +37,9 @@ abstract class SecretKey(
 
 interface NamedSecretKey {
     val secretName: String
-}
+}
+
+interface HasSecretKey<T>{
+    val secretKey: T
+}
+
diff --git a/util/aws-tools/src/main/kotlin/ai/tock/aws/secretmanager/AwsSecretManagerService.kt b/util/aws-tools/src/main/kotlin/ai/tock/aws/secretmanager/AwsSecretManagerService.kt
@@ -193,4 +193,17 @@ class AwsSecretManagerService : SecretManagerService {
     override fun createSecretKeyInstance(secretName: String) = AwsSecretKey(secretName)
 
     override fun isSecretTypeSupported(secret: SecretKey): Boolean = secret is AwsSecretKey
+
+    override fun deleteSecret(secretName: String) {
+        try {
+            val deleteRequest = DeleteSecretRequest()
+                .withSecretId(secretName)
+                .withForceDeleteWithoutRecovery(true)
+            secretsManagerClient.deleteSecret(deleteRequest)
+            logger.info { "The secret '$secretName' has been successfully deleted." }
+        } catch (e: Exception) {
+            logger.error(e) { "Failed to delete the secret '$secretName'." }
+            throw e
+        }
+    }
 }
diff --git a/util/env-tools/src/main/kotlin/ai/tock/env/secretmanager/EnvSecretManagerService.kt b/util/env-tools/src/main/kotlin/ai/tock/env/secretmanager/EnvSecretManagerService.kt
@@ -21,8 +21,6 @@ import ai.tock.shared.security.SecretManagerProviderType
 import ai.tock.shared.security.SecretManagerService
 import ai.tock.shared.security.credentials.AIProviderSecret
 import ai.tock.shared.security.credentials.Credentials
-import ai.tock.shared.security.genAISecretPrefix
-import ai.tock.shared.security.key.AwsSecretKey
 import ai.tock.shared.security.key.SecretKey
 import kotlinx.serialization.json.Json
 
@@ -56,4 +54,6 @@ class EnvSecretManagerService: SecretManagerService {
     override fun createSecretKeyInstance(secretName: String) = error("Not supported")
 
     override fun isSecretTypeSupported(secret: SecretKey) = error("Not supported")
+
+    override fun deleteSecret(secretName: String) = error("Not supported")
 }
diff --git a/util/gcp-tools/src/main/kotlin/ai/tock/gcp/secretmanager/GcpSecretManagerService.kt b/util/gcp-tools/src/main/kotlin/ai/tock/gcp/secretmanager/GcpSecretManagerService.kt
@@ -134,4 +134,14 @@ class GcpSecretManagerService: SecretManagerService {
     override fun createSecretKeyInstance(secretName: String) = GcpSecretKey(secretName)
 
     override fun isSecretTypeSupported(secret: SecretKey): Boolean = secret is GcpSecretKey
+
+    override fun deleteSecret(secretName: String) {
+        try {
+            client.deleteSecret(SecretName.of(EnvConfig.gcpProjectId, secretName))
+            logger.info { "The secret '$secretName' has been successfully deleted." }
+        } catch (e: Exception) {
+            logger.error(e) { "Failed to delete the secret '$secretName'." }
+            throw e
+        }
+    }
 }