Vault Support (GSI-973) (#8)

* Update lock files and template files

* Bump version from 1.3.1 -> 1.4.0

Update lock files, openapi, config, readme

Add vault config to sms config

* Add secrets handler to sms

* Add unit tests for sms vault

* Log InvalidPath error but don't re-raise

Remove custom errors -- are not used

* Add integration tests and vault fixture

* Add unit tests for the SecretsHandler

* Remove VaultFixture.get_secret()

* Make vault container persistent for tests

Compartmentalize VaultConfig

* Update config to use sms instead of ekss

* Only offer mass deletion endpoint

* Move vault_path from config to API

* Delete unused dummy test functionality and a test

* Rename vault fixture to remove pragmas

* Remove dummy secrets handler, just use mock

.devcontainer/.dev_config.yaml

@@ -2,6 +2,9 @@ service_instance_id: "1"
 token_hashes:
 # plaintext token: 43fadc91-b98f-4925-bd31-1b054b13dc55
 - 7ad83b6b9183c91674eec897935bc154ba9ff9704f8be0840e77f476b5062b6e
+vault_token: "dev-token"
+vault_url: "http://vault:8200"
+
 db_connection_str: mongodb://mongodb:27017
 db_prefix: "test_"
 db_permissions:

.devcontainer/devcontainer.json

@@ -52,7 +52,8 @@
 				"visualstudioexptteam.vscodeintellicode",
 				"ymotongpoo.licenser",
 				"charliermarsh.ruff",
-				"ms-python.mypy-type-checker"
+				"ms-python.mypy-type-checker",
+				"-ms-python.autopep8"
 			]
 		}
 	},
@@ -71,4 +72,4 @@
 		// details can be found here: https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker
 		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
 	}
-}
+}

.pre-commit-config.yaml

@@ -45,13 +45,13 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.2
+    rev: v0.7.0
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.1
+    rev: v1.13.0
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.pyproject_generation/pyproject_custom.toml

@@ -1,11 +1,12 @@
 [project]
 name = "sms"
-version = "1.3.1"
+version = "1.4.0"
 description = "State Management Service - Provides a REST API for basic infrastructure technology state management."
 dependencies = [
     "typer >= 0.12",
     "ghga-service-commons[api] >= 3.1",
     "hexkit[mongodb,s3,akafka] >= 3.5",
+    "hvac>=2",
 ]
 
 [project.urls]

README.md

@@ -24,21 +24,21 @@ We recommend using the provided Docker container.
 
 A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/state-management-service):
 ```bash
-docker pull ghga/state-management-service:1.3.1
+docker pull ghga/state-management-service:1.4.0
 ```
 
 Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile):
 ```bash
 # Execute in the repo's root dir:
-docker build -t ghga/state-management-service:1.3.1 .
+docker build -t ghga/state-management-service:1.4.0 .
 ```
 
 For production-ready deployment, we recommend using Kubernetes, however,
 for simple use cases, you could execute the service using docker
 on a single server:
 ```bash
 # The entrypoint is preconfigured:
-docker run -p 8080:8080 ghga/state-management-service:1.3.1 --help
+docker run -p 8080:8080 ghga/state-management-service:1.4.0 --help
 ```
 
 If you prefer not to use containers, you may install the service from source:
@@ -125,6 +125,26 @@ The service requires the following configuration parameters:
 
   - **Additional properties**: Refer to *[#/$defs/S3ObjectStorageNodeConfig](#%24defs/S3ObjectStorageNodeConfig)*.
 
+- **`vault_url`** *(string, required)*: URL for the Vault.
+
+
+  Examples:
+
+  ```json
+  "http://vault:8200"
+  ```
+
+
+- **`vault_token`** *(string, required)*: Token for the Vault.
+
+
+  Examples:
+
+  ```json
+  "dev-token"
+  ```
+
+
 - **`token_hashes`** *(array, required)*: List of token hashes corresponding to the tokens that can be used to authenticate calls to this service. Hashes are made with SHA-256.
 
   - **Items** *(string)*

config_schema.json

@@ -188,6 +188,22 @@
       "title": "Object Storages",
       "type": "object"
     },
+    "vault_url": {
+      "description": "URL for the Vault",
+      "examples": [
+        "http://vault:8200"
+      ],
+      "title": "Vault Url",
+      "type": "string"
+    },
+    "vault_token": {
+      "description": "Token for the Vault",
+      "examples": [
+        "dev-token"
+      ],
+      "title": "Vault Token",
+      "type": "string"
+    },
     "token_hashes": {
       "description": "List of token hashes corresponding to the tokens that can be used to authenticate calls to this service. Hashes are made with SHA-256.",
       "examples": [
@@ -411,6 +427,8 @@
     "service_instance_id",
     "kafka_servers",
     "object_storages",
+    "vault_url",
+    "vault_token",
     "token_hashes",
     "db_prefix",
     "db_connection_str"

example_config.yaml

@@ -46,4 +46,6 @@ service_instance_id: '1'
 service_name: sms
 token_hashes:
 - 7ad83b6b9183c91674eec897935bc154ba9ff9704f8be0840e77f476b5062b6e
+vault_token: dev-token
+vault_url: http://vault:8200
 workers: 1

openapi.yaml

@@ -57,7 +57,7 @@ components:
 info:
   description: A service for basic infrastructure technology state management.
   title: State Management Service
-  version: 1.3.1
+  version: 1.4.0
 openapi: 3.1.0
 paths:
   /documents/permissions:
@@ -383,5 +383,71 @@ paths:
       tags:
       - StateManagementService
       - sms-s3
+  /secrets/{vault_path}:
+    delete:
+      description: Delete all secrets from the specified vault.
+      operationId: delete_secrets
+      parameters:
+      - in: path
+        name: vault_path
+        required: true
+        schema:
+          description: The path to the vault
+          examples:
+          - ekss
+          - sms
+          title: Vault Path
+          type: string
+      responses:
+        '204':
+          description: Successful Response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      security:
+      - HTTPBearer: []
+      summary: Delete all secrets from the specified vault.
+      tags:
+      - StateManagementService
+      - sms-vault
+    get:
+      description: Returns a list of secrets in the specified vault
+      operationId: get_secrets
+      parameters:
+      - in: path
+        name: vault_path
+        required: true
+        schema:
+          description: The path to the vault
+          examples:
+          - ekss
+          - sms
+          title: Vault Path
+          type: string
+      responses:
+        '200':
+          content:
+            application/json:
+              schema:
+                items:
+                  type: string
+                title: Response Get Secrets
+                type: array
+          description: Successful Response
+        '422':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      security:
+      - HTTPBearer: []
+      summary: Returns a list of secrets in the vault
+      tags:
+      - StateManagementService
+      - sms-vault
 tags:
 - name: StateManagementService

pyproject.toml

@@ -21,12 +21,13 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "sms"
-version = "1.3.1"
+version = "1.4.0"
 description = "State Management Service - Provides a REST API for basic infrastructure technology state management."
 dependencies = [
     "typer >= 0.12",
     "ghga-service-commons[api] >= 3.1",
     "hexkit[mongodb,s3,akafka] >= 3.5",
+    "hvac>=2",
 ]
 
 [project.license]

src/sms/adapters/inbound/fastapi_/dummies.py

@@ -25,13 +25,16 @@
 from sms.ports.inbound.docs_handler import DocsHandlerPort
 from sms.ports.inbound.events_handler import EventsHandlerPort
 from sms.ports.inbound.objects_handler import ObjectsHandlerPort
+from sms.ports.inbound.secrets_handler import SecretsHandlerPort
 
 config_dummy = DependencyDummy("config_dummy")
 docs_handler_port = DependencyDummy("docs_handler_port")
 objects_handler_port = DependencyDummy("objects_handler_port")
 events_handler_port = DependencyDummy("events_handler_port")
+secrets_handler_port = DependencyDummy("secrets_handler_port")
 
 ConfigDummy = Annotated[Config, Depends(config_dummy)]
 DocsHandlerPortDummy = Annotated[DocsHandlerPort, Depends(docs_handler_port)]
 ObjectsHandlerPortDummy = Annotated[ObjectsHandlerPort, Depends(objects_handler_port)]
 EventsHandlerPortDummy = Annotated[EventsHandlerPort, Depends(events_handler_port)]
+SecretsHandlerPortDummy = Annotated[SecretsHandlerPort, Depends(secrets_handler_port)]

src/sms/adapters/inbound/fastapi_/routers/secrets.py

@@ -0,0 +1,70 @@
+# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FastAPI routes for HashiCorp Vault state management."""
+
+from typing import Annotated
+
+from fastapi import APIRouter, HTTPException, status
+from pydantic import Field
+
+from sms.adapters.inbound.fastapi_ import dummies
+from sms.adapters.inbound.fastapi_.http_authorization import (
+    TokenAuthContext,
+    require_token,
+)
+
+secrets_router = APIRouter()
+
+vault_path_field = Field(
+    default=..., description="The path to the vault", examples=["ekss", "sms"]
+)
+
+
+@secrets_router.get(
+    "/{vault_path}",
+    operation_id="get_secrets",
+    summary="Returns a list of secrets in the vault",
+    status_code=status.HTTP_200_OK,
+    response_model=list[str],
+)
+async def get_secrets(
+    secrets_handler: dummies.SecretsHandlerPortDummy,
+    _token: Annotated[TokenAuthContext, require_token],
+    vault_path: Annotated[str, vault_path_field],
+):
+    """Returns a list of secrets in the specified vault"""
+    try:
+        return secrets_handler.get_secrets(vault_path)
+    except Exception as exc:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from exc
+
+
+@secrets_router.delete(
+    "/{vault_path}",
+    operation_id="delete_secrets",
+    summary="""Delete all secrets from the specified vault.""",
+    status_code=status.HTTP_204_NO_CONTENT,
+)
+async def delete_secrets(
+    secrets_handler: dummies.SecretsHandlerPortDummy,
+    _token: Annotated[TokenAuthContext, require_token],
+    vault_path: Annotated[str, vault_path_field],
+):
+    """Delete all secrets from the specified vault."""
+    try:
+        secrets_handler.delete_secrets(vault_path)
+    except Exception as exc:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from exc

src/sms/adapters/inbound/fastapi_/routes.py

@@ -19,12 +19,14 @@
 from sms.adapters.inbound.fastapi_.routers.documents import mongodb_router
 from sms.adapters.inbound.fastapi_.routers.events import kafka_router
 from sms.adapters.inbound.fastapi_.routers.objects import s3_router
+from sms.adapters.inbound.fastapi_.routers.secrets import secrets_router
 
 router = APIRouter(tags=["StateManagementService"])
 
 router.include_router(mongodb_router, prefix="/documents", tags=["sms-mongodb"])
 router.include_router(s3_router, prefix="/objects", tags=["sms-s3"])
 router.include_router(kafka_router, prefix="/events", tags=["sms-kafka"])
+router.include_router(secrets_router, prefix="/secrets", tags=["sms-vault"])
 
 
 @router.get(

src/sms/config.py

@@ -24,12 +24,13 @@
 from hexkit.log import LoggingConfig
 from hexkit.providers.akafka import KafkaConfig
 from pydantic import Field, SecretStr, field_validator, model_validator
-from pydantic_settings import BaseSettings
+
+from sms.core.secrets_handler import VaultConfig
 
 SERVICE_NAME: str = "sms"
 
 
-class SmsConfig(BaseSettings):
+class SmsConfig(VaultConfig):
     """Configuration specific to the SMS."""
 
     token_hashes: list[str] = Field(