Fix vault auth (GSI-1185) (#13)

* Remove vault_token from config and raise 403 errors when needed

* Bump version from 2.0.0 -> 3.0.0

.devcontainer/.dev_config.yaml

@@ -2,7 +2,6 @@ service_instance_id: "1"
 token_hashes:
 # plaintext token: 43fadc91-b98f-4925-bd31-1b054b13dc55
 - 7ad83b6b9183c91674eec897935bc154ba9ff9704f8be0840e77f476b5062b6e
-vault_token: "dev-token"
 vault_url: "http://vault:8200"
 vault_secrets_mount_point: secret
 vault_path: sms

.pyproject_generation/pyproject_custom.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sms"
-version = "2.0.0"
+version = "3.0.0"
 description = "State Management Service - Provides a REST API for basic infrastructure technology state management."
 dependencies = [
     "typer >= 0.12",

README.md

@@ -24,21 +24,21 @@ We recommend using the provided Docker container.
 
 A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/state-management-service):
 ```bash
-docker pull ghga/state-management-service:2.0.0
+docker pull ghga/state-management-service:3.0.0
 ```
 
 Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile):
 ```bash
 # Execute in the repo's root dir:
-docker build -t ghga/state-management-service:2.0.0 .
+docker build -t ghga/state-management-service:3.0.0 .
 ```
 
 For production-ready deployment, we recommend using Kubernetes, however,
 for simple use cases, you could execute the service using docker
 on a single server:
 ```bash
 # The entrypoint is preconfigured:
-docker run -p 8080:8080 ghga/state-management-service:2.0.0 --help
+docker run -p 8080:8080 ghga/state-management-service:3.0.0 --help
 ```
 
 If you prefer not to use containers, you may install the service from source:
@@ -135,16 +135,6 @@ The service requires the following configuration parameters:
   ```
 
 
-- **`vault_token`** *(string, required)*: Token for the Vault.
-
-
-  Examples:
-
-  ```json
-  "dev-token"
-  ```
-
-
 - **`vault_secrets_mount_point`** *(string)*: Name used to address the secret engine under a custom mount path. Default: `"secret"`.
 
 

config_schema.json

@@ -196,14 +196,6 @@
       "title": "Vault Url",
       "type": "string"
     },
-    "vault_token": {
-      "description": "Token for the Vault",
-      "examples": [
-        "dev-token"
-      ],
-      "title": "Vault Token",
-      "type": "string"
-    },
     "vault_secrets_mount_point": {
       "default": "secret",
       "description": "Name used to address the secret engine under a custom mount path.",
@@ -535,7 +527,6 @@
     "kafka_servers",
     "object_storages",
     "vault_url",
-    "vault_token",
     "vault_path",
     "token_hashes",
     "db_prefix",

example_config.yaml

@@ -53,7 +53,6 @@ vault_path: sms
 vault_role_id: '**********'
 vault_secret_id: '**********'
 vault_secrets_mount_point: secret
-vault_token: dev-token
 vault_url: http://vault:8200
 vault_verify: true
 workers: 1

openapi.yaml

@@ -57,7 +57,7 @@ components:
 info:
   description: A service for basic infrastructure technology state management.
   title: State Management Service
-  version: 2.0.0
+  version: 3.0.0
 openapi: 3.1.0
 paths:
   /documents/permissions:

pyproject.toml

@@ -21,7 +21,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "sms"
-version = "2.0.0"
+version = "3.0.0"
 description = "State Management Service - Provides a REST API for basic infrastructure technology state management."
 dependencies = [
     "typer >= 0.12",

src/sms/adapters/inbound/fastapi_/routers/secrets.py

@@ -48,6 +48,10 @@ async def get_secrets(
     """Returns a list of secrets in the specified vault"""
     try:
         return secrets_handler.get_secrets(vault_path)
+    except PermissionError as err:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN, detail=str(err)
+        ) from err
     except Exception as exc:
         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from exc
 
@@ -66,5 +70,9 @@ async def delete_secrets(
     """Delete all secrets from the specified vault."""
     try:
         secrets_handler.delete_secrets(vault_path)
+    except PermissionError as err:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN, detail=str(err)
+        ) from err
     except Exception as exc:
         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from exc

src/sms/core/secrets_handler.py

@@ -19,7 +19,7 @@
 
 from hvac import Client as HvacClient
 from hvac.api.auth_methods import Kubernetes
-from hvac.exceptions import InvalidPath
+from hvac.exceptions import Forbidden, InvalidPath
 from pydantic import Field, SecretStr, field_validator
 from pydantic_settings import BaseSettings
 
@@ -34,9 +34,6 @@ class VaultConfig(BaseSettings):
     vault_url: str = Field(
         default=..., description="URL for the Vault", examples=["http://vault:8200"]
     )
-    vault_token: str = Field(
-        default=..., description="Token for the Vault", examples=["dev-token"]
-    )
     vault_secrets_mount_point: str = Field(
         default="secret",
         examples=["secret"],
@@ -82,13 +79,30 @@ class VaultConfig(BaseSettings):
         description="Path to service account token used by kube auth adapter.",
     )
 
+    @field_validator("vault_verify")
+    @classmethod
+    def validate_vault_ca(cls, value: bool | str) -> bool | str:
+        """Check that the CA bundle can be read if it is specified."""
+        if isinstance(value, str):
+            path = Path(value)
+            if not path.exists():
+                raise ValueError(f"Vault CA bundle not found at: {path}")
+            try:
+                bundle = path.open().read()
+            except OSError as error:
+                raise ValueError("Vault CA bundle cannot be read") from error
+            if "-----BEGIN CERTIFICATE-----" not in bundle:
+                raise ValueError("Vault CA bundle does not contain a certificate")
+        return value
+
 
 class SecretsHandler(SecretsHandlerPort):
     """Adapter wrapping hvac.Client"""
 
     def __init__(self, config: VaultConfig):
         """Initialized approle-based client and log in"""
         self._config = config
+        self.client = HvacClient(url=config.vault_url)
         self._auth_mount_point = config.vault_auth_mount_point
         self._secrets_mount_point = config.vault_secrets_mount_point
         self._kube_role = config.vault_kube_role
@@ -123,7 +137,6 @@ def _login(self):
                 )
             else:
                 self._kube_adapter.login(role=self._kube_role, jwt=jwt)
-
         elif self._auth_mount_point:
             self.client.auth.approle.login(
                 role_id=self._role_id,
@@ -135,31 +148,6 @@ def _login(self):
                 role_id=self._role_id, secret_id=self._secret_id
             )
 
-    @field_validator("vault_verify")
-    @classmethod
-    def validate_vault_ca(cls, value: bool | str) -> bool | str:
-        """Check that the CA bundle can be read if it is specified."""
-        if isinstance(value, str):
-            path = Path(value)
-            if not path.exists():
-                raise ValueError(f"Vault CA bundle not found at: {path}")
-            try:
-                bundle = path.open().read()
-            except OSError as error:
-                raise ValueError("Vault CA bundle cannot be read") from error
-            if "-----BEGIN CERTIFICATE-----" not in bundle:
-                raise ValueError("Vault CA bundle does not contain a certificate")
-        return value
-
-    @property
-    def client(self) -> HvacClient:
-        """Return an instance of a vault client"""
-        return HvacClient(
-            url=self._config.vault_url,
-            token=self._config.vault_token,
-            verify=self._config.vault_verify,
-        )
-
     def get_secrets(self, vault_path: str) -> list[str]:
         """Return the IDs of all secrets in the specified vault."""
         self._check_auth()
@@ -178,6 +166,12 @@ def get_secrets(self, vault_path: str) -> list[str]:
             )
             log.warning(msg, vault_path)
             return []
+        except Forbidden as err:
+            permission_error = PermissionError(
+                f"Permission not configured for vault path '{vault_path}'",
+            )
+            log.error(permission_error)
+            raise permission_error from err
 
     def delete_secrets(self, vault_path: str):
         """Delete all secrets from the specified vault."""
@@ -188,7 +182,14 @@ def delete_secrets(self, vault_path: str):
             return
 
         for secret in secrets:
-            self.client.secrets.kv.v2.delete_metadata_and_all_versions(
-                path=f"{vault_path}/{secret}",
-                mount_point=self._config.vault_secrets_mount_point,
-            )
+            try:
+                self.client.secrets.kv.v2.delete_metadata_and_all_versions(
+                    path=f"{vault_path}/{secret}",
+                    mount_point=self._config.vault_secrets_mount_point,
+                )
+            except Forbidden as err:
+                permission_error = PermissionError(
+                    f"Permission not configured for vault path '{vault_path}'",
+                )
+                log.error(permission_error)
+                raise permission_error from err

tests/fixtures/test_config.yaml

@@ -2,7 +2,6 @@ service_instance_id: "1"
 token_hashes:
 # plaintext token: 43fadc91-b98f-4925-bd31-1b054b13dc55
 - 7ad83b6b9183c91674eec897935bc154ba9ff9704f8be0840e77f476b5062b6e
-vault_token: "dev-token"
 vault_url: "http://vault:8200"
 vault_path: sms
 vault_role_id: dummy-role

tests/fixtures/vault.py

@@ -20,6 +20,7 @@
 
 import hvac
 import pytest
+from hvac.api.auth_methods import Kubernetes
 from hvac.exceptions import InvalidPath
 from testcontainers.core.generic import DockerContainer
 
@@ -30,7 +31,7 @@
 VAULT_URL = DEFAULT_TEST_CONFIG.vault_url
 DEFAULT_PORT = 8200
 VAULT_PATH = DEFAULT_TEST_CONFIG.vault_path
-VAULT_TOKEN = DEFAULT_TEST_CONFIG.vault_token
+VAULT_TOKEN = "dev-token"
 VAULT_MOUNT_POINT = DEFAULT_TEST_CONFIG.vault_secrets_mount_point
 
 
@@ -43,12 +44,58 @@ class VaultFixture:
     def __init__(self, config: VaultConfig):
         self.config = config
         self.vaults_used: set[str] = set()
+        self.client = hvac.Client(url=self.config.vault_url)
+        self._auth_mount_point = config.vault_auth_mount_point
+        self._secrets_mount_point = config.vault_secrets_mount_point
+        self._kube_role = config.vault_kube_role
+
+        if self._kube_role:
+            # use kube role and service account token
+            self._kube_adapter = Kubernetes(self.client.adapter)
+            self._service_account_token_path = config.service_account_token_path
+        elif config.vault_role_id and config.vault_secret_id:
+            # use role and secret ID instead
+            self._role_id = config.vault_role_id.get_secret_value()
+            self._secret_id = config.vault_secret_id.get_secret_value()
+        else:
+            raise ValueError(
+                "There is no way to log in to vault:\n"
+                + "Neither kube role nor both role and secret ID were provided."
+            )
+
+    def _check_auth(self):
+        """Check if authentication timed out and re-authenticate if needed"""
+        if not self.client.is_authenticated():
+            self._login()
+
+    def _login(self):
+        """Log in using Kubernetes Auth or AppRole"""
+        if self._kube_role:
+            with self._service_account_token_path.open() as token_file:
+                jwt = token_file.read()
+            if self._auth_mount_point:
+                self._kube_adapter.login(
+                    role=self._kube_role, jwt=jwt, mount_point=self._auth_mount_point
+                )
+            else:
+                self._kube_adapter.login(role=self._kube_role, jwt=jwt)
+
+        elif self._auth_mount_point:
+            self.client.auth.approle.login(
+                role_id=self._role_id,
+                secret_id=self._secret_id,
+                mount_point=self._auth_mount_point,
+            )
+        else:
+            self.client.auth.approle.login(
+                role_id=self._role_id, secret_id=self._secret_id
+            )
 
     def store_secret(self, *, key: str, vault_path: str = VAULT_PATH):
         """Store a secret in vault"""
-        client = hvac.Client(url=self.config.vault_url, token=VAULT_TOKEN)
+        self._check_auth()
 
-        client.secrets.kv.v2.create_or_update_secret(
+        self.client.secrets.kv.v2.create_or_update_secret(
             path=f"{vault_path}/{key}",
             secret={key: f"secret_for_{key}"},
             mount_point=self.config.vault_secrets_mount_point,
@@ -57,15 +104,15 @@ def store_secret(self, *, key: str, vault_path: str = VAULT_PATH):
 
     def delete_all_secrets(self, vault_path: str):
         """Remove all secrets from the vault to reset for next test"""
-        client = hvac.Client(url=self.config.vault_url, token=VAULT_TOKEN)
+        self._check_auth()
 
         with suppress(InvalidPath):
-            secrets = client.secrets.kv.v2.list_secrets(
+            secrets = self.client.secrets.kv.v2.list_secrets(
                 path=vault_path,
                 mount_point=self.config.vault_secrets_mount_point,
             )
             for key in secrets["data"]["keys"]:
-                client.secrets.kv.v2.delete_metadata_and_all_versions(
+                self.client.secrets.kv.v2.delete_metadata_and_all_versions(
                     path=f"{vault_path}/{key}",
                     mount_point=self.config.vault_secrets_mount_point,
                 )
@@ -107,7 +154,6 @@ def vault_container_fixture() -> Generator[VaultContainerFixture, None, None]:
             vault_role_id=role_id,
             vault_secret_id=secret_id,
             vault_verify=DEFAULT_TEST_CONFIG.vault_verify,
-            vault_token=VAULT_TOKEN,
             vault_path=VAULT_PATH,
             vault_secrets_mount_point=VAULT_MOUNT_POINT,
         )
@@ -144,18 +190,15 @@ def configure_vault(*, host: str, port: int):
     # create access policy to bind to role
     policy = f"""
     path "{VAULT_MOUNT_POINT}/data/{VAULT_PATH}/*" {{
-        capabilities = ["read", "create"]
+        capabilities = ["read", "list", "create", "update", "delete"]
     }}
     path "{VAULT_MOUNT_POINT}/metadata/{VAULT_PATH}/*" {{
-        capabilities = ["delete"]
+        capabilities = ["read", "list", "create", "update", "delete"]
     }}
     """
 
     # inject policy
-    client.sys.create_or_update_policy(
-        name=VAULT_PATH,
-        policy=policy,
-    )
+    client.sys.create_or_update_policy(name=VAULT_PATH, policy=policy)
 
     role_name = "test_role"
     # create role and bind policy
@@ -170,9 +213,7 @@ def configure_vault(*, host: str, port: int):
     role_id = response["data"]["role_id"]
 
     # retrieve secret_id
-    response = client.auth.approle.generate_secret_id(
-        role_name=role_name,
-    )
+    response = client.auth.approle.generate_secret_id(role_name=role_name)
     secret_id = response["data"]["secret_id"]
 
     # log out root token client