S3 Support (GSI-934) (#5)

* Update template files

* Add initial support for S3 management

* Bump version from 1.1.0 -> 1.2.0

* Update debian dockerfile

* Move dummy/mock classes into separate module

Divide unit tests by type

Minor refactoring for documents tests

* Add some unit tests for objects management

* Add object deletion test

* Add catch-all error and tests for unhandled S3 exceptions

* Fix test config

* Suppress deletion 404 and raise existence check 404

* Add S3 integration tests

* Add 'does_bucket_exist' to storage mock

Change test bucket names to spinal case

Tweak double inequality check

* Mongo: raise error in read ops when db or coll not found

* Remove commented-out code

* Fix dockerfile

* Use fastapi.status codes in objects router

Fix typos

* Add object storage federation with test fixture

* Do minor improvement for get_patched_config

* Fix some doc strings

* Use multi s3 container as context manager

.devcontainer/.dev_config.yaml

@@ -6,3 +6,17 @@ db_connection_str: mongodb://mongodb:27017
 db_prefix: "test_"
 db_permissions:
 - "*.*:*"
+
+object_storages:
+  primary:
+    bucket: "permanent"
+    credentials:
+      s3_endpoint_url: http://localstack:4566
+      s3_access_key_id: test
+      s3_secret_access_key: test
+  secondary:
+    bucket: "staging"
+    credentials:
+      s3_endpoint_url: http://localstack:4566
+      s3_access_key_id: test
+      s3_secret_access_key: test

.github/workflows/ci_release.yaml

@@ -8,6 +8,10 @@ jobs:
   push_to_docker_hub:
     name: Push to Docker Hub
 
+    strategy:
+      matrix:
+        flavor: ["", "debian"]
+
     runs-on: ubuntu-latest
 
     steps:
@@ -16,3 +20,4 @@ jobs:
           tag: ${{ github.event.release.tag_name }}
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+          flavor: ${{ matrix.flavor }}

.pre-commit-config.yaml

@@ -45,13 +45,13 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.2
+    rev: v0.5.6
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.1
+    rev: v1.11.1
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.pyproject_generation/pyproject_custom.toml

@@ -1,11 +1,11 @@
 [project]
 name = "sms"
-version = "1.1.0"
+version = "1.2.0"
 description = "State Management Service - Provides a REST API for basic infrastructure technology state management."
 dependencies = [
     "typer >= 0.12",
     "ghga-service-commons[api] >= 3.1",
-    "hexkit[mongodb] >= 3.2",
+    "hexkit[mongodb,s3] >= 3.5",
 ]
 
 [project.urls]

.template/mandatory_files.txt

@@ -23,6 +23,7 @@ lock/requirements-dev.txt
 lock/requirements.txt
 
 Dockerfile
+Dockerfile.debian
 config_schema.json
 example_config.yaml
 LICENSE

Dockerfile

@@ -13,35 +13,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-## creating building container
-FROM python:3.12-slim-bookworm AS builder
-# update and install dependencies
-RUN apt update
-RUN apt upgrade -y
+# BASE: a base image with updated packages
+FROM python:3.12-alpine AS base
+RUN apk upgrade --no-cache --available
+
+# BUILDER: a container to build the service wheel
+FROM base AS builder
 RUN pip install build
-# copy code
 COPY . /service
 WORKDIR /service
-# build wheel
 RUN python -m build
 
-# creating running container
-FROM python:3.12-slim-bookworm
-# update and install dependencies
-RUN apt update
-RUN apt upgrade -y
-# copy and install requirements and wheel
+# DEP-BUILDER: a container to (build and) install dependencies
+FROM base AS dep-builder
+RUN apk update
+RUN apk add build-base gcc g++ libffi-dev zlib-dev
+RUN apk upgrade --available
 WORKDIR /service
 COPY --from=builder /service/lock/requirements.txt /service
 RUN pip install --no-deps -r requirements.txt
-RUN rm requirements.txt
+
+# RUNNER: a container to run the service
+FROM base AS runner
+WORKDIR /service
+RUN rm -rf /usr/local/lib/python3.12
+COPY --from=dep-builder /usr/local/lib/python3.12 /usr/local/lib/python3.12
 COPY --from=builder /service/dist/ /service
 RUN pip install --no-deps *.whl
 RUN rm *.whl
-# create new user and execute as that user
-RUN useradd --create-home appuser
+RUN adduser -D appuser
 WORKDIR /home/appuser
 USER appuser
-# set environment
 ENV PYTHONUNBUFFERED=1
+
+# Please adapt to package name:
 ENTRYPOINT ["sms"]

Dockerfile.debian

@@ -0,0 +1,47 @@
+# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## creating building container
+FROM python:3.12-slim-bookworm AS builder
+# update and install dependencies
+RUN apt update
+RUN apt upgrade -y
+RUN pip install build
+# copy code
+COPY . /service
+WORKDIR /service
+# build wheel
+RUN python -m build
+
+# creating running container
+FROM python:3.12-slim-bookworm
+# update and install dependencies
+RUN apt update
+RUN apt upgrade -y
+# copy and install requirements and wheel
+WORKDIR /service
+COPY --from=builder /service/lock/requirements.txt /service
+RUN pip install --no-deps -r requirements.txt
+RUN rm requirements.txt
+COPY --from=builder /service/dist/ /service
+RUN pip install --no-deps *.whl
+RUN rm *.whl
+# create new user and execute as that user
+RUN useradd --create-home appuser
+WORKDIR /home/appuser
+USER appuser
+# set environment
+ENV PYTHONUNBUFFERED=1
+ENTRYPOINT ["sms"]

README.md

@@ -24,21 +24,21 @@ We recommend using the provided Docker container.
 
 A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/state-management-service):
 ```bash
-docker pull ghga/state-management-service:1.1.0
+docker pull ghga/state-management-service:1.2.0
 ```
 
 Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile):
 ```bash
 # Execute in the repo's root dir:
-docker build -t ghga/state-management-service:1.1.0 .
+docker build -t ghga/state-management-service:1.2.0 .
 ```
 
 For production-ready deployment, we recommend using Kubernetes, however,
 for simple use cases, you could execute the service using docker
 on a single server:
 ```bash
 # The entrypoint is preconfigured:
-docker run -p 8080:8080 ghga/state-management-service:1.1.0 --help
+docker run -p 8080:8080 ghga/state-management-service:1.2.0 --help
 ```
 
 If you prefer not to use containers, you may install the service from source:
@@ -55,7 +55,11 @@ sms --help
 ### Parameters
 
 The service requires the following configuration parameters:
-- **`token_hashes`** *(array)*: List of token hashes corresponding to the tokens that can be used to authenticate calls to this service. Hashes are made with SHA-256.
+- **`object_storages`** *(object, required)*: Can contain additional properties.
+
+  - **Additional properties**: Refer to *[#/$defs/S3ObjectStorageNodeConfig](#%24defs/S3ObjectStorageNodeConfig)*.
+
+- **`token_hashes`** *(array, required)*: List of token hashes corresponding to the tokens that can be used to authenticate calls to this service. Hashes are made with SHA-256.
 
   - **Items** *(string)*
 
@@ -67,7 +71,7 @@ The service requires the following configuration parameters:
   ```
 
 
-- **`db_prefix`** *(string)*: Prefix to add to all database names used in the SMS.
+- **`db_prefix`** *(string, required)*: Prefix to add to all database names used in the SMS.
 
 
   Examples:
@@ -139,7 +143,7 @@ The service requires the following configuration parameters:
   ```
 
 
-- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
+- **`db_connection_str`** *(string, format: password, required)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
 
 
   Examples:
@@ -153,7 +157,7 @@ The service requires the following configuration parameters:
 
 - **`service_name`** *(string)*: Short name of this service. Default: `"sms"`.
 
-- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. This is included in log messages.
+- **`service_instance_id`** *(string, required)*: A string that uniquely identifies this instance across all instances of this service. This is included in log messages.
 
 
   Examples:
@@ -293,6 +297,96 @@ The service requires the following configuration parameters:
   ```
 
 
+## Definitions
+
+
+- <a id="%24defs/S3Config"></a>**`S3Config`** *(object)*: S3-specific config params.
+Inherit your config class from this class if you need
+to talk to an S3 service in the backend.<br>  Args:
+    s3_endpoint_url (str): The URL to the S3 endpoint.
+    s3_access_key_id (str):
+        Part of credentials for login into the S3 service. See:
+        https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
+    s3_secret_access_key (str):
+        Part of credentials for login into the S3 service. See:
+        https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
+    s3_session_token (Optional[str]):
+        Optional part of credentials for login into the S3 service. See:
+        https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
+    aws_config_ini (Optional[Path]):
+        Path to a config file for specifying more advanced S3 parameters.
+        This should follow the format described here:
+        https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-a-configuration-file
+        Defaults to None. Cannot contain additional properties.
+
+  - **`s3_endpoint_url`** *(string, required)*: URL to the S3 API.
+
+
+    Examples:
+
+    ```json
+    "http://localhost:4566"
+    ```
+
+
+  - **`s3_access_key_id`** *(string, required)*: Part of credentials for login into the S3 service. See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html.
+
+
+    Examples:
+
+    ```json
+    "my-access-key-id"
+    ```
+
+
+  - **`s3_secret_access_key`** *(string, format: password, required)*: Part of credentials for login into the S3 service. See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html.
+
+
+    Examples:
+
+    ```json
+    "my-secret-access-key"
+    ```
+
+
+  - **`s3_session_token`**: Part of credentials for login into the S3 service. See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html. Default: `null`.
+
+    - **Any of**
+
+      - *string, format: password*
+
+      - *null*
+
+
+    Examples:
+
+    ```json
+    "my-session-token"
+    ```
+
+
+  - **`aws_config_ini`**: Path to a config file for specifying more advanced S3 parameters. This should follow the format described here: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-a-configuration-file. Default: `null`.
+
+    - **Any of**
+
+      - *string, format: path*
+
+      - *null*
+
+
+    Examples:
+
+    ```json
+    "~/.aws/config"
+    ```
+
+
+- <a id="%24defs/S3ObjectStorageNodeConfig"></a>**`S3ObjectStorageNodeConfig`** *(object)*: Configuration for one specific object storage node and one bucket in it.<br>  The bucket is the main bucket that the service is responsible for. Cannot contain additional properties.
+
+  - **`bucket`** *(string, required)*
+
+  - **`credentials`**: Refer to *[#/$defs/S3Config](#%24defs/S3Config)*.
+
 
 ### Usage: