Update template files and dependencies

.deprecated_files

@@ -10,6 +10,7 @@
 .github/workflows/check_mandatory_and_static_files.yaml
 .github/workflows/dev_cd.yaml
 .github/workflows/unit_and_int_tests.yaml
+.github/workflows/cd.yaml
 
 scripts/check_mandatory_and_static_files.py
 scripts/update_static_files.py

.devcontainer/dev_launcher

@@ -1,3 +1,7 @@
 #!/bin/bash
 
+set -a
+. <(python tests/fixtures/auth_keys.py)
+set +a
+
 wps run-rest

.devcontainer/devcontainer.json

@@ -25,12 +25,16 @@
 				],
 				"python.testing.unittestEnabled": false,
 				"python.testing.pytestEnabled": true,
+				"editor.codeActionsOnSave": {
+					"source.organizeImports": true
+				},
 				"editor.formatOnSave": true,
 				"editor.renderWhitespace": "all",
 				"editor.rulers": [
 					88
 				],
-				"editor.defaultFormatter": "ms-python.black-formatter",
+				"ruff.organizeImports": true,
+				"editor.defaultFormatter": "charliermarsh.ruff",
 				"licenser.license": "Custom",
 				"licenser.customHeaderFile": "/workspace/.devcontainer/license_header.txt"
 			},
@@ -52,7 +56,6 @@
 				"visualstudioexptteam.vscodeintellicode",
 				"ymotongpoo.licenser",
 				"charliermarsh.ruff",
-				"ms-python.black-formatter",
 				"ms-python.mypy-type-checker"
 			]
 		}

.github/workflows/ci_release.yaml

@@ -0,0 +1,15 @@
+name: CI on release
+
+on:
+  release:
+      types: [published]
+
+jobs:
+  push_to_docker_hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ghga-de/gh-action-ci@v1
+        with:
+          tag: ${{ github.event.release.tag_name }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/ci_workflow_dispatch.yaml

@@ -0,0 +1,30 @@
+name: Build on PR or dispatch
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+jobs:
+  fetch-tag:
+    runs-on: ubuntu-latest
+    if: ( github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'build') ) || ( github.event.action == 'labeled' && github.event.label.name == 'build' )
+    steps:
+      - id: fetch-tag
+        uses: ghga-de/gh-action-fetch-tag@v1
+    outputs:
+      latest_tag: ${{ steps.fetch-tag.outputs.latest_tag }}
+
+  push_to_docker_hub:
+    needs: fetch-tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ghga-de/gh-action-ci@v1
+        with:
+          tag: ${{ needs.fetch-tag.outputs.latest_tag }}-${{ github.sha }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/static_code_analysis.yaml

@@ -16,10 +16,9 @@ jobs:
         env:
           SKIP: no-commit-to-branch
       - name: ruff
-        uses: chartboost/ruff-action@v1
-      - name: black
         run: |
-          black --check .
+          ruff check --output-format=github .
+          ruff format --check .
       - name: mypy
         run: |
           mypy .

.gitignore

@@ -133,6 +133,12 @@ dmypy.json
 # ignore VS Code settings:
 .vscode/
 
+# key stores
+*.key
+*.rnd
+.keystore
+.ssl/
+
 # desktop settings and thumbnails
 .DS_Store
 desktop.ini

.mandatory_files

@@ -7,7 +7,6 @@
 
 .devcontainer/dev_launcher
 .devcontainer/docker-compose.yml
-.devcontainer/Dockerfile
 
 tests/__init__.py
 tests/fixtures/__init__.py

.pre-commit-config.yaml

@@ -51,16 +51,13 @@ repos:
       - id: debug-statements
       - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.292
+    rev: v0.1.7
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-  - repo: https://github.com/psf/black
-    rev: 23.9.1
-    hooks:
-      - id: black
+      - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.7.1
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.readme_template.md

@@ -1,5 +1,5 @@
 
-[![tests](https://github.com/ghga-de/$name/actions/workflows/unit_and_int_tests.yaml/badge.svg)](https://github.com/ghga-de/$name/actions/workflows/unit_and_int_tests.yaml)
+[![tests](https://github.com/ghga-de/$name/actions/workflows/tests.yaml/badge.svg)](https://github.com/ghga-de/$name/actions/workflows/tests.yaml)
 [![Coverage Status](https://coveralls.io/repos/github/ghga-de/$name/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/$name?branch=main)
 
 # $title

.ruff.toml

@@ -20,6 +20,8 @@ ignore = [
     "D400", # first doc line ends in period
     "D401", # non-imperative-mood
     "D107", # missing docstring in __init__
+    "D206", # indent-with-spaces (ignored for formatter)
+    "D300", # triple-single-quotes (ignored for formatter)
 ]
 
 line-length = 88

.static_files

@@ -30,12 +30,13 @@ scripts/list_outdated_dependencies.py
 scripts/README.md
 
 .github/workflows/check_config_docs.yaml
+.github/workflows/check_openapi_spec.yaml
+.github/workflows/check_readme.yaml
 .github/workflows/check_template_files.yaml
+.github/workflows/ci_release.yaml
+.github/workflows/ci_workflow_dispatch.yaml
 .github/workflows/static_code_analysis.yaml
 .github/workflows/tests.yaml
-.github/workflows/check_openapi_spec.yaml
-.github/workflows/check_readme.yaml
-.github/workflows/cd.yaml
 
 example_data/README.md
 

README.md

@@ -1,5 +1,5 @@
 
-[![tests](https://github.com/ghga-de/work-package-service/actions/workflows/unit_and_int_tests.yaml/badge.svg)](https://github.com/ghga-de/work-package-service/actions/workflows/unit_and_int_tests.yaml)
+[![tests](https://github.com/ghga-de/work-package-service/actions/workflows/tests.yaml/badge.svg)](https://github.com/ghga-de/work-package-service/actions/workflows/tests.yaml)
 [![Coverage Status](https://coveralls.io/repos/github/ghga-de/work-package-service/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/work-package-service?branch=main)
 
 # Work Package Service
@@ -138,6 +138,16 @@ The service requires the following configuration parameters:
   ```
 
 
+- **`kafka_security_protocol`** *(string)*: Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL. Must be one of: `["PLAINTEXT", "SSL"]`. Default: `"PLAINTEXT"`.
+
+- **`kafka_ssl_cafile`** *(string)*: Certificate Authority file path containing certificates used to sign broker certificates. If a CA not specified, the default system CA will be used if found by OpenSSL. Default: `""`.
+
+- **`kafka_ssl_certfile`** *(string)*: Optional filename of client certificate, as well as any CA certificates needed to establish the certificate's authenticity. Default: `""`.
+
+- **`kafka_ssl_keyfile`** *(string)*: Optional filename containing the client private key. Default: `""`.
+
+- **`kafka_ssl_password`** *(string)*: Optional password to be used for the client private key. Default: `""`.
+
 - **`dataset_change_event_topic`** *(string)*: Name of the topic for events that inform about datasets.
 
 
@@ -192,7 +202,7 @@ The service requires the following configuration parameters:
 
   - **Items** *(string)*
 
-- **`auth_check_claims`** *(object)*: A dict of all GHGA internal claims that shall be verified. Default: `{"email": null, "exp": null, "iat": null, "name": null}`.
+- **`auth_check_claims`** *(object)*: A dict of all GHGA internal claims that shall be verified. Default: `{"name": null, "email": null, "iat": null, "exp": null}`.
 
 - **`auth_map_claims`** *(object)*: A mapping of claims to attributes in the GHGA auth context. Can contain additional properties. Default: `{}`.
 

config_schema.json

@@ -71,6 +71,40 @@
       "title": "Kafka Servers",
       "type": "array"
     },
+    "kafka_security_protocol": {
+      "default": "PLAINTEXT",
+      "description": "Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL.",
+      "enum": [
+        "PLAINTEXT",
+        "SSL"
+      ],
+      "title": "Kafka Security Protocol",
+      "type": "string"
+    },
+    "kafka_ssl_cafile": {
+      "default": "",
+      "description": "Certificate Authority file path containing certificates used to sign broker certificates. If a CA not specified, the default system CA will be used if found by OpenSSL.",
+      "title": "Kafka Ssl Cafile",
+      "type": "string"
+    },
+    "kafka_ssl_certfile": {
+      "default": "",
+      "description": "Optional filename of client certificate, as well as any CA certificates needed to establish the certificate's authenticity.",
+      "title": "Kafka Ssl Certfile",
+      "type": "string"
+    },
+    "kafka_ssl_keyfile": {
+      "default": "",
+      "description": "Optional filename containing the client private key.",
+      "title": "Kafka Ssl Keyfile",
+      "type": "string"
+    },
+    "kafka_ssl_password": {
+      "default": "",
+      "description": "Optional password to be used for the client private key.",
+      "title": "Kafka Ssl Password",
+      "type": "string"
+    },
     "dataset_change_event_topic": {
       "description": "Name of the topic for events that inform about datasets.",
       "examples": [
@@ -124,10 +158,10 @@
     },
     "auth_check_claims": {
       "default": {
+        "name": null,
         "email": null,
-        "exp": null,
         "iat": null,
-        "name": null
+        "exp": null
       },
       "description": "A dict of all GHGA internal claims that shall be verified.",
       "title": "Auth Check Claims",

example_config.yaml

@@ -22,8 +22,13 @@ db_name: dev-db
 docs_url: /docs
 download_access_url: http://127.0.0.1:8080/download-access
 host: 127.0.0.1
+kafka_security_protocol: PLAINTEXT
 kafka_servers:
 - kafka:9092
+kafka_ssl_cafile: ''
+kafka_ssl_certfile: ''
+kafka_ssl_keyfile: ''
+kafka_ssl_password: ''
 log_level: info
 openapi_url: /openapi.json
 port: 8080