Use configured logging (GSI-614) (#19)

* Add logging and update docs * Remove extra 'debug-statements' pre-commit hooks * Add log statements to repository and event sub --------- Co-authored-by: TheByronHimes <[email protected]>
ghga-de · Feb 7, 2024 · 5aed6d0 · 5aed6d0
1 parent e079dbc
commit 5aed6d0
Show file tree

Hide file tree

Showing 8 changed files with 183 additions and 49 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -47,9 +47,6 @@ repos:
         args: [--fix=lf]
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
-      - id: debug-statements
-      - id: debug-statements
-      - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.2.1
     hooks:

diff --git a/README.md b/README.md
@@ -83,46 +83,69 @@ wps --help
 ### Parameters
 
 The service requires the following configuration parameters:
-- **`datasets_collection`** *(string)*: The name of the database collection for datasets. Default: `"datasets"`.
-
-- **`work_packages_collection`** *(string)*: The name of the database collection for work packages. Default: `"workPackages"`.
+- **`log_level`** *(string)*: The minimum log level to capture. Must be one of: `["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"]`. Default: `"INFO"`.
 
-- **`work_package_valid_days`** *(integer)*: How many days a work package (and its access token) stays valid. Default: `30`.
+- **`service_name`** *(string)*: Default: `"wps"`.
 
-- **`work_package_signing_key`** *(string, format: password)*: The private key for signing work order tokens.
+- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.
 
 
   Examples:
 
   ```json
-  "{\"crv\": \"P-256\", \"kty\": \"EC\", \"x\": \"...\", \"y\": \"...\"}"
+  "germany-bw-instance-001"
   ```
 
 
-- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
+- **`log_format`**: If set, will replace JSON formatting with the specified string format. If not set, has no effect. In addition to the standard attributes, the following can also be specified: timestamp, service, instance, level, correlation_id, and details. Default: `null`.
+
+  - **Any of**
+
+    - *string*
+
+    - *null*
 
 
   Examples:
 
   ```json
-  "mongodb://localhost:27017"
+  "%(timestamp)s - %(service)s - %(level)s - %(message)s"
   ```
 
 
-- **`db_name`** *(string)*: Default: `"work-packages"`.
+  ```json
+  "%(asctime)s - Severity: %(levelno)s - %(msg)s"
+  ```
 
-- **`service_name`** *(string)*: Default: `"wps"`.
 
-- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.
+- **`datasets_collection`** *(string)*: The name of the database collection for datasets. Default: `"datasets"`.
+
+- **`work_packages_collection`** *(string)*: The name of the database collection for work packages. Default: `"workPackages"`.
+
+- **`work_package_valid_days`** *(integer)*: How many days a work package (and its access token) stays valid. Default: `30`.
+
+- **`work_package_signing_key`** *(string, format: password)*: The private key for signing work order tokens.
 
 
   Examples:
 
   ```json
-  "germany-bw-instance-001"
+  "{\"crv\": \"P-256\", \"kty\": \"EC\", \"x\": \"...\", \"y\": \"...\"}"
   ```
 
 
+- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
+
+
+  Examples:
+
+  ```json
+  "mongodb://localhost:27017"
+  ```
+
+
+- **`db_name`** *(string)*: Default: `"work-packages"`.
+
 - **`kafka_servers`** *(array)*: A list of connection strings to connect to Kafka bootstrap servers.
 
   - **Items** *(string)*

diff --git a/config_schema.json b/config_schema.json
@@ -2,6 +2,50 @@
   "additionalProperties": false,
   "description": "Modifies the orginal Settings class provided by the user",
   "properties": {
+    "log_level": {
+      "default": "INFO",
+      "description": "The minimum log level to capture.",
+      "enum": [
+        "CRITICAL",
+        "ERROR",
+        "WARNING",
+        "INFO",
+        "DEBUG",
+        "TRACE"
+      ],
+      "title": "Log Level",
+      "type": "string"
+    },
+    "service_name": {
+      "default": "wps",
+      "title": "Service Name",
+      "type": "string"
+    },
+    "service_instance_id": {
+      "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.",
+      "examples": [
+        "germany-bw-instance-001"
+      ],
+      "title": "Service Instance Id",
+      "type": "string"
+    },
+    "log_format": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "If set, will replace JSON formatting with the specified string format. If not set, has no effect. In addition to the standard attributes, the following can also be specified: timestamp, service, instance, level, correlation_id, and details",
+      "examples": [
+        "%(timestamp)s - %(service)s - %(level)s - %(message)s",
+        "%(asctime)s - Severity: %(levelno)s - %(msg)s"
+      ],
+      "title": "Log Format"
+    },
     "datasets_collection": {
       "default": "datasets",
       "description": "The name of the database collection for datasets",
@@ -45,19 +89,6 @@
       "title": "Db Name",
       "type": "string"
     },
-    "service_name": {
-      "default": "wps",
-      "title": "Service Name",
-      "type": "string"
-    },
-    "service_instance_id": {
-      "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.",
-      "examples": [
-        "germany-bw-instance-001"
-      ],
-      "title": "Service Instance Id",
-      "type": "string"
-    },
     "kafka_servers": {
       "description": "A list of connection strings to connect to Kafka bootstrap servers.",
       "examples": [
@@ -311,9 +342,9 @@
     }
   },
   "required": [
+    "service_instance_id",
     "work_package_signing_key",
     "db_connection_str",
-    "service_instance_id",
     "kafka_servers",
     "dataset_change_event_topic",
     "dataset_upsertion_event_type",

diff --git a/example_config.yaml b/example_config.yaml
@@ -30,6 +30,8 @@ kafka_ssl_cafile: ''
 kafka_ssl_certfile: ''
 kafka_ssl_keyfile: ''
 kafka_ssl_password: ''
+log_format: null
+log_level: INFO
 openapi_url: /openapi.json
 port: 8080
 service_instance_id: wps_1

diff --git a/src/wps/adapters/inbound/event_sub.py b/src/wps/adapters/inbound/event_sub.py
@@ -16,6 +16,7 @@
 
 """KafkaEventSubscriber receiving events that announce datasets"""
 
+import logging
 from contextlib import suppress
 
 from ghga_event_schemas import pydantic_ as event_schemas
@@ -30,6 +31,8 @@
 
 __all__ = ["EventSubTranslatorConfig", "EventSubTranslator"]
 
+log = logging.getLogger(__name__)
+
 
 class EventSubTranslatorConfig(BaseSettings):
     """Config for dataset creation related events."""
@@ -83,6 +86,10 @@ async def _handle_upsertion(self, payload: JsonObject):
             stage = WorkType[validated_payload.stage.name]
         except KeyError:
             # stage does not correspond to a work type, ignore event
+            log.info(
+                "Ignoring dataset event with unknown stage %s",
+                validated_payload.stage.name,
+            )
             return
 
         files = [

diff --git a/src/wps/config.py b/src/wps/config.py
@@ -18,15 +18,18 @@
 from ghga_service_commons.api import ApiConfigBase
 from ghga_service_commons.auth.ghga import AuthConfig
 from hexkit.config import config_from_yaml
+from hexkit.log import LoggingConfig
 from hexkit.providers.akafka import KafkaConfig
 from hexkit.providers.mongodb import MongoDbConfig
 
 from wps.adapters.inbound.event_sub import EventSubTranslatorConfig
 from wps.adapters.outbound.http import AccessCheckConfig
 from wps.core.repository import WorkPackageConfig
 
+SERVICE_NAME = "wps"
 
-@config_from_yaml(prefix="wps")
+
+@config_from_yaml(prefix=SERVICE_NAME)
 class Config(
     ApiConfigBase,
     AuthConfig,
@@ -35,8 +38,9 @@ class Config(
     KafkaConfig,
     MongoDbConfig,
     WorkPackageConfig,
+    LoggingConfig,
 ):
     """Config parameters and their defaults."""
 
-    service_name: str = "wps"
+    service_name: str = SERVICE_NAME
     db_name: str = "work-packages"