From 2792809c64e1ca7e378c6387104af8f63ef3b148 Mon Sep 17 00:00:00 2001 From: TheByronHimes Date: Wed, 7 Feb 2024 08:19:59 +0000 Subject: [PATCH] Add logging and update docs --- README.md | 47 +++++++++++++++++++++++++++--------- config_schema.json | 59 ++++++++++++++++++++++++++++++++++----------- example_config.yaml | 2 ++ src/wps/config.py | 8 ++++-- src/wps/main.py | 3 +++ 5 files changed, 91 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 4d6c982..93e9cfd 100644 --- a/README.md +++ b/README.md @@ -83,46 +83,69 @@ wps --help ### Parameters The service requires the following configuration parameters: -- **`datasets_collection`** *(string)*: The name of the database collection for datasets. Default: `"datasets"`. - -- **`work_packages_collection`** *(string)*: The name of the database collection for work packages. Default: `"workPackages"`. +- **`log_level`** *(string)*: The minimum log level to capture. Must be one of: `["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"]`. Default: `"INFO"`. -- **`work_package_valid_days`** *(integer)*: How many days a work package (and its access token) stays valid. Default: `30`. +- **`service_name`** *(string)*: Default: `"wps"`. -- **`work_package_signing_key`** *(string, format: password)*: The private key for signing work order tokens. +- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id. Examples: ```json - "{\"crv\": \"P-256\", \"kty\": \"EC\", \"x\": \"...\", \"y\": \"...\"}" + "germany-bw-instance-001" ``` -- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/. +- **`log_format`**: If set, will replace JSON formatting with the specified string format. If not set, has no effect. In addition to the standard attributes, the following can also be specified: timestamp, service, instance, level, correlation_id, and details. Default: `null`. + + - **Any of** + + - *string* + + - *null* Examples: ```json - "mongodb://localhost:27017" + "%(timestamp)s - %(service)s - %(level)s - %(message)s" ``` -- **`db_name`** *(string)*: Default: `"work-packages"`. + ```json + "%(asctime)s - Severity: %(levelno)s - %(msg)s" + ``` -- **`service_name`** *(string)*: Default: `"wps"`. -- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id. +- **`datasets_collection`** *(string)*: The name of the database collection for datasets. Default: `"datasets"`. + +- **`work_packages_collection`** *(string)*: The name of the database collection for work packages. Default: `"workPackages"`. + +- **`work_package_valid_days`** *(integer)*: How many days a work package (and its access token) stays valid. Default: `30`. + +- **`work_package_signing_key`** *(string, format: password)*: The private key for signing work order tokens. Examples: ```json - "germany-bw-instance-001" + "{\"crv\": \"P-256\", \"kty\": \"EC\", \"x\": \"...\", \"y\": \"...\"}" ``` +- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/. + + + Examples: + + ```json + "mongodb://localhost:27017" + ``` + + +- **`db_name`** *(string)*: Default: `"work-packages"`. + - **`kafka_servers`** *(array)*: A list of connection strings to connect to Kafka bootstrap servers. - **Items** *(string)* diff --git a/config_schema.json b/config_schema.json index c76b84d..7a61321 100644 --- a/config_schema.json +++ b/config_schema.json @@ -2,6 +2,50 @@ "additionalProperties": false, "description": "Modifies the orginal Settings class provided by the user", "properties": { + "log_level": { + "default": "INFO", + "description": "The minimum log level to capture.", + "enum": [ + "CRITICAL", + "ERROR", + "WARNING", + "INFO", + "DEBUG", + "TRACE" + ], + "title": "Log Level", + "type": "string" + }, + "service_name": { + "default": "wps", + "title": "Service Name", + "type": "string" + }, + "service_instance_id": { + "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.", + "examples": [ + "germany-bw-instance-001" + ], + "title": "Service Instance Id", + "type": "string" + }, + "log_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If set, will replace JSON formatting with the specified string format. If not set, has no effect. In addition to the standard attributes, the following can also be specified: timestamp, service, instance, level, correlation_id, and details", + "examples": [ + "%(timestamp)s - %(service)s - %(level)s - %(message)s", + "%(asctime)s - Severity: %(levelno)s - %(msg)s" + ], + "title": "Log Format" + }, "datasets_collection": { "default": "datasets", "description": "The name of the database collection for datasets", @@ -45,19 +89,6 @@ "title": "Db Name", "type": "string" }, - "service_name": { - "default": "wps", - "title": "Service Name", - "type": "string" - }, - "service_instance_id": { - "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.", - "examples": [ - "germany-bw-instance-001" - ], - "title": "Service Instance Id", - "type": "string" - }, "kafka_servers": { "description": "A list of connection strings to connect to Kafka bootstrap servers.", "examples": [ @@ -311,9 +342,9 @@ } }, "required": [ + "service_instance_id", "work_package_signing_key", "db_connection_str", - "service_instance_id", "kafka_servers", "dataset_change_event_topic", "dataset_upsertion_event_type", diff --git a/example_config.yaml b/example_config.yaml index d1cdec7..6103b8a 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -30,6 +30,8 @@ kafka_ssl_cafile: '' kafka_ssl_certfile: '' kafka_ssl_keyfile: '' kafka_ssl_password: '' +log_format: null +log_level: INFO openapi_url: /openapi.json port: 8080 service_instance_id: wps_1 diff --git a/src/wps/config.py b/src/wps/config.py index 83281bc..03006c4 100644 --- a/src/wps/config.py +++ b/src/wps/config.py @@ -18,6 +18,7 @@ from ghga_service_commons.api import ApiConfigBase from ghga_service_commons.auth.ghga import AuthConfig from hexkit.config import config_from_yaml +from hexkit.log import LoggingConfig from hexkit.providers.akafka import KafkaConfig from hexkit.providers.mongodb import MongoDbConfig @@ -25,8 +26,10 @@ from wps.adapters.outbound.http import AccessCheckConfig from wps.core.repository import WorkPackageConfig +SERVICE_NAME = "wps" -@config_from_yaml(prefix="wps") + +@config_from_yaml(prefix=SERVICE_NAME) class Config( ApiConfigBase, AuthConfig, @@ -35,8 +38,9 @@ class Config( KafkaConfig, MongoDbConfig, WorkPackageConfig, + LoggingConfig, ): """Config parameters and their defaults.""" - service_name: str = "wps" + service_name: str = SERVICE_NAME db_name: str = "work-packages" diff --git a/src/wps/main.py b/src/wps/main.py index a1a2b96..dc2a253 100644 --- a/src/wps/main.py +++ b/src/wps/main.py @@ -16,6 +16,7 @@ """In this module object construction and dependency injection is carried out.""" from ghga_service_commons.api import run_server +from hexkit.log import configure_logging from wps.config import Config from wps.inject import prepare_consumer, prepare_rest_app @@ -24,6 +25,7 @@ async def run_rest_app() -> None: """Run the HTTP REST API.""" config = Config() # type: ignore + configure_logging(config=config) async with prepare_rest_app(config=config) as app: await run_server(app=app, config=config) @@ -32,6 +34,7 @@ async def run_rest_app() -> None: async def consume_events(run_forever: bool = True) -> None: """Run an event consumer listening to the configured topic.""" config = Config() # type: ignore + configure_logging(config=config) async with prepare_consumer(config=config) as consumer: await consumer.event_subscriber.run(forever=run_forever)