[FEATURE] - make setup_logging independent (#166)

pyproject.toml

@@ -100,6 +100,8 @@ worker = [
     "pydantic-settings",
     "sqlalchemy",
     "sqlalchemy-utils",
+    "python-multipart",
+    "python-jose",
     "celery",
     "onetl",
     "asgi-correlation-id",
@@ -108,6 +110,7 @@ worker = [
     "uuid6",
     "coloredlogs",
     "python-json-logger",
+    "pyyaml",
 ]
 
 scheduler = [

syncmaster/backend/middlewares/__init__.py

@@ -4,7 +4,6 @@
 from fastapi import FastAPI
 
 from syncmaster.backend.middlewares.cors import apply_cors_middleware
-from syncmaster.backend.middlewares.logging import setup_logging
 from syncmaster.backend.middlewares.monitoring.metrics import (
     apply_monitoring_metrics_middleware,
 )
@@ -13,6 +12,7 @@
 from syncmaster.backend.middlewares.session import apply_session_middleware
 from syncmaster.backend.middlewares.static_files import apply_static_files
 from syncmaster.backend.settings import ServerAppSettings as Settings
+from syncmaster.settings.log import setup_logging
 
 
 def apply_middlewares(

syncmaster/db/repositories/credentials_repository.py

@@ -2,24 +2,28 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-from typing import NoReturn
+from typing import TYPE_CHECKING, NoReturn
 
 from sqlalchemy import ScalarResult, insert, select
 from sqlalchemy.exc import DBAPIError, IntegrityError, NoResultFound
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models import AuthData
 from syncmaster.db.repositories.base import Repository
 from syncmaster.db.repositories.utils import decrypt_auth_data, encrypt_auth_data
 from syncmaster.exceptions import SyncmasterError
 from syncmaster.exceptions.credentials import AuthDataNotFoundError
 
+if TYPE_CHECKING:
+    from syncmaster.backend.settings import ServerAppSettings
+    from syncmaster.scheduler.settings import SchedulerAppSettings
+    from syncmaster.worker.settings import WorkerAppSettings
+
 
 class CredentialsRepository(Repository[AuthData]):
     def __init__(
         self,
-        settings: Settings,
+        settings: WorkerAppSettings | SchedulerAppSettings | ServerAppSettings,
         session: AsyncSession,
         model: type[AuthData],
     ):

syncmaster/db/repositories/utils.py

@@ -1,16 +1,22 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
 import json
+from typing import TYPE_CHECKING
 
 from cryptography.fernet import Fernet
 from pydantic import SecretStr
 
-from syncmaster.backend.settings import ServerAppSettings as Settings
+if TYPE_CHECKING:
+    from syncmaster.backend.settings import ServerAppSettings
+    from syncmaster.scheduler.settings import SchedulerAppSettings
+    from syncmaster.worker.settings import WorkerAppSettings
 
 
 def decrypt_auth_data(
     value: str,
-    settings: Settings,
+    settings: WorkerAppSettings | SchedulerAppSettings | ServerAppSettings,
 ) -> dict:
     decryptor = Fernet(settings.encryption.crypto_key)
     decrypted = decryptor.decrypt(value)
@@ -24,7 +30,7 @@ def _json_default(value):
 
 def encrypt_auth_data(
     value: dict,
-    settings: Settings,
+    settings: WorkerAppSettings | SchedulerAppSettings | ServerAppSettings,
 ) -> str:
     encryptor = Fernet(settings.encryption.crypto_key)
     serialized = json.dumps(

syncmaster/scheduler/__main__.py

@@ -3,10 +3,10 @@
 import asyncio
 import logging
 
-from syncmaster.scheduler.logging import setup_logging
 from syncmaster.scheduler.settings import SchedulerAppSettings as Settings
 from syncmaster.scheduler.transfer_fetcher import TransferFetcher
 from syncmaster.scheduler.transfer_job_manager import TransferJobManager
+from syncmaster.settings.log import setup_logging
 
 logger = logging.getLogger(__name__)
 

syncmaster/settings/log/__init__.py

@@ -1,13 +1,33 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
 
+import logging
 import textwrap
+from logging.config import dictConfig
 from pathlib import Path
 from typing import Literal
 
+import yaml
 from pydantic import BaseModel, Field
 
 LOG_PATH = Path(__file__).parent.resolve()
+logger = logging.getLogger(__name__)
+
+
+class LoggingSetupError(Exception):
+    pass
+
+
+def setup_logging(config_path: Path) -> None:
+    """Parse file with logging configuration, and setup logging accordingly"""
+    if not config_path.exists():
+        raise OSError(f"Logging configuration file '{config_path}' does not exist")
+
+    try:
+        config = yaml.safe_load(config_path.read_text())
+        dictConfig(config)
+    except Exception as e:
+        raise LoggingSetupError(f"Error reading logging configuration '{config_path}'") from e
 
 
 class LoggingSettings(BaseModel):

syncmaster/worker/transfer.py

@@ -11,11 +11,10 @@
 from sqlalchemy import select
 from sqlalchemy.orm import Session, selectinload
 
-from syncmaster.backend.middlewares.logging import setup_logging
-from syncmaster.backend.settings import ServerAppSettings as Settings
 from syncmaster.db.models import AuthData, Run, Status, Transfer
 from syncmaster.db.repositories.utils import decrypt_auth_data
 from syncmaster.exceptions.run import RunNotFoundError
+from syncmaster.settings.log import setup_logging
 from syncmaster.worker import celery
 from syncmaster.worker.controller import TransferController
 from syncmaster.worker.settings import WorkerAppSettings
@@ -37,7 +36,7 @@ def run_transfer_task(self: Celery, run_id: int) -> None:
         )
 
 
-def run_transfer(session: Session, run_id: int, settings: Settings):
+def run_transfer(session: Session, run_id: int, settings: WorkerAppSettings):
     logger.info("Start transfer")
     run = session.get(
         Run,
@@ -86,7 +85,7 @@ def run_transfer(session: Session, run_id: int, settings: Settings):
 
 @after_setup_logger.connect
 def setup_loggers(*args, **kwargs):
-    setup_logging(Settings().logging.get_log_config_path())
+    setup_logging(WorkerAppSettings().logging.get_log_config_path())
 
 
 @before_task_publish.connect()