improve logging and use proj.db in dev debug setup

GeodetischeInfrastructuur · Dec 20, 2023 · f639741 · f639741
1 parent 197cab7
commit f639741
Show file tree

Hide file tree

Showing 6 changed files with 185 additions and 18 deletions.
diff --git a/.vscode/debug-logging.yaml b/.vscode/debug-logging.yaml
@@ -0,0 +1,112 @@
+disable_existing_loggers: false
+formatters:
+  access:
+    (): uvicorn.logging.AccessFormatter
+    fmt: '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s'
+  default:
+    (): uvicorn.logging.DefaultFormatter
+    fmt: "%(levelprefix)s %(message)s"
+    use_colors: null
+handlers:
+  access:
+    class: logging.StreamHandler
+    formatter: access
+    stream: ext://sys.stdout
+  default:
+    class: logging.StreamHandler
+    formatter: default
+    stream: ext://sys.stderr
+loggers:
+  coordinate_transformation_api.assets:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.assets.static:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.cityjson:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.cityjson.models:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.constants:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.crs_transform:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.fastapi_rfc7807:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.fastapi_rfc7807.middleware:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.fastapi_rfc7807.schema:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.limit_middleware:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.limit_middleware.middleware:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.main:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.models:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.settings:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.types:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  coordinate_transformation_api.util:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  uvicorn:
+    handlers:
+      - default
+    level: DEBUG
+    propagate: false
+  uvicorn.access:
+    handlers:
+      - access
+    level: DEBUG
+    propagate: false
+  uvicorn.error:
+    level: DEBUG
+version: 1
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -13,6 +13,7 @@
                 "--reload",
                 "--port", "8000",
                 "coordinate_transformation_api.main:app",
+                "--log-config", "${workspaceFolder}/.vscode/debug-logging.yaml"
             ],
             "env": {
                 "LOG_LEVEL": "DEBUG",

diff --git a/Dockerfile b/Dockerfile
@@ -11,7 +11,9 @@ ENV ENV PIP_ROOT_USER_ACTION=ignore
 
 WORKDIR /src
 
-RUN apt update && apt install jq curl -y
+RUN apt-get update && apt-get install jq curl -y
+
+RUN rm -f dist/*
 
 RUN pip install --upgrade setuptools && \
     pip install --upgrade pip && \
@@ -20,12 +22,12 @@ RUN pip install --upgrade setuptools && \
 
 WORKDIR /assets
 
-RUN curl -o nl_nsgi_nlgeo2018.tif https://cdn.proj.org/nl_nsgi_nlgeo2018.tif && \
-    curl -o nl_nsgi_rdcorr2018.tif https://cdn.proj.org/nl_nsgi_rdcorr2018.tif && \
-    curl -o nl_nsgi_rdtrans2018.tif https://cdn.proj.org/nl_nsgi_rdtrans2018.tif
-
-RUN curl -sL -H "Accept: application/octet-stream" $(curl -s "https://api.github.com/repos/GeodetischeInfrastructuur/transformations/releases/tags/${NSGI_PROJ_DB_VERSION}" | jq -r '.assets[] | select(.name=="proj.db").url') -o proj.db
+RUN curl -sL -o nl_nsgi_nlgeo2018.tif https://cdn.proj.org/nl_nsgi_nlgeo2018.tif && \
+    curl -sL -o nl_nsgi_rdcorr2018.tif https://cdn.proj.org/nl_nsgi_rdcorr2018.tif && \
+    curl -sL -o nl_nsgi_rdtrans2018.tif https://cdn.proj.org/nl_nsgi_rdtrans2018.tif && \
+    curl -sL -H "Accept: application/octet-stream" $(curl -s "https://api.github.com/repos/GeodetischeInfrastructuur/transformations/releases/tags/${NSGI_PROJ_DB_VERSION}" | jq -r '.assets[] | select(.name=="proj.db").url') -o proj.db
 
+RUN ls -lah /src/dist/ >&2
 
 FROM python:3.11.4-slim-bullseye as runner
 

diff --git a/README.md b/README.md
@@ -82,6 +82,15 @@ Check test coverage (install `coverage` with `pip install coverage`):
 python3 -m coverage run --source=src/coordinate_transformation_api -m pytest -v tests && python3 -m coverage report -m
 ```
 
+### Install NSGI proj.db
+
+Execute the following shell one-liner to install the NSGI `proj.db` from the [GeodetischeInfrastructuur/transformations](https://github.com/GeodetischeInfrastructuur/transformations/releases) repo:
+
+
+```sh
+curl -sL -H "Accept: application/octet-stream" $(curl -s "https://api.github.com/repos/GeodetischeInfrastructuur/transformations/releases/latest" | jq -r '.assets[] | select(.name=="proj.db").url') -o "$(python3 -c 'import pyproj;print(pyproj.datadir.get_data_dir());')"/proj.db
+```
+
 ## Install
 
 ```bash

diff --git a/src/coordinate_transformation_api/main.py b/src/coordinate_transformation_api/main.py
@@ -2,8 +2,11 @@
 import json
 import logging
 import os
+import pkgutil
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
 from importlib import resources as impresources
-from typing import Annotated, Callable, Union
+from typing import Annotated, Any, Callable, Union
 
 import pyproj
 import uvicorn
@@ -18,6 +21,7 @@
 from geojson_pydantic import Feature
 from geojson_pydantic.geometries import Geometry, GeometryCollection
 
+import coordinate_transformation_api
 from coordinate_transformation_api import assets
 from coordinate_transformation_api.cityjson.models import CityjsonV113
 from coordinate_transformation_api.fastapi_rfc7807 import middleware
@@ -62,11 +66,12 @@
     TRANSFORMATIONS_EXCLUDE = yaml.safe_load(f)["transformations"]["exclude"]
 
 
-logging.config.fileConfig(str(logging_conf), disable_existing_loggers=False)
-logger = logging.getLogger(__name__)
-logger.setLevel(app_settings.log_level)
-if not app_settings.debug:  # suppres pyproj warnings in prod
-    logging.getLogger("pyproj").setLevel(logging.ERROR)
+# logging.config.fileConfig(str(logging_conf), disable_existing_loggers=False)
+# logger = logging.getLogger("uvicorn")
+# # logger.addHandler(logging.StreamHandler())
+
+
+# logger = logging.getLogger("uvicorn")
 
 
 OPEN_API_SPEC: dict
@@ -76,12 +81,23 @@
 crs_identifiers: list[str] = OPEN_API_SPEC["components"]["schemas"]["crs-enum"]["enum"]
 CRS_LIST = [Crs.from_crs_str(x) for x in crs_identifiers]
 BASE_DIR: str = os.path.dirname(__file__)
-
+logger: logging.Logger
 
 CrsEnum: enum = enum.Enum("CrsEnum", {f"{x}": x for x in crs_identifiers})  # type: ignore
 
 
-app: FastAPI = FastAPI(docs_url=None)
+@asynccontextmanager
+async def lifespan(_app: FastAPI) -> AsyncGenerator:
+    global logger  # noqa: PLW0603
+    logger = logging.getLogger(__name__)
+    logger.info(f"settings: {app_settings}")
+    logger.info(f"pyproj datadir: {pyproj.datadir.get_data_dir()}")
+    if not app_settings.debug:  # suppres pyproj warnings in prod
+        logging.getLogger("pyproj").setLevel(logging.ERROR)
+    yield
+
+
+app: FastAPI = FastAPI(docs_url=None, lifespan=lifespan)
 # note: order of adding middleware is required for it to work
 middleware.register(app)
 app.add_middleware(
@@ -424,15 +440,37 @@ async def post_transform(  # noqa: ANN201, PLR0913
 
 app.openapi = lambda: OPEN_API_SPEC  # type: ignore
 
-logger.info(f"pyproj datadir: {pyproj.datadir.get_data_dir()}")
+
+def get_logging_config() -> Any:  # noqa: ANN401
+    logging_config = uvicorn.config.LOGGING_CONFIG
+    logging_config["loggers"]["uvicorn"]["level"] = app_settings.log_level
+    logging_config["loggers"]["uvicorn.error"]["level"] = app_settings.log_level
+    logging_config["loggers"]["uvicorn.access"]["level"] = app_settings.log_level
+
+    package = coordinate_transformation_api
+    for _importer, modname, _ispkg in pkgutil.walk_packages(
+        path=package.__path__, prefix=package.__name__ + ".", onerror=lambda _: None
+    ):
+        logging_config["loggers"][modname] = {
+            "handlers": ["default"],
+            "level": app_settings.log_level,
+            "propagate": False,
+        }
+    return logging_config
 
 
 def main() -> None:
     uvicorn.run(
-        "coordinate_transformation_api.main:app",
-        workers=2,
+        f"{__name__}:app",
+        workers=1,
         port=8000,
         host="0.0.0.0",  # noqa: S104
+        log_level=app_settings.log_level.lower(),
+        log_config=get_logging_config(),
+        access_log=app_settings.access_log,
+        loop="uvloop",
+        server_header=False,
+        date_header=False,
     )
 
 

diff --git a/src/coordinate_transformation_api/settings.py b/src/coordinate_transformation_api/settings.py
@@ -62,7 +62,7 @@ class AppSettings(BaseSettings):
         default=2000000,
         description="max size request body in bytes",
     )
-    log_level: str = Field(alias="LOG_LEVEL", default="ERROR")
+    log_level: str = Field(alias="LOG_LEVEL", default="INFO")
     debug: bool = Field(
         alias="DEBUG",
         default=False,
@@ -84,6 +84,11 @@ class AppSettings(BaseSettings):
         default=None,
         description="CORS origins, either a comma separated list of HTTPS urls of the value `*` to allow CORS on all origins",
     )
+    access_log: bool = Field(
+        alias="ACCESS_LOG",
+        default=False,
+        description="enable access log, defaults to False",
+    )
 
     @classmethod
     def settings_customise_sources(  # type: ignore  # noqa: PLR0913