diff --git a/.dockerignore b/.dockerignore index ce39f0170..947d389a0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,9 +7,7 @@ !MANIFEST.in !tilecloud_chain/* !docker/run -!development.ini -!production.ini -!gunicorn.conf.py +!application.ini !package*.json !.nvmrc !screenshot.js diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4e28dacb6..bbc51551c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -54,6 +54,7 @@ jobs: path: /tmp/pre-commit.patch retention-days: 1 if: failure() + - name: Print environment information run: c2cciutils-env env: @@ -64,20 +65,13 @@ jobs: - name: Checks run: make checks - - run: git diff --exit-code --patch > /tmp/ruff.patch || true && git reset --hard - if: failure() - - uses: actions/upload-artifact@v4 - with: - name: Apply Ruff lint fix.patch - path: /tmp/ruff.patch - retention-days: 1 - if: failure() - name: Tests run: make tests - run: c2cciutils-docker-logs if: always() + - uses: actions/upload-artifact@v4 with: name: results diff --git a/Dockerfile b/Dockerfile index 675a43189..d9c8ff3a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,7 @@ RUN --mount=type=cache,target=/var/lib/apt/lists \ # From c2cwsgiutils -CMD ["gunicorn", "--paste=/app/production.ini"] +CMD ["/venv/bin/pserve", "c2c:///app/application.ini"] ENV LOG_TYPE=console \ DEVELOPMENT=0 \ @@ -86,7 +86,7 @@ ENV TILEGENERATION_CONFIGFILE=/etc/tilegeneration/config.yaml \ TILECLOUD_CHAIN_LOG_LEVEL=INFO \ TILECLOUD_LOG_LEVEL=INFO \ C2CWSGIUTILS_LOG_LEVEL=WARN \ - GUNICORN_LOG_LEVEL=WARN \ + WAITRESS_LOG_LEVEL=INFO \ SQL_LOG_LEVEL=WARN \ OTHER_LOG_LEVEL=WARN \ VISIBLE_ENTRY_POINT=/ \ @@ -96,7 +96,8 @@ ENV TILEGENERATION_CONFIGFILE=/etc/tilegeneration/config.yaml \ TILE_QUEUE_SIZE=2 \ TILE_CHUNK_SIZE=1 \ TILE_SERVER_LOGLEVEL=quiet \ - TILE_MAPCACHE_LOGLEVEL=verbose + TILE_MAPCACHE_LOGLEVEL=verbose \ + WAITRESS_THREADS=10 EXPOSE 8080 @@ -113,10 +114,6 @@ RUN --mount=type=cache,target=/root/.cache \ && mv docker/run /usr/bin/ \ && python3 -m compileall -q /app/tilecloud_chain -RUN mkdir -p /prometheus-metrics \ - && chmod a+rwx /prometheus-metrics -ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics - # Do the lint, used by the tests FROM base AS tests diff --git a/README.md b/README.md index 4b12d9961..b7f0a34ca 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# TileCloud Chain +# TileCloud-chain The goal of TileCloud Chain is to provide tools around tile generation on a chain like: @@ -52,33 +52,21 @@ Only the latest release is supported and version < 1.11 contains security iss Build it: ```bash -git submodule update --recursive -python3 -m venv .build/venv -.build/venv/bin/pip install -r requirements.txt -.build/venv/bin/pip install -e . -.build/venv/bin/pip install -r dev-requirements.txt +make build ``` ## Run prospector ```bash -.build/venv/bin/prospector +make prospector ``` ## Run the tests -Setup your environment: - -```bash -touch tilecloud_chain/OpenLayers.js -docker build --tag camptocamp/tilecloud-chain . -docker compose -p tilecloud up -``` - To run the tests: ```bash -docker compose -p tilecloud exec test python setup.py nosetests --logging-filter=tilecloud,tilecloud_chain --attr '!'nopy3 +make tests ``` ## Documentation @@ -86,20 +74,6 @@ docker compose -p tilecloud exec test python setup.py nosetests --logging-filter As documentation you can read the [USAGE.rst](https://github.com/camptocamp/tilecloud-chain/blob/master/tilecloud_chain/USAGE.rst) and the [configuration reference](https://github.com/camptocamp/tilecloud-chain/blob/master/tilecloud_chain/CONFIG.md). -## VSCode - -You can add that in your workspace configuration to use the JSON schema: - -```json -{ - "yaml.schemas": { - "../tilecloud-chain/tilecloud_chain/schema.json": [ - "tilecloud-chain/tilecloud_chain/tests/tilegeneration/*.yaml" - ] - } -} -``` - ## Contributing Install the pre-commit hooks: diff --git a/development.ini b/application.ini similarity index 82% rename from development.ini rename to application.ini index 1683172d2..97ebb066b 100644 --- a/development.ini +++ b/application.ini @@ -18,24 +18,31 @@ c2c.base_path = /c2c tilegeneration_configfile = %(TILEGENERATION_CONFIGFILE)s -[pipeline:main] -pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry app +[filter:translogger] +use = egg:Paste#translogger +setup_console_handler = False [filter:proxy-prefix] use = egg:PasteDeploy#prefix prefix = %(VISIBLE_ENTRY_POINT)s +[pipeline:main] +pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry app + [server:main] use = egg:waitress#main listen = *:8080 +threads = %(WAITRESS_THREADS)s +trusted_proxy = True +clear_untrusted_proxy_headers = False ### -# logging configuration +# Logging configuration # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/logging.html ### [loggers] -keys = root, c2cwsgi, tilecloud, tilecloud_chain, sqlalchemy +keys = root, waitress, c2cwsgiutils, tilecloud, tilecloud_chain, sqlalchemy [handlers] keys = console, json @@ -58,7 +65,7 @@ level = %(TILECLOUD_CHAIN_LOG_LEVEL)s handlers = qualname = tilecloud_chain -[logger_c2cwsgi] +[logger_c2cwsgiutils] level = %(C2CWSGIUTILS_LOG_LEVEL)s handlers = qualname = c2cwsgiutils @@ -71,6 +78,11 @@ qualname = sqlalchemy.engine # "level = DEBUG" logs SQL queries and results. # "level = WARN" logs neither. (Recommended for production systems.) +[logger_waitress] +level = %(WAITRESS_LOG_LEVEL)s +handlers = +qualname = waitress + [handler_console] class = StreamHandler args = (sys.stdout,) diff --git a/docker-compose.sample.override.yaml b/docker-compose.override.sample.yaml similarity index 84% rename from docker-compose.sample.override.yaml rename to docker-compose.override.sample.yaml index b0b239eba..df1a0500d 100644 --- a/docker-compose.sample.override.yaml +++ b/docker-compose.override.sample.yaml @@ -1,13 +1,13 @@ -version: '2.2' - services: application: &app ports: - '9050:8080' command: - - pserve + - /venv/bin/pserve - --reload - - c2c:///app/development.ini + - c2c:///app/application.ini + environment: + - DEVELOPMENT=TRUE volumes: - ./tilecloud_chain:/app/tilecloud_chain:ro # - ../tilecloud/tilecloud:/usr/local/lib/python3.10/dist-packages/tilecloud:ro diff --git a/docker-compose.yaml b/docker-compose.yaml index 0fee572c0..b4a6f6af7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -85,7 +85,7 @@ services: slave: <<: *app command: - - generate-tiles + - /venv/bin/generate-tiles - '--role=slave' - '--daemon' environment: diff --git a/gunicorn.conf.py b/gunicorn.conf.py deleted file mode 100644 index ad9456908..000000000 --- a/gunicorn.conf.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Gunicorn configuration file. - -https://docs.gunicorn.org/en/stable/settings.html -""" - -import os - -import gunicorn.arbiter -import gunicorn.workers.base -from c2cwsgiutils import get_config_defaults, prometheus -from prometheus_client import multiprocess - -bind = ":8080" # pylint: disable=invalid-name - -worker_class = "gthread" # pylint: disable=invalid-name -workers = os.environ.get("GUNICORN_WORKERS", 2) -threads = os.environ.get("GUNICORN_THREADS", 10) - -preload = "true" # pylint: disable=invalid-name - -accesslog = "-" # pylint: disable=invalid-name -access_log_format = os.environ.get( - "GUNICORN_ACCESS_LOG_FORMAT", - '%(H)s %({Host}i)s %(m)s %(U)s?%(q)s "%(f)s" "%(a)s" %(s)s %(B)s %(D)s %(p)s', -) - -### -# logging configuration -# https://docs.python.org/3/library/logging.config.html#logging-config-dictschema -### -logconfig_dict = { - "version": 1, - "root": { - "level": os.environ["OTHER_LOG_LEVEL"], - "handlers": [os.environ["LOG_TYPE"]], - }, - "loggers": { - "gunicorn.error": {"level": os.environ["GUNICORN_LOG_LEVEL"]}, - # "level = INFO" logs SQL queries. - # "level = DEBUG" logs SQL queries and results. - # "level = WARN" logs neither. (Recommended for production systems.) - "sqlalchemy.engine": {"level": os.environ["SQL_LOG_LEVEL"]}, - "c2cwsgiutils": {"level": os.environ["C2CWSGIUTILS_LOG_LEVEL"]}, - "tilecloud": {"level": os.environ["TILECLOUD_LOG_LEVEL"]}, - "tilecloud_chain": {"level": os.environ["TILECLOUD_CHAIN_LOG_LEVEL"]}, - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "generic", - "stream": "ext://sys.stdout", - }, - "json": { - "class": "tilecloud_chain.JsonLogHandler", - "formatter": "generic", - "stream": "ext://sys.stdout", - }, - }, - "formatters": { - "generic": { - "format": "%(asctime)s [%(process)d] [%(levelname)-5.5s] %(message)s", - "datefmt": "[%Y-%m-%d %H:%M:%S %z]", - "class": "logging.Formatter", - } - }, -} - -raw_paste_global_conf = ["=".join(e) for e in get_config_defaults().items()] - - -def on_starting(server: gunicorn.arbiter.Arbiter) -> None: - """ - Will start the prometheus server. - - Called just before the master process is initialized. - """ - del server - - prometheus.start() - - -def post_fork(server: gunicorn.arbiter.Arbiter, worker: gunicorn.workers.base.Worker) -> None: - """ - Will cleanup the configuration we get from the main process. - - Called just after a worker has been forked. - """ - del server, worker - - prometheus.cleanup() - - -def child_exit(server: gunicorn.arbiter.Arbiter, worker: gunicorn.workers.base.Worker) -> None: - """ - Remove the metrics for the exited worker. - - Called just after a worker has been exited, in the master process. - """ - del server - - multiprocess.mark_process_dead(worker.pid) # type: ignore [no-untyped-call] diff --git a/poetry.lock b/poetry.lock index 0bd6a5aae..f982cc1f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1426,6 +1426,24 @@ files = [ {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "paste" +version = "3.10.1" +description = "Tools for using a Web Server Gateway Interface stack" +optional = false +python-versions = ">=3" +files = [ + {file = "Paste-3.10.1-py3-none-any.whl", hash = "sha256:995e9994b6a94a2bdd8bd9654fb70ca3946ffab75442468bacf31b4d06481c3d"}, + {file = "paste-3.10.1.tar.gz", hash = "sha256:1c3d12065a5e8a7a18c0c7be1653a97cf38cc3e9a5a0c8334a9dd992d3a05e4a"}, +] + +[package.dependencies] +setuptools = "*" + +[package.extras] +flup = ["flup"] +openid = ["python-openid"] + [[package]] name = "pastedeploy" version = "3.1.0" @@ -1723,6 +1741,36 @@ files = [ [package.dependencies] prospector = ">=1.13.0" +[[package]] +name = "psutil" +version = "6.1.0" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"}, + {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"}, + {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"}, + {file = "psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"}, + {file = "psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, + {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, + {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, + {file = "psutil-6.1.0-cp36-cp36m-win32.whl", hash = "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"}, + {file = "psutil-6.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"}, + {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, + {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, + {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, +] + +[package.extras] +dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] + [[package]] name = "psycopg2" version = "2.9.10" @@ -3419,4 +3467,4 @@ test = ["zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "2554891c4bc8419edc08b1274511535fb5a5443858d01d1474955d5486af24e1" +content-hash = "c20155e459d9523c6e4cf77cb381082abcae70d8cf49e63772095eeed8464748" diff --git a/production.ini b/production.ini deleted file mode 100644 index b6107a2c1..000000000 --- a/production.ini +++ /dev/null @@ -1,26 +0,0 @@ -### -# app configuration -# http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/environment.html -### - -[app:app] -use = egg:tilecloud-chain -filter-with = proxy-prefix - -pyramid.reload_templates = %(DEVELOPMENT)s -pyramid.debug_authorization = %(DEVELOPMENT)s -pyramid.debug_notfound = %(DEVELOPMENT)s -pyramid.debug_routematch = %(DEVELOPMENT)s -pyramid.debug_templates = %(DEVELOPMENT)s -pyramid.default_locale_name = en - -c2c.base_path = /c2c - -tilegeneration_configfile = %(TILEGENERATION_CONFIGFILE)s - -[pipeline:main] -pipeline = egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry app - -[filter:proxy-prefix] -use = egg:PasteDeploy#prefix -prefix = %(VISIBLE_ENTRY_POINT)s diff --git a/pyproject.toml b/pyproject.toml index c010de95e..6ee250b6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,8 @@ jsonschema-validator-new = "0.3.2" azure-storage-blob = "12.23.1" waitress = "3.0.1" certifi = "2024.8.30" +Paste = "3.10.1" +psutil = "6.1.0" [tool.poetry.group.dev.dependencies] prospector = { extras = ["with_mypy", "with_bandit", "with_pyroma", "with_ruff"], version = "1.13.2" } diff --git a/tilecloud_chain/USAGE.rst b/tilecloud_chain/USAGE.rst index 77d3c175e..071f95405 100644 --- a/tilecloud_chain/USAGE.rst +++ b/tilecloud_chain/USAGE.rst @@ -574,7 +574,7 @@ To use the pyramid view use the following configuration: Internal WSGI server ~~~~~~~~~~~~~~~~~~~~ -in ``production.ini``: +in ``application.ini``: .. code:: @@ -582,17 +582,6 @@ in ``production.ini``: use = egg:tilecloud_chain#server configfile = %(here)s/tilegeneration/config.yaml -with the Apache configuration: - -.. code:: - - WSGIDaemonProcess tiles:${instanceid} display-name=%{GROUP} user=${modwsgi_user} - WSGIScriptAlias /${instanceid}/tiles ${directory}/apache/wmts.wsgi - - WSGIProcessGroup tiles:${instanceid} - WSGIApplicationGroup %{GLOBAL} - - Commands -------- diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index e83cdc7d8..ff14ae0db 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -496,7 +496,6 @@ def __init__( "handlers": [os.environ["LOG_TYPE"]], }, "loggers": { - "gunicorn.error": {"level": os.environ["GUNICORN_LOG_LEVEL"]}, # "level = INFO" logs SQL queries. # "level = DEBUG" logs SQL queries and results. # "level = WARN" logs neither. (Recommended for production systems.) diff --git a/tilecloud_chain/server.py b/tilecloud_chain/server.py index d37e2be2f..89f1e71ef 100644 --- a/tilecloud_chain/server.py +++ b/tilecloud_chain/server.py @@ -32,19 +32,27 @@ import logging import mimetypes import os +import resource import time +from collections.abc import Generator from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast from urllib.parse import parse_qs, urlencode import botocore.exceptions +import c2cwsgiutils.prometheus import c2cwsgiutils.pyramid +import prometheus_client +import prometheus_client.core +import prometheus_client.metrics_core +import prometheus_client.multiprocess +import prometheus_client.registry +import psutil import pyramid.response import pyramid.session import requests import tilecloud.store.s3 from azure.core.exceptions import ResourceNotFoundError from c2cwsgiutils import health_check -from c2cwsgiutils.prometheus import MemoryMapCollector from prometheus_client import REGISTRY, Summary from pyramid.config import Configurator from pyramid.httpexceptions import HTTPException, exception_response @@ -863,10 +871,58 @@ def forbidden(request: pyramid.request.Request) -> pyramid.response.Response: ) +class _ResourceCollector(prometheus_client.registry.Collector): + """Collect the resources used by Python.""" + + def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]: + """Get the gauge from smap file.""" + gauge = prometheus_client.core.GaugeMetricFamily( + c2cwsgiutils.prometheus.build_metric_name("python_resource"), + "Python resources", + labels=["name"], + ) + r = resource.getrusage(resource.RUSAGE_SELF) + for field in dir(r): + if field.startswith("ru_"): + gauge.add_metric([field[3:]], getattr(r, field)) + yield gauge + + +class _MemoryInfoCollector(prometheus_client.registry.Collector): + """Collect the resources used by Python.""" + + process = psutil.Process(os.getpid()) + + def collect(self) -> Generator[prometheus_client.core.GaugeMetricFamily, None, None]: + """Get the gauge from smap file.""" + gauge = prometheus_client.core.GaugeMetricFamily( + c2cwsgiutils.prometheus.build_metric_name("python_memory_info"), + "Python memory info", + labels=["name"], + ) + memory_info = self.process.memory_info() + gauge.add_metric(["rss"], memory_info.rss) + gauge.add_metric(["vms"], memory_info.vms) + gauge.add_metric(["shared"], memory_info.shared) + gauge.add_metric(["text"], memory_info.text) + gauge.add_metric(["lib"], memory_info.lib) + gauge.add_metric(["data"], memory_info.data) + gauge.add_metric(["dirty"], memory_info.dirty) + yield gauge + + def main(global_config: Any, **settings: Any) -> Router: """Start the server in Pyramid.""" del global_config # unused + REGISTRY.register(c2cwsgiutils.prometheus.MemoryMapCollector("pss")) + if os.environ.get("TILECLOUD_CHAIN_PROMETHEUS_MEMORY_MAP", "false").lower() in ("true", "1", "on"): + REGISTRY.register(c2cwsgiutils.prometheus.MemoryMapCollector("rss")) + REGISTRY.register(c2cwsgiutils.prometheus.MemoryMapCollector("size")) + REGISTRY.register(_ResourceCollector()) + REGISTRY.register(_MemoryInfoCollector()) + prometheus_client.start_http_server(int(os.environ["C2C_PROMETHEUS_PORT"])) + config = Configurator(settings=settings) config.set_session_factory( @@ -932,8 +988,4 @@ def main(global_config: Any, **settings: Any) -> Router: config.scan("tilecloud_chain.views") - if os.environ.get("TILECLOUD_CHAIN_PROMETHEUS_MEMORY_MAP", "false").lower() in ("true", "1", "on"): - REGISTRY.register(MemoryMapCollector("rss")) - REGISTRY.register(MemoryMapCollector("size")) - return config.make_wsgi_app()