diff --git a/.env.dev b/.env.dev index db7fba29b..f9518fe9f 100644 --- a/.env.dev +++ b/.env.dev @@ -16,7 +16,7 @@ LAYMAN_AUTHN_MODULES=layman.authn.oauth2,layman.authn.http_header LAYMAN_AUTHN_HTTP_HEADER_NAME=a0468616f9968eaecdc3377988aba650 GRANT_CREATE_PUBLIC_WORKSPACE=EVERYONE GRANT_PUBLISH_IN_PUBLIC_WORKSPACE=EVERYONE -LAYMAN_ROLE_SERVICE_URI=postgresql://docker:docker@postgresql:5432/gis?schema=_role_service +LAYMAN_ROLE_SERVICE_URI=postgresql://docker:docker@postgresql:5432/gis?autosave=conservative&schema=_role_service # connection parameters to PostgreSQL database LAYMAN_PG_HOST=postgresql diff --git a/CHANGELOG.md b/CHANGELOG.md index 996809686..ff48e44c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -566,8 +566,8 @@ make timgen-build - [#380](https://github.com/LayerManager/layman/issues/380) Enable to upload geojson with "id" attribute with non-unique values. - [#383](https://github.com/LayerManager/layman/issues/383) Add new Makefile target `upgrade-after-timeout` to finish upgrade in case of GeoServer call timeout. - Fix [GET Workspace Layer](doc/rest.md#get-workspace-layer) documentation; `style` item was incorrectly used instead of `sld`. -- [#347](https://github.com/LayerManager/layman/issues/347) Upgrade PostgreSQL 10 to 13.3 and PostGIS 2.4 to 3.1. Use docker image from [layermanager/postgis@hub.docker.com](https://hub.docker.com/repository/docker/layermanager/postgis@github.com), source is located at [layermanager/docker-postgis@github.com](https://github.com/LayerManager/docker-postgis). -- [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/r/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/docker). +- [#347](https://github.com/LayerManager/layman/issues/347) Upgrade PostgreSQL 10 to 13.3 and PostGIS 2.4 to 3.1. Use docker image from [layermanager/postgis@hub.docker.com](https://hub.docker.com/repository/docker/layermanager/postgis), source is located at [layermanager/docker-postgis@github.com](https://github.com/LayerManager/docker-postgis). +- [#367](https://github.com/LayerManager/layman/issues/367) Upgrade gdal from 2.4 to 3.3. Use docker image from [osgeo/gdal@hub.docker.com](https://hub.docker.com/repository/docker/osgeo/gdal), source is located at [osgeo/gdal@github.com](https://github.com/OSGeo/gdal/tree/master/docker). - [#367](https://github.com/LayerManager/layman/issues/367) Upgrade also - python from 3.6 to 3.8 - flask from 1.1 to 2.0 diff --git a/doc/env-settings.md b/doc/env-settings.md index dcbac4e9a..03c698bcd 100644 --- a/doc/env-settings.md +++ b/doc/env-settings.md @@ -100,7 +100,7 @@ List of [users](models.md#user) and [roles](models.md#role) giving them permissi List of [users](models.md#user) and [roles](models.md#role) giving them permission to publish new [publication](models.md#publication) in already created [public workspace](models.md#public-workspace). ### LAYMAN_ROLE_SERVICE_URI -URL of Role Service with schema in format `postgresql://:@:/?schema=`. If you want to use internal Role Service, set it to `postgresql://{LAYMAN_PG_USER}:{LAYMAN_PG_PASSWORD}@{LAYMAN_PG_HOST}:{LAYMAN_PG_PORT}/{LAYMAN_PG_DBNAME}?schema=_role_service`. +URL of Role Service with schema in format `postgresql://:@:/?schema=`. If you want to use internal Role Service, set it to `postgresql://:@:/?schema=_role_service` (replace variable names with their values). URL scheme must be `postgresql`. URL host must be mentioned explicitly, as well as DB schema in `schema` URL query parameter. ## Layman Test Client Settings diff --git a/doc/rest.md b/doc/rest.md index d6cac6b62..0140498c9 100644 --- a/doc/rest.md +++ b/doc/rest.md @@ -475,7 +475,7 @@ Content-Type: ## Workspace Layer Chunk -Layer Chunk endpoint enables to upload layer data files asynchronously by splitting them into small parts called *chunks* that are uploaded independently. The endpoint is expected to be operated using [Resumable.js](https://github.com/23/resumable.js/) library. Resumable.js can split and upload files by chunks using [HTML File API](https://developer.mozilla.org/en-US/docs/Web/API/File), widely [supported by major browsers](https://caniuse.com/#feat=fileapi). +Layer Chunk endpoint enables to upload layer data files asynchronously by splitting them into small parts called *chunks* that are uploaded independently. The endpoint is expected to be operated using [Resumable.js](https://github.com/23/resumable.js/) library. Resumable.js can split and upload files by chunks using [HTML File API](https://developer.mozilla.org/en-US/docs/Web/API/File), widely supported by major browsers. Check [Asynchronous file upload](async-file-upload.md) example. diff --git a/src/geoserver/role_service.py b/src/geoserver/role_service.py index e8fc9038f..84a4ac2d5 100644 --- a/src/geoserver/role_service.py +++ b/src/geoserver/role_service.py @@ -4,7 +4,9 @@ import sys from urllib.parse import urlparse from xml.sax.saxutils import escape +import psycopg2 +from db import util as db_util from requests_util import url_util from . import authn @@ -18,7 +20,7 @@ def setup_jdbc_role_service(data_dir, service_url, role_service_name, db_schema): logger.info(f"Ensuring GeoServer DB role service '{role_service_name}' " - f"for URL: {service_url}.") + f"for URL: {url_util.redact_uri(service_url)}.") role_service_path = os.path.join(data_dir, ROLE_SERVICE_PATH) layman_role_service_path = os.path.join(role_service_path, role_service_name) @@ -59,3 +61,19 @@ def set_primary_role_service(data_dir, role_service_name): element.text = role_service_name security_path = os.path.join(data_dir, 'security/config.xml') security_xml.write(security_path) + + +def check_jdbc_role_service(role_service_db_uri, role_service_schema): + + try: + db_util.get_connection_pool(db_uri_str=role_service_db_uri, encapsulate_exception=False) + except psycopg2.OperationalError as exc: + secret_conn_dict = url_util.redact_uri(role_service_db_uri) + raise Exception(f"Failed to connect to role service database {secret_conn_dict}") from exc + + try: + db_util.run_query(f"select name, parent from {role_service_schema}.roles limit 0", + uri_str=role_service_db_uri, encapsulate_exception=False) + except BaseException as exc: + secret_conn_dict = url_util.redact_uri(role_service_db_uri) + raise Exception(f"Error querying role service database {secret_conn_dict}") from exc diff --git a/src/layman_settings.py b/src/layman_settings.py index e5408af21..037e893aa 100644 --- a/src/layman_settings.py +++ b/src/layman_settings.py @@ -6,7 +6,7 @@ import db import geoserver -from layman_settings_util import read_clients_dict_from_env +from layman_settings_util import read_clients_dict_from_env, validate_layman_role_service_uri import micka @@ -110,6 +110,7 @@ class EnumWfsWmsStatus(Enum): # Name of schema, where Layman maintains internal GS JDBC Role Service. LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA = '_role_service' LAYMAN_ROLE_SERVICE_URI = os.environ['LAYMAN_ROLE_SERVICE_URI'] +validate_layman_role_service_uri(LAYMAN_ROLE_SERVICE_URI, 'LAYMAN_ROLE_SERVICE_URI') LAYMAN_ROLE_SERVICE_SCHEMA = parse_qs(urlparse(LAYMAN_ROLE_SERVICE_URI).query)['schema'][0] LAYMAN_GS_USER_GROUP_SERVICE = os.getenv('LAYMAN_GS_USER_GROUP_SERVICE', '') or 'default' @@ -124,7 +125,7 @@ class EnumWfsWmsStatus(Enum): LAYMAN_GS_PATH = '/geoserver/' LAYMAN_GS_URL = f"http://{LAYMAN_GS_HOST}:{LAYMAN_GS_PORT}{LAYMAN_GS_PATH}" -geoserver.set_settings(LAYMAN_GS_URL, LAYMAN_GS_ROLE_SERVICE, LAYMAN_GS_USER_GROUP_SERVICE, DEFAULT_CONNECTION_TIMEOUT, ) +geoserver.set_settings(LAYMAN_GS_URL, LAYMAN_GS_ROLE_SERVICE, LAYMAN_GS_USER_GROUP_SERVICE, DEFAULT_CONNECTION_TIMEOUT) geoserver.GS_AUTH = LAYMAN_GS_AUTH LAYMAN_GS_ROLE = os.environ['LAYMAN_GS_ROLE'] diff --git a/src/layman_settings_util.py b/src/layman_settings_util.py index 90ef1208a..645f8d2ad 100644 --- a/src/layman_settings_util.py +++ b/src/layman_settings_util.py @@ -1,4 +1,5 @@ import os +from urllib import parse def read_clients_dict_from_env(): @@ -16,3 +17,16 @@ def read_clients_dict_from_env(): }) idx += 1 return client_dicts + + +def validate_layman_role_service_uri(layman_role_service_uri_str, env_name): + uri = parse.urlparse(layman_role_service_uri_str) + exp_uri_schemes = {'postgresql'} + assert uri.scheme in exp_uri_schemes, \ + f"{env_name} must have one of URL schemes {exp_uri_schemes}, but '{uri.scheme}' was found." + + assert uri.hostname, f"{env_name} must have explicit `host` part." + + query = parse.parse_qs(uri.query) + schema = query.pop('schema', [None])[0] + assert schema, f"{env_name} must have query parameter `schema`." diff --git a/src/wait_for_deps.py b/src/wait_for_deps.py index 056f0051e..9825ac86d 100644 --- a/src/wait_for_deps.py +++ b/src/wait_for_deps.py @@ -5,7 +5,7 @@ import geoserver import layman_settings as settings - +from geoserver.role_service import check_jdbc_role_service ATTEMPT_INTERVAL = 2 MAX_ATTEMPTS = 60 @@ -48,6 +48,11 @@ def main(): attempt += 1 print() + # Check PostgreSQL role service and stops immediately in case of any error + print(f"Checking PostgreSQL role service") + check_jdbc_role_service(settings.LAYMAN_ROLE_SERVICE_URI, settings.LAYMAN_ROLE_SERVICE_SCHEMA) + print() + # QGIS Server wait_for_msg = f"QGIS Server, url={settings.LAYMAN_QGIS_URL}" print(f"Waiting for {wait_for_msg}")