[TEMP] Enable to use SQLite3 in v1_23_change_oauth2_sub_username_to_user_id.py

.env.dev

@@ -91,7 +91,8 @@ OAUTH2_AUTH_URL=http://localhost:8083/o/authorize
 OAUTH2_TOKEN_URL=http://wagtail:8000/o/token/
 OAUTH2_CALLBACK_URL=http://localhost:3000/client/authn/oauth2-provider/callback
 OAUTH2_INTROSPECTION_URL=http://wagtail:8000/o/introspect/
-OAUTH2_INTROSPECTION_SUB_KEY=username
+OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE=true
+OAUTH2_INTROSPECTION_SUB_KEY=userId
 OAUTH2_USER_PROFILE_URL=http://wagtail:8000/profile
 
 

CHANGELOG.md

@@ -9,10 +9,30 @@
   ```
 - Stop using environment variable `LAYMAN_GS_ROLE_SERVICE`, it has no effect to Layman anymore. Layman now uses [role service](doc/security.md#role-service) identified by new environment variable [LAYMAN_ROLE_SERVICE_URI](doc/env-settings.md#LAYMAN_ROLE_SERVICE_URI). The service is called `layman_role_service` on GeoServer.
 - Set new environment variable [LAYMAN_ROLE_SERVICE_URI](doc/env-settings.md#LAYMAN_ROLE_SERVICE_URI)
+- If you are using Wagtail as OAuth2 provider
+  - Set new environment variable [OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE](doc/env-settings.md#OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE):
+    ```
+    OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE=true
+    ```  
+  - Change environment variable [OAUTH2_INTROSPECTION_SUB_KEY](doc/env-settings.md#OAUTH2_INTROSPECTION_SUB_KEY):
+    ```
+    OAUTH2_INTROSPECTION_SUB_KEY=userId
+    ```
+  - After running `make upgrade-demo` or `make-upgrade-demo-full`, run also script `v1_23_change_oauth2_sub_username_to_user_id.py`:
+    ```bash
+    docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root -e LAYMAN_WAGTAIL_DB_URI=<URI_of_Wagtail_db> layman bash -c "cd src && python3 -B v1_23_change_oauth2_sub_username_to_user_id.py"
+    ```
+      - `URI_of_Wagtail_db` is PostgreSQL connection URI to Wagtail database, e.g. `postgresql://user:[email protected]:5432/wagtail_db_name`
+      - The script changes OAuth2 "sub" values in Layman prime DB schema from Wagtail usernames to Wagtail user IDs. See [940](https://github.com/LayerManager/layman/issues/940).
+    - In case of development settings, run following script instead:
+      ```bash
+      docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root -e LAYMAN_WAGTAIL_DB_URI=<URI_of_Wagtail_db> layman_dev bash -c "cd src && python3 -B v1_23_change_oauth2_sub_username_to_user_id.py"
+      ```
 ### Migrations and checks
 #### Schema migrations
 - [#165](https://github.com/LayerManager/layman/issues/165) Add column `role_name` to table `rights` in prime DB schema. Add constraint that exactly one of columns `role_name` and `id_user` is not null.
 - [#165](https://github.com/LayerManager/layman/issues/165) Create DB schema `_role_service` that can be used as [role service](doc/security.md#role-service).
+- [#165](https://github.com/LayerManager/layman/issues/165) Column `name` in table `workspace` in prime DB schema length is changed to 59 characters.
 #### Data migrations
 - [#165](https://github.com/LayerManager/layman/issues/165) Delete technical roles and user-role relations in GeoServer `default` role service, which is now replaced by JDBC role service.
 ### Changes
@@ -36,6 +56,9 @@
   - [GET](doc/rest.md#get-workspace-map)/[PATCH](doc/rest.md#patch-workspace-map) Workspace Map
   - GET Workspace [Layers](doc/rest.md#get-workspace-layers)/[Maps](doc/rest.md#get-workspace-maps)
   - GET [Layers](doc/rest.md#get-layers)/[Maps](doc/rest.md#get-maps)/[Publications](doc/rest.md#get-publications)
+- [#165](https://github.com/LayerManager/layman/issues/165) Name of [users](doc/models.md#username) and [public workspaces](doc/models.md#public-workspace) are from now on restricted to a maximum length of 59 characters.
+- [940](https://github.com/LayerManager/layman/issues/940) Enable to use `userId` as OAuth2 "sub" instead of `username`. This is recommended option for Wagtail. See [OAUTH2_INTROSPECTION_SUB_KEY](doc/env-settings.md#OAUTH2_INTROSPECTION_SUB_KEY) for more details.
+- [941](https://github.com/LayerManager/layman/issues/941) Wagtail database is now persistent when restarting Layman or Wagtail.
 - All changes from [v1.22.1](#v1221), [v1.22.2](#v1222) and [v1.22.3](#v1223).
 - [#960](https://github.com/LayerManager/layman/issues/960) Handle WMS requests with HTTP error more efficiently in timgen.
 - [#962](https://github.com/LayerManager/layman/issues/962) Make values of `layman_metadata.publication_status` and `status` key(s) more consistent in responses of PATCH Workspace [Layer](doc/rest.md#patch-workspace-layer)/[Map](doc/rest.md#patch-workspace-map) and GET Workspace [Layer](doc/rest.md#get-workspace-layer)/[Map](doc/rest.md#get-workspace-map).

Makefile

@@ -22,7 +22,7 @@ start-demo-only:
 	docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps layman celery_worker flower timgen layman_client nginx
 
 start-demo-full-with-optional-deps:
-	mkdir -p layman_data deps/qgis/data
+	mkdir -p layman_data deps/qgis/data deps/wagtail/data
 	docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
 	docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
 	docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
@@ -60,7 +60,7 @@ deps-stop:
 	docker compose -f docker-compose.deps.yml stop
 
 start-dev:
-	mkdir -p layman_data layman_data_test tmp deps/qgis/data
+	mkdir -p layman_data layman_data_test tmp deps/qgis/data deps/wagtail/data
 	docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql
 	docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py"
 	docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up --force-recreate -d
@@ -107,7 +107,7 @@ upgrade-dev:
 	docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps layman_dev bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py"
 
 prepare-dirs:
-	mkdir -p layman_data layman_data_test tmp deps/qgis/data
+	mkdir -p layman_data layman_data_test tmp deps/qgis/data deps/wagtail/data
 
 build-dev:
 	docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml build --force-rm layman_dev
@@ -143,7 +143,7 @@ reset-data-directories:
 	docker compose -f docker-compose.deps.yml rm -fsv
 	docker volume rm layman_redis-data || true
 	sudo rm -rf layman_data layman_data_test deps/*/data
-	mkdir -p layman_data layman_data_test tmp deps/qgis/data
+	mkdir -p layman_data layman_data_test tmp deps/qgis/data deps/wagtail/data
 
 clear-python-cache-dev:
 	docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps layman_dev bash /code/src/clear-python-cache.sh
@@ -271,6 +271,9 @@ postgresql-psql:
 postgresql-psql-test:
 	docker compose -f docker-compose.deps.yml run -e PGPASSWORD=docker --entrypoint "psql -U docker -p 5432 -h postgresql layman_test" --rm postgresql
 
+postgresql-bash-exec:
+	docker compose -f docker-compose.deps.yml exec -e PGPASSWORD=docker postgresql "bash"
+
 redis-cli-db:
 	docker compose -f docker-compose.deps.yml exec redis redis-cli -h redis -p 6379 -n 0
 
@@ -313,11 +316,16 @@ wagtail-exec:
 	docker compose -f docker-compose.deps.yml exec wagtail bash
 
 wagtail-restart:
+	mkdir -p deps/wagtail/data
 	docker compose -f docker-compose.deps.yml up --force-recreate --no-deps -d wagtail
 
 wagtail-stop:
 	docker compose -f docker-compose.deps.yml stop wagtail
 
+wagtail-reset-datadir:
+	mkdir -p deps/wagtail/data
+	rm -rf deps/wagtail/data/*
+
 micka-restart:
 	docker compose -f docker-compose.deps.yml up --force-recreate --no-deps -d micka
 

deps/wagtail/docker/Dockerfile

@@ -21,15 +21,13 @@ RUN pip install -r /app/requirements.txt
 # Copy the entire project code.
 COPY laymanportal /app/
 
-# Prepare the app.
-RUN python manage.py migrate
+# Prepare the app: migrations (not needed as db.sqlite3a was generated after migrations)
+# RUN python manage.py migrate
+
+# Prepare the app: collect statis files
 RUN python manage.py collectstatic --noinput
 
 # Create a "coderedcms" user account to run the app.
 RUN useradd coderedcms
 RUN chown -R coderedcms /app/
 USER coderedcms
-
-# Finally, run the app on port 8000.
-EXPOSE 8000
-CMD exec waitress-serve --listen "*:8000" "laymanportal.wsgi:application"

deps/wagtail/laymanportal/laymanportal/settings/base.py

@@ -125,7 +125,7 @@
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.sqlite3",
-        "NAME": os.path.join(BASE_DIR, "db.sqlite3"),
+        "NAME": os.path.join(BASE_DIR, "data/db.sqlite3"),
     }
 }
 

deps/wagtail/laymanportal/start_wagtail.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+if [ ! -f /app/data/db.sqlite3 ]; then
+    echo "File db.sqlite3 not found, copying the default one."
+    cp /app/initial_data/db.sqlite3 /app/data/
+fi
+
+exec waitress-serve --listen "*:8000" "laymanportal.wsgi:application"

doc/env-settings.md

@@ -85,7 +85,10 @@ URL of LTC OAuth2 callback endpoint to be called after successful OAuth2 authori
 URL of OAuth2 Introspection endpoint.
 
 ### OAUTH2_INTROSPECTION_SUB_KEY
-Name of the key in OAuth2 introspection response whose value is OAuth2 subject (also known as "sub"). Value `username` is suitable for Wagtail. If not set or set to empty string, `sub` is used, that is suitable option for Liferay.
+Name of the key in OAuth2 introspection response whose value is OAuth2 subject (also known as "sub"). Value `userId` is suitable for Wagtail (together with setting [OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE](#OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE) to `true`). If not set or set to empty string, `sub` is used, that is suitable option for Liferay.
+
+### OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE
+Set to `true` if you want [OAUTH2_INTROSPECTION_SUB_KEY](#OAUTH2_INTROSPECTION_SUB_KEY) to be read from [OAUTH2_USER_PROFILE_URL](#OAUTH2_USER_PROFILE_URL) instead of [OAUTH2_INTROSPECTION_URL](#OAUTH2_INTROSPECTION_URL). Default value is `false`. Value `true` is suitable for Wagtail.
 
 ### OAUTH2_USER_PROFILE_URL
 URL of User Profile endpoint used to obtain user's ID, name, email, etc.

doc/models.md

@@ -73,7 +73,7 @@
 
 ## Username
 - Username is a string identifying one [user](#user), so it is unique among all users.
-- The string is lower-case (in contrast with [role name](#role)).
+- The string is lower-case (in contrast with [role name](#role)), maximum length is 59 characters.
 - Each user is represented by max. one username.
 - Username is also used to identify user's [personal workspace](#personal-workspace) when communicating with [Layman REST API](rest.md).
 - Username can be reserved by [PATCH Current User](rest.md#patch-current-user).
@@ -92,6 +92,7 @@
 ## Workspace
 - Workspace is folder for [publications](#publication).
 - Each workspace is identified by name that is unique among all workspaces.
+- The name is lower-case, maximum length is 59 characters.
 - Workspace name is sometimes used for structuring publication-related data. For example, it's part of REST API URL (`/rest/workspaces/<workspace_name>/...`), directory names (`<LAYMAN_DATA_DIR>/workspaces/<workspace_name>/...`), DB schemas, or OGC Web Services (`/geoserver/<workspace_name>/...`, `/geoserver/<workspace_name>_wms/...`).
 - Workspace's REST API consists of all [map and layer endpoints](rest.md) endpoints.
 - There are following types of workspaces:

doc/rest.md

@@ -834,7 +834,7 @@ Query parameters:
   - `true`: If `username` sent in body parameter is already reserved by another user or `username` is an empty string, layman will definitely reserve some `username`, preferably similar to the value sent in `username` body parameter or to one of claims.
 
 Body parameters:
-- *username*: String. [Username](models.md#username) that should be reserved for current user. Username can be reserved only once and it cannot be changed. See URL parameter `adjust_username` for other details.
+- *username*: String. [Username](models.md#username) that should be reserved for current user (maximum length is 59 characters). Username can be reserved only once and cannot be changed. See URL parameter `adjust_username` for other details.
 
 #### Response
 Content-Type: `application/json`

docker-compose.deps.yml

@@ -67,6 +67,10 @@ services:
         dockerfile: docker/Dockerfile
       ports:
         - 8083:8000
+      command: /app/start_wagtail.sh
+      volumes:
+        - ./deps/wagtail/data:/app/data/
+        - ./deps/wagtail/sample:/app/initial_data/
 
    micka:
       container_name: micka

docker-compose.dev.yml

@@ -24,6 +24,7 @@ services:
         - ./deps/geoserver/data:/geoserver/data_dir
         - ./deps/geoserver/sample/geoserver_data:/geoserver/initial_data_dir
         - ./deps/qgis/data:/qgis/data
+        - ./deps/wagtail/data:/wagtail/data
       depends_on:
         - timgen
         - redis

src/db/util.py

@@ -1,5 +1,7 @@
+import contextlib
 import logging
 import re
+import sqlite3
 from urllib import parse
 import psycopg2
 import psycopg2.pool
@@ -35,6 +37,35 @@ def get_connection_pool(db_uri_str=None, encapsulate_exception=True):
 
 
 def run_query(query, data=None, uri_str=None, encapsulate_exception=True, log_query=False):
+    if uri_str is None or uri_str.startswith('postgres:') or uri_str.startswith('postgresql:'):
+        method = _run_query_postgres
+    elif uri_str.startswith('sqlite:'):
+        method = _run_query_sqlite
+    else:
+        raise NotImplementedError(f"Unsupported database protocol: {uri_str}")
+
+    return method(query, data=data, uri_str=uri_str, encapsulate_exception=encapsulate_exception, log_query=log_query)
+
+
+def _run_query_sqlite(query, data=None, *, uri_str, encapsulate_exception=True, log_query=False):
+    assert data is None, f"data is not yet implemented"
+    db_uri_parsed = parse.urlparse(uri_str)
+    db_path = db_uri_parsed.path
+    try:
+        if log_query:
+            logger.info(f"query={query}")
+        with contextlib.closing(sqlite3.connect(db_path)) as conn:  # auto-closes
+            with conn:  # auto-commits
+                result = list(conn.execute(query))
+    except BaseException as exc:
+        if encapsulate_exception:
+            logger.error(f"_run_query_sqlite, query={query}, data={data}, exc={exc}")
+            raise Error(2) from exc
+        raise exc
+    return result
+
+
+def _run_query_postgres(query, data=None, uri_str=None, encapsulate_exception=True, log_query=False):
     pool = get_connection_pool(db_uri_str=uri_str, encapsulate_exception=encapsulate_exception, )
     conn = pool.getconn()
     conn.autocommit = True
@@ -47,7 +78,7 @@ def run_query(query, data=None, uri_str=None, encapsulate_exception=True, log_qu
         conn.commit()
     except BaseException as exc:
         if encapsulate_exception:
-            logger.error(f"run_query, query={query}, data={data}, exc={exc}")
+            logger.error(f"_run_query_postgres, query={query}, data={data}, exc={exc}")
             raise Error(2) from exc
         raise exc
     finally:

src/layman/authn/oauth2/__init__.py

@@ -72,6 +72,14 @@ def authenticate():
                 # current_app.logger.info(f"r_json={r_json}")
                 if r_json['active'] is True and r_json.get('token_type', 'Bearer') == 'Bearer':
                     valid_resp = r_json
+                    if settings.OAUTH2_INTROSPECTION_USE_SUB_KEY_FROM_USER_PROFILE:
+                        response = requests.get(USER_PROFILE_URL, headers={
+                            'Authorization': f'Bearer {access_token}',
+                        }, timeout=settings.DEFAULT_CONNECTION_TIMEOUT)
+                        response.raise_for_status()
+                        user_profile_json = response.json()
+                        # current_app.logger.info(f"user_profile_json={user_profile_json}")
+                        valid_resp[INTROSPECTION_SUB_KEY] = user_profile_json[INTROSPECTION_SUB_KEY]
                     break
             except ValueError:
                 continue

src/layman/authn/oauth2_test.py

@@ -217,15 +217,6 @@ def test_get_current_user_anonymous(client):
     assert claims['nickname'] == 'Anonymous', claims
 
 
-@pytest.mark.usefixtures('app_context')
-def test_patch_current_user_anonymous(client):
-    rest_path = url_for('rest_current_user.patch')
-    response = client.patch(rest_path)
-    assert response.status_code == 403
-    resp_json = response.get_json()
-    assert resp_json['code'] == 30
-
-
 @pytest.mark.usefixtures('active_token_introspection_url', 'user_profile_url', 'ensure_layman')
 def test_patch_current_user_without_username():
 

src/layman/common/prime_db_schema/__init__.py

@@ -1,4 +1,5 @@
 from layman.common.prime_db_schema import users as users_util, workspaces as workspaces_util
+from layman.http import LaymanError
 
 get_usernames = users_util.get_usernames
 get_workspaces = workspaces_util.get_workspace_names
@@ -26,4 +27,6 @@ def ensure_workspace(workspace):
 
 
 def check_workspace_name(workspace):
+    if len(workspace) > 59:
+        raise LaymanError(56)
     workspaces_util.check_workspace_name(workspace)

src/layman/error_list.py

@@ -56,4 +56,5 @@
     53: (500, 'Error when publishing on GeoServer. It happens for example for raster files with wrong explicit CRS.'),
     54: (400, 'Wrong header value'),
     55: (400, 'Publication is not complete'),  # raised by process_client only
+    56: (403, 'Username or workspace name is too long. Maximum is 59 characters.')
 }

src/layman/upgrade/__init__.py

@@ -43,6 +43,7 @@
         ]),
         ((1, 23, 0), [
             upgrade_v1_23.adjust_db_for_roles,
+            upgrade_v1_23.restrict_workspace_name_length,
             upgrade_v1_23.create_role_service_schema,
         ]),
     ],

src/layman/upgrade/upgrade_v1_23.py

@@ -168,3 +168,22 @@ def delete_user_roles():
     role_not_exists = response.status_code == 404
     if not role_not_exists:
         response.raise_for_status()
+
+
+def restrict_workspace_name_length():
+    logger.info(f'    Restrict workspace name length')
+
+    select_too_long = f"""
+select name
+from {settings.LAYMAN_PRIME_SCHEMA}.workspaces
+where length(name) > 59
+;"""
+    too_long_workspace_name = db_util.run_query(select_too_long)
+    if len(too_long_workspace_name) > 0:
+        raise NotImplementedError(f"Too long workspace names: {[name[0] for name in too_long_workspace_name]}")
+
+    alter_column = f"""
+ALTER TABLE {settings.LAYMAN_PRIME_SCHEMA}.workspaces
+    ALTER COLUMN name TYPE VARCHAR(59) COLLATE pg_catalog."default"
+;"""
+    db_util.run_statement(alter_column)