diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f9912e98ab..d450961c81 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -80,6 +80,9 @@ jobs: - name: Environment information run: c2cciutils-env + env: + # To display the content of the github object + GITHUB_EVENT: ${{ toJson(github) }} - run: python3 -m pip install --user --requirement=requirements.txt @@ -90,14 +93,43 @@ jobs: if: always() - run: make build-config - # Build and lint QGIS images + # Build and lint QGIS images - run: QGIS_VERSION=3.34-gdal3.8 make build-qgisserver - run: QGIS_VERSION=3.34-gdal3.8 make prospector-qgisserver - # Tests - run: make preparetest - - run: c2cciutils-docker-logs + + # Auto generate Alembic script + - run: | + docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=main check \ + || docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=main \ + revision --autogenerate --message='${{ github.event.pull_request.title }}' + docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=static check \ + || docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=static \ + revision --autogenerate --message='${{ github.event.pull_request.title }}' + sudo chmod go+rw -R commons/c2cgeoportal_commons/alembic/ + # Replace 'main' by schema and 'main. by f'{schema}. + sed -i "s/'main'/schema/g" commons/c2cgeoportal_commons/alembic/*/*.py + sed -i "s/'main\.'/f'{schema}./g" commons/c2cgeoportal_commons/alembic/*/*.py + # Replace 'main_static' by staticschema and 'main_static. by f'{staticschema}. + sed -i "s/'main_static'/staticschema/g" commons/c2cgeoportal_commons/alembic/*/*.py + sed -i "s/'main_static\.'/f'{staticschema}./g" commons/c2cgeoportal_commons/alembic/*/*.py + git checkout commons/c2cgeoportal_commons/alembic/main/ee25d267bf46_main_interface_desktop.py + git checkout commons/c2cgeoportal_commons/alembic/main/415746eb9f6_changes_for_v2.py + git add commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py + pre-commit run --files commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py || true + git add commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py + git diff --staged --patch > /tmp/alembic.patch + git diff --staged --exit-code + - uses: actions/upload-artifact@v4 + with: + name: Add Alembic upgrade script.patch + path: /tmp/alembic.patch + retention-days: 1 if: failure() + + # Tests + - run: c2cciutils-docker-logs - run: make tests-commons - run: c2cciutils-docker-logs - run: make tests-geoportal @@ -105,7 +137,6 @@ jobs: - run: make tests-admin - run: c2cciutils-docker-logs - run: make tests-qgisserver - - run: c2cciutils-docker-logs - run: c2cciutils-docker-logs if: always() - run: docker compose down diff --git a/admin/c2cgeoportal_admin/routes.py b/admin/c2cgeoportal_admin/routes.py index 235d413f0c..3e25ea2d63 100644 --- a/admin/c2cgeoportal_admin/routes.py +++ b/admin/c2cgeoportal_admin/routes.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2023, Camptocamp SA +# Copyright (c) 2017-2024, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -57,6 +57,7 @@ def includeme(config): from c2cgeoportal_commons.models.main import ( # pylint: disable=import-outside-toplevel Functionality, Interface, + LayerCOG, LayerGroup, LayerVectorTiles, LayerWMS, @@ -78,6 +79,7 @@ def includeme(config): ("layers_wms", LayerWMS), ("layers_wmts", LayerWMTS), ("layers_vectortiles", LayerVectorTiles), + ("layers_cog", LayerCOG), ("ogc_servers", OGCServer), ("restriction_areas", RestrictionArea), ("users", User), diff --git a/admin/c2cgeoportal_admin/views/layers_cog.py b/admin/c2cgeoportal_admin/views/layers_cog.py new file mode 100644 index 0000000000..e7214d5e64 --- /dev/null +++ b/admin/c2cgeoportal_admin/views/layers_cog.py @@ -0,0 +1,135 @@ +# Copyright (c) 2017-2024, Camptocamp SA +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + + +from functools import partial +from typing import Optional, cast + +import sqlalchemy +import sqlalchemy.orm.query +from c2cgeoform.schema import GeoFormSchemaNode +from c2cgeoform.views.abstract_views import ( + DeleteResponse, + GridResponse, + IndexResponse, + ListField, + ObjectResponse, + SaveResponse, +) +from deform.widget import FormWidget +from pyramid.httpexceptions import HTTPNotFound +from pyramid.view import view_config, view_defaults + +from c2cgeoportal_admin import _ +from c2cgeoportal_admin.schemas.interfaces import interfaces_schema_node +from c2cgeoportal_admin.schemas.metadata import metadata_schema_node +from c2cgeoportal_admin.schemas.restriction_areas import restrictionareas_schema_node +from c2cgeoportal_admin.schemas.treeitem import parent_id_node +from c2cgeoportal_admin.views.layers import LayerViews +from c2cgeoportal_commons.lib.literal import Literal +from c2cgeoportal_commons.models.main import LayerCOG, LayerGroup + +_list_field = partial(ListField, LayerCOG) # type: ignore[var-annotated] + + +base_schema = GeoFormSchemaNode(LayerCOG, widget=FormWidget(fields_template="layer_fields")) +base_schema.add(metadata_schema_node(LayerCOG.metadatas, LayerCOG)) +base_schema.add(interfaces_schema_node(LayerCOG.interfaces)) +base_schema.add(restrictionareas_schema_node(LayerCOG.restrictionareas)) +base_schema.add_unique_validator(LayerCOG.name, LayerCOG.id) +base_schema.add(parent_id_node(LayerGroup)) + + +@view_defaults(match_param="table=layers_cog") +class LayerCOGViews(LayerViews[LayerCOG]): + """The vector tiles administration view.""" + + _list_fields = ( + LayerViews._list_fields # typer: ignore[misc] # pylint: disable=protected-access + + [_list_field("url")] + + LayerViews._extra_list_fields # pylint: disable=protected-access + ) + + _id_field = "id" + _model = LayerCOG + _base_schema = base_schema + + def _base_query(self) -> sqlalchemy.orm.query.Query[LayerCOG]: + return super()._sub_query(self._request.dbsession.query(LayerCOG).distinct()) + + def _sub_query( + self, query: Optional[sqlalchemy.orm.query.Query[LayerCOG]] + ) -> sqlalchemy.orm.query.Query[LayerCOG]: + del query + return self._base_query() + + @view_config(route_name="c2cgeoform_index", renderer="../templates/index.jinja2") # type: ignore[misc] + def index(self) -> IndexResponse: + return super().index() + + @view_config(route_name="c2cgeoform_grid", renderer="fast_json") # type: ignore[misc] + def grid(self) -> GridResponse: + return super().grid() + + def schema(self) -> GeoFormSchemaNode: + try: + obj = self._get_object() + except HTTPNotFound: + obj = None + + schema = cast(GeoFormSchemaNode, self._base_schema.clone()) + if obj is not None: + schema["url"].description = Literal( + _("{}
Current runtime value is: {}").format( + schema["url"].description, + obj.url_description(self._request), + ) + ) + return schema + + @view_config(route_name="c2cgeoform_item", request_method="GET", renderer="../templates/edit.jinja2") # type: ignore[misc] + def view(self) -> ObjectResponse: + if self._is_new(): + dbsession = self._request.dbsession + default_cog = LayerCOG.get_default(dbsession) + if default_cog: + return self.copy(default_cog, excludes=["name", "url"]) + return super().edit() + + @view_config(route_name="c2cgeoform_item", request_method="POST", renderer="../templates/edit.jinja2") # type: ignore[misc] + def save(self) -> SaveResponse: + return super().save() + + @view_config(route_name="c2cgeoform_item", request_method="DELETE", renderer="fast_json") # type: ignore[misc] + def delete(self) -> DeleteResponse: + return super().delete() + + @view_config( # type: ignore[misc] + route_name="c2cgeoform_item_duplicate", request_method="GET", renderer="../templates/edit.jinja2" + ) + def duplicate(self) -> ObjectResponse: + return super().duplicate() diff --git a/admin/tests/__init__.py b/admin/tests/__init__.py index d25016e993..2fdfcf47c8 100644 --- a/admin/tests/__init__.py +++ b/admin/tests/__init__.py @@ -9,13 +9,15 @@ def get_test_default_layers(dbsession, default_ogc_server): - from c2cgeoportal_commons.models.main import LayerVectorTiles, LayerWMS, LayerWMTS - - default_wms = LayerWMS("wms-defaults") - default_wms.ogc_server = default_ogc_server - default_wms.time_widget = "datepicker" - default_wms.time_mode = "value" - dbsession.add(default_wms) + from c2cgeoportal_commons.models.main import LayerCOG, LayerVectorTiles, LayerWMS, LayerWMTS + + default_wms = None + if default_ogc_server: + default_wms = LayerWMS("wms-defaults") + default_wms.ogc_server = default_ogc_server + default_wms.time_widget = "datepicker" + default_wms.time_mode = "value" + dbsession.add(default_wms) default_wmts = LayerWMTS("wmts-defaults") default_wmts.url = "https:///wmts.geo.admin_default.ch.org?service=wms&request=GetCapabilities" default_wmts.layer = "default" @@ -24,8 +26,11 @@ def get_test_default_layers(dbsession, default_ogc_server): default_vectortiles = LayerVectorTiles("vectortiles-defaults") default_vectortiles.style = "https://vectortiles-staging.geoportail.lu/styles/roadmap/style.json" dbsession.add(default_vectortiles) + default_cog = LayerCOG("cog-defaults") + default_cog.url = "https://example.com/image.tiff" + dbsession.add(default_cog) dbsession.flush() - return {"wms": default_wms, "wmts": default_wmts, "vectortiles": default_vectortiles} + return {"wms": default_wms, "wmts": default_wmts, "vectortiles": default_vectortiles, "cog": default_cog} def factory_build_layers(layer_builder, dbsession, add_dimension=True): diff --git a/admin/tests/conftest.py b/admin/tests/conftest.py index d490599c02..dc32b465aa 100644 --- a/admin/tests/conftest.py +++ b/admin/tests/conftest.py @@ -1,9 +1,15 @@ +from typing import Any + +import pyramid.request import pytest import sqlalchemy.exc import transaction from pyramid import testing from pyramid.paster import bootstrap +from pyramid.router import Router +from pyramid.scripting import AppEnvironment from sqlalchemy.exc import DBAPIError +from sqlalchemy.orm import Session, SessionTransaction from webtest import TestApp as WebTestApp # Avoid warning with pytest from c2cgeoportal_commons.testing import generate_mappers, get_engine, get_session_factory, get_tm_session @@ -12,7 +18,7 @@ @pytest.fixture(scope="session") @pytest.mark.usefixtures("settings") -def dbsession(settings): +def dbsession(settings: dict[str, Any]) -> Session: generate_mappers() engine = get_engine(settings) session_factory = get_session_factory(engine) @@ -23,7 +29,7 @@ def dbsession(settings): @pytest.fixture(scope="function") @pytest.mark.usefixtures("dbsession") -def transact(dbsession): +def transact(dbsession: Session) -> SessionTransaction: t = dbsession.begin_nested() yield t try: @@ -33,13 +39,13 @@ def transact(dbsession): dbsession.expire_all() -def raise_db_error(_): +def raise_db_error(_: Any) -> None: raise DBAPIError("this is a test !", None, None) @pytest.fixture(scope="function") @pytest.mark.usefixtures("dbsession") -def raise_db_error_on_query(dbsession): +def raise_db_error_on_query(dbsession: Session) -> None: query = dbsession.query dbsession.query = raise_db_error yield @@ -47,7 +53,7 @@ def raise_db_error_on_query(dbsession): @pytest.fixture(scope="session") -def app_env(): +def app_env() -> AppEnvironment: file_name = "/opt/c2cgeoportal/admin/tests/tests.ini" with bootstrap(file_name) as env: yield env @@ -55,7 +61,7 @@ def app_env(): @pytest.fixture(scope="session") @pytest.mark.usefixtures("app_env", "dbsession") -def app(app_env, dbsession): +def app(app_env: AppEnvironment, dbsession: Session) -> Router: config = testing.setUp(registry=app_env["registry"]) config.add_request_method(lambda request: dbsession, "dbsession", reify=True) config.add_route("user_add", "user_add") @@ -68,12 +74,12 @@ def app(app_env, dbsession): @pytest.fixture(scope="session") @pytest.mark.usefixtures("app_env") -def settings(app_env): +def settings(app_env: AppEnvironment) -> Any: yield app_env.get("registry").settings @pytest.fixture(scope="session") # noqa: ignore=F811 @pytest.mark.usefixtures("app") -def test_app(request, app): +def test_app(request: pyramid.request.Request, app: Router) -> WebTestApp: testapp = WebTestApp(app) yield testapp diff --git a/admin/tests/test_layers_cog.py b/admin/tests/test_layers_cog.py new file mode 100644 index 0000000000..a89f013634 --- /dev/null +++ b/admin/tests/test_layers_cog.py @@ -0,0 +1,243 @@ +# pylint: disable=no-self-use + +import re +from typing import Any + +import pytest +from sqlalchemy.orm import Session, SessionTransaction +from webtest import TestApp as WebTestApp # Avoid warning with pytest + +from . import AbstractViewsTests, factory_build_layers, get_test_default_layers + + +@pytest.fixture(scope="function") +@pytest.mark.usefixtures("dbsession", "transact") +def layer_cog_test_data(dbsession: Session, transact: SessionTransaction) -> dict[str, Any]: + del transact + + from c2cgeoportal_commons.models.main import LayerCOG + + def layer_builder(i: int) -> LayerCOG: + name = f"layer_cog_{i}" + layer = LayerCOG(name=name) + layer.public = 1 == i % 2 + layer.url = "https://example.com/image.tiff" + return layer + + data = factory_build_layers(layer_builder, dbsession, add_dimension=False) + data["default"] = get_test_default_layers(dbsession, None) + + dbsession.flush() + + yield data + + +@pytest.mark.usefixtures("layer_cog_test_data", "test_app") +class TestLayerVectortiles(AbstractViewsTests): + _prefix = "/admin/layers_cog" + + def test_index_rendering(self, test_app: WebTestApp) -> None: + resp = self.get(test_app) + + self.check_left_menu(resp, "COG Layers") + + expected = [ + ("actions", "", "false"), + ("id", "id", "true"), + ("name", "Name"), + ("description", "Description"), + ("public", "Public"), + ("geo_table", "Geo table"), + ("exclude_properties", "Exclude properties"), + ("url", "URL"), + ("interfaces", "Interfaces"), + ("restrictionareas", "Restriction areas", "false"), + ("parents_relation", "Parents", "false"), + ("metadatas", "Metadatas", "false"), + ] + self.check_grid_headers(resp, expected) + + def test_grid_complex_column_val(self, test_app: WebTestApp, layer_cog_test_data: dict[str, Any]) -> None: + json = self.check_search(test_app, search="layer", sort="name") + + row = json["rows"][0] + layer = layer_cog_test_data["layers"][0] + + assert layer.name == row["name"] + assert layer.id == int(row["_id_"]) + + def test_new(self, test_app: WebTestApp, layer_cog_test_data: dict[str, Any], dbsession: Session) -> None: + default_cog = layer_cog_test_data["default"]["cog"] + default_cog.name = "so can I not be found" + dbsession.flush() + + form = self.get_item(test_app, "new").form + + assert "" == self.get_first_field_named(form, "name").value + assert "" == self.get_first_field_named(form, "id").value + assert "" == self.get_first_field_named(form, "url").value + + def test_grid_search(self, test_app: WebTestApp) -> None: + self.check_search(test_app, "layer_cog_10", total=1) + + def test_base_edit(self, test_app: WebTestApp, layer_cog_test_data: dict[str, Any]) -> None: + layer = layer_cog_test_data["layers"][10] + + form = self.get_item(test_app, layer.id).form + + assert "layer_cog_10" == self.get_first_field_named(form, "name").value + assert "" == self.get_first_field_named(form, "description").value + + def test_public_checkbox_edit(self, test_app: WebTestApp, layer_cog_test_data: dict[str, Any]) -> None: + layer = layer_cog_test_data["layers"][10] + form = self.get_item(test_app, layer.id).form + assert not form["public"].checked + + layer = layer_cog_test_data["layers"][11] + form = self.get_item(test_app, layer.id).form + assert form["public"].checked + + def test_edit( + self, test_app: WebTestApp, layer_cog_test_data: dict[str, Any], dbsession: Session + ) -> None: + from c2cgeoportal_commons.models.main import Log, LogAction + + layer = layer_cog_test_data["layers"][0] + + form = self.get_item(test_app, layer.id).form + + assert str(layer.id) == self.get_first_field_named(form, "id").value + assert "hidden" == self.get_first_field_named(form, "id").attrs["type"] + assert layer.name == self.get_first_field_named(form, "name").value + assert str(layer.description or "") == self.get_first_field_named(form, "description").value + assert layer.public is False + assert layer.public == form["public"].checked + assert str(layer.geo_table or "") == form["geo_table"].value + assert str(layer.exclude_properties or "") == form["exclude_properties"].value + # assert str(layer.url or "") == form["url"].value + + interfaces = layer_cog_test_data["interfaces"] + assert {interfaces[0].id, interfaces[2].id} == {i.id for i in layer.interfaces} + self._check_interfaces(form, interfaces, layer) + + ras = layer_cog_test_data["restrictionareas"] + assert {ras[0].id, ras[2].id} == {i.id for i in layer.restrictionareas} + self._check_restrictionsareas(form, ras, layer) + + new_values = { + "name": "new_name", + "description": "new description", + "public": True, + "geo_table": "new_geo_table", + "exclude_properties": "property1,property2", + "url": "https://example.com/image.tiff", + } + + for key, value in new_values.items(): + self.set_first_field_named(form, key, value) + form["interfaces"] = [interfaces[1].id, interfaces[3].id] + form["restrictionareas"] = [ras[1].id, ras[3].id] + + resp = form.submit("submit") + assert str(layer.id) == re.match( + rf"http://localhost{self._prefix}/(.*)\?msg_col=submit_ok", resp.location + ).group(1) + + dbsession.expire(layer) + for key, value in new_values.items(): + if isinstance(value, bool): + assert value == getattr(layer, key) + else: + assert str(value or "") == str(getattr(layer, key) or "") + assert {interfaces[1].id, interfaces[3].id} == {interface.id for interface in layer.interfaces} + assert {ras[1].id, ras[3].id} == {ra.id for ra in layer.restrictionareas} + + log = dbsession.query(Log).one() + assert log.date != None + assert log.action == LogAction.UPDATE + assert log.element_type == "layer_cog" + assert log.element_id == layer.id + assert log.element_name == layer.name + assert log.username == "test_user" + + def test_submit_new( + self, dbsession: Session, test_app: WebTestApp, layer_cog_test_data: dict[str, Any] + ) -> None: + from c2cgeoportal_commons.models.main import LayerCOG, Log, LogAction + + resp = test_app.post( + "/admin/layers_cog/new", + { + "name": "new_name", + "description": "new description", + "public": True, + "url": "https://example.com/image.tiff", + }, + status=302, + ) + + layer = dbsession.query(LayerCOG).filter(LayerCOG.name == "new_name").one() + assert str(layer.id) == re.match( + r"http://localhost/admin/layers_cog/(.*)\?msg_col=submit_ok", resp.location + ).group(1) + + log = dbsession.query(Log).one() + assert log.date != None + assert log.action == LogAction.INSERT + assert log.element_type == "layer_cog" + assert log.element_id == layer.id + assert log.element_name == layer.name + assert log.username == "test_user" + + def test_duplicate( + self, layer_cog_test_data: dict[str, Any], test_app: WebTestApp, dbsession: Session + ) -> None: + from c2cgeoportal_commons.models.main import LayerCOG + + layer = layer_cog_test_data["layers"][3] + + resp = test_app.get(f"/admin/layers_cog/{layer.id}/duplicate", status=200) + form = resp.form + + assert "" == self.get_first_field_named(form, "id").value + assert layer.name == self.get_first_field_named(form, "name").value + assert str(layer.description or "") == self.get_first_field_named(form, "description").value + assert layer.public is True + assert layer.public == form["public"].checked + assert str(layer.geo_table or "") == form["geo_table"].value + assert str(layer.exclude_properties or "") == form["exclude_properties"].value + # assert str(layer.url or "") == form["url"].value + interfaces = layer_cog_test_data["interfaces"] + assert {interfaces[3].id, interfaces[1].id} == {i.id for i in layer.interfaces} + self._check_interfaces(form, interfaces, layer) + + self.set_first_field_named(form, "name", "clone") + resp = form.submit("submit") + + layer = dbsession.query(LayerCOG).filter(LayerCOG.name == "clone").one() + assert str(layer.id) == re.match( + r"http://localhost/admin/layers_cog/(.*)\?msg_col=submit_ok", resp.location + ).group(1) + + assert layer.id == layer.metadatas[0].item_id + assert layer_cog_test_data["layers"][3].metadatas[0].name == layer.metadatas[0].name + assert layer_cog_test_data["layers"][3].metadatas[1].name == layer.metadatas[1].name + + def test_delete(self, test_app: WebTestApp, dbsession: Session) -> None: + from c2cgeoportal_commons.models.main import Layer, LayerCOG, Log, LogAction, TreeItem + + layer = dbsession.query(LayerCOG).first() + + test_app.delete(f"/admin/layers_cog/{layer.id}", status=200) + + assert dbsession.query(LayerCOG).get(layer.id) is None + assert dbsession.query(Layer).get(layer.id) is None + assert dbsession.query(TreeItem).get(layer.id) is None + + log = dbsession.query(Log).one() + assert log.date != None + assert log.action == LogAction.DELETE + assert log.element_type == "layer_cog" + assert log.element_id == layer.id + assert log.element_name == layer.name + assert log.username == "test_user" diff --git a/admin/tests/test_layers_vectortiles.py b/admin/tests/test_layers_vectortiles.py index a5e1083731..6e1c757284 100644 --- a/admin/tests/test_layers_vectortiles.py +++ b/admin/tests/test_layers_vectortiles.py @@ -12,12 +12,7 @@ def layer_vectortiles_test_data(dbsession, transact): del transact - from c2cgeoportal_commons.models.main import LayerVectorTiles, OGCServer - - servers = [OGCServer(name=f"server_{i}") for i in range(0, 4)] - for i, server in enumerate(servers): - server.url = f"http://wms.geo.admin.ch_{i}" - server.image_type = "image/jpeg" if i % 2 == 0 else "image/png" + from c2cgeoportal_commons.models.main import LayerVectorTiles def layer_builder(i): name = f"layer_vectortiles_{i}" @@ -29,7 +24,7 @@ def layer_builder(i): return layer data = factory_build_layers(layer_builder, dbsession) - data["default"] = get_test_default_layers(dbsession, server) + data["default"] = get_test_default_layers(dbsession, None) dbsession.flush() diff --git a/commons/c2cgeoportal_commons/alembic/main/a4558f032d7d_add_support_of_cog_layers.py b/commons/c2cgeoportal_commons/alembic/main/a4558f032d7d_add_support_of_cog_layers.py new file mode 100644 index 0000000000..e61a74a098 --- /dev/null +++ b/commons/c2cgeoportal_commons/alembic/main/a4558f032d7d_add_support_of_cog_layers.py @@ -0,0 +1,72 @@ +# Copyright (c) 2024, Camptocamp SA +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +""" +Add support of COG layers. + +Revision ID: a4558f032d7d +Revises: b6b09f414fe8 +Create Date: 2024-04-22 12:22:09.336641 +""" + +import sqlalchemy as sa +from alembic import op +from c2c.template.config import config + +# revision identifiers, used by Alembic. +revision = "a4558f032d7d" +down_revision = "b6b09f414fe8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade.""" + schema = config["schema"] + + # ### commands auto generated by Alembic ### + op.create_table( + "layer_cog", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Unicode(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["main.layer.id"], + ), + sa.PrimaryKeyConstraint("id"), + schema=schema, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade.""" + schema = config["schema"] + + # ### commands auto generated by Alembic ### + op.drop_table("layer_cog", schema=schema) + # ### end Alembic commands ### diff --git a/commons/c2cgeoportal_commons/alembic/main/b6b09f414fe8_sync_model_database.py b/commons/c2cgeoportal_commons/alembic/main/b6b09f414fe8_sync_model_database.py new file mode 100644 index 0000000000..5bedcd12d3 --- /dev/null +++ b/commons/c2cgeoportal_commons/alembic/main/b6b09f414fe8_sync_model_database.py @@ -0,0 +1,163 @@ +# Copyright (c) 2024, Camptocamp SA +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +""" +Sync the model and the database. + +Revision ID: b6b09f414fe8 +Revises: 44c91d82d419 +Create Date: 2024-04-22 07:17:25.399062 +""" + +import sqlalchemy as sa +from alembic import op +from c2c.template.config import config + +# revision identifiers, used by Alembic. +revision = "b6b09f414fe8" +down_revision = "44c91d82d419" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade.""" + schema = config["schema"] + + # names are required + op.alter_column("dimension", "name", existing_type=sa.VARCHAR(), nullable=False, schema=schema) + op.alter_column("interface", "name", existing_type=sa.VARCHAR(), nullable=False, schema=schema) + op.alter_column("restrictionarea", "name", existing_type=sa.VARCHAR(), nullable=False, schema=schema) + # Set a default value to boolean ind int + op.execute(f"UPDATE {schema}.layer SET public = true WHERE public IS NULL") + op.alter_column("layer", "public", existing_type=sa.BOOLEAN(), nullable=False, schema=schema) + op.execute(f"UPDATE {schema}.layergroup_treeitem SET ordering = 0 WHERE ordering IS NULL") + op.alter_column( + "layergroup_treeitem", "ordering", existing_type=sa.INTEGER(), nullable=False, schema=schema + ) + op.alter_column("metadata", "name", existing_type=sa.VARCHAR(), nullable=False, schema=schema) + op.execute(f"UPDATE {schema}.ogc_server SET wfs_support = false WHERE wfs_support IS NULL") + op.alter_column( + "ogc_server", + "wfs_support", + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default="false", + schema=schema, + ) + op.execute(f"UPDATE {schema}.ogc_server SET is_single_tile = false WHERE is_single_tile IS NULL") + op.alter_column( + "ogc_server", + "is_single_tile", + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default="false", + schema=schema, + ) + op.execute(f"UPDATE {schema}.restrictionarea SET readwrite = false WHERE readwrite IS NULL") + op.alter_column("restrictionarea", "readwrite", existing_type=sa.BOOLEAN(), nullable=False, schema=schema) + # Add missing index + op.create_index( + "idx_restrictionarea_area", + "restrictionarea", + ["area"], + unique=False, + schema=schema, + postgresql_using="gist", + ) + # label is required + op.alter_column("tsearch", "label", existing_type=sa.VARCHAR(), nullable=False, schema=schema) + # Add default value + op.execute(f"UPDATE {schema}.tsearch SET public = true WHERE public IS NULL") + op.alter_column( + "tsearch", + "public", + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default="true", + schema=schema, + ) + op.execute(f"UPDATE {schema}.tsearch SET from_theme = false WHERE from_theme IS NULL") + op.alter_column( + "tsearch", + "from_theme", + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default="false", + schema=schema, + ) + + +def downgrade() -> None: + """Downgrade.""" + schema = config["schema"] + + op.alter_column( + "tsearch", + "from_theme", + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default="false", + schema=schema, + ) + op.alter_column( + "tsearch", + "public", + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default="true", + schema=schema, + ) + op.alter_column("tsearch", "label", existing_type=sa.VARCHAR(), nullable=True, schema=schema) + op.drop_index( + "idx_restrictionarea_area", table_name="restrictionarea", schema=schema, postgresql_using="gist" + ) + op.alter_column("restrictionarea", "readwrite", existing_type=sa.BOOLEAN(), nullable=True, schema=schema) + op.alter_column("restrictionarea", "name", existing_type=sa.VARCHAR(), nullable=True, schema=schema) + op.alter_column( + "ogc_server", + "is_single_tile", + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default="false", + schema=schema, + ) + op.alter_column( + "ogc_server", + "wfs_support", + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default="false", + schema=schema, + ) + op.alter_column("metadata", "name", existing_type=sa.VARCHAR(), nullable=True, schema=schema) + op.alter_column( + "layergroup_treeitem", "ordering", existing_type=sa.INTEGER(), nullable=True, schema=schema + ) + op.alter_column("layer", "public", existing_type=sa.BOOLEAN(), nullable=True, schema=schema) + op.alter_column("interface", "name", existing_type=sa.VARCHAR(), nullable=True, schema=schema) + op.alter_column("dimension", "name", existing_type=sa.VARCHAR(), nullable=True, schema=schema) diff --git a/commons/c2cgeoportal_commons/alembic/script.py.mako b/commons/c2cgeoportal_commons/alembic/script.py.mako index 87065a1a92..0162517e39 100644 --- a/commons/c2cgeoportal_commons/alembic/script.py.mako +++ b/commons/c2cgeoportal_commons/alembic/script.py.mako @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, Camptocamp SA +# Copyright (c) 2024, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -34,7 +34,9 @@ Create Date: ${create_date} """ from alembic import op +import sqlalchemy as sa from c2c.template.config import config +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = ${repr(up_revision)} diff --git a/commons/c2cgeoportal_commons/alembic/static/910b4ca53b68_sync_model_database.py b/commons/c2cgeoportal_commons/alembic/static/910b4ca53b68_sync_model_database.py new file mode 100644 index 0000000000..5ee2ae74bc --- /dev/null +++ b/commons/c2cgeoportal_commons/alembic/static/910b4ca53b68_sync_model_database.py @@ -0,0 +1,182 @@ +# Copyright (c) 2024, Camptocamp SA +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +""" +Sync the model and the database. + +Revision ID: 910b4ca53b68 +Revises: 76d72fb3fcb9 +Create Date: 2024-04-22 07:17:27.468564 +""" + +import sqlalchemy as sa +from alembic import op +from c2c.template.config import config +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "910b4ca53b68" +down_revision = "76d72fb3fcb9" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade.""" + staticschema = config["schema_static"] + + # Add required fields in oauth2 + op.alter_column( + "oauth2_authorizationcode", + "redirect_uri", + existing_type=sa.VARCHAR(), + nullable=False, + schema=staticschema, + ) + op.alter_column( + "oauth2_authorizationcode", + "expire_at", + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + schema=staticschema, + ) + op.alter_column( + "oauth2_bearertoken", + "access_token", + existing_type=sa.VARCHAR(length=100), + nullable=False, + schema=staticschema, + ) + op.alter_column( + "oauth2_bearertoken", + "refresh_token", + existing_type=sa.VARCHAR(length=100), + nullable=False, + schema=staticschema, + ) + op.alter_column( + "oauth2_bearertoken", + "expire_at", + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=False, + schema=staticschema, + ) + op.alter_column( + "oauth2_client", "client_id", existing_type=sa.VARCHAR(), nullable=False, schema=staticschema + ) + op.alter_column( + "oauth2_client", "secret", existing_type=sa.VARCHAR(), nullable=False, schema=staticschema + ) + op.alter_column( + "oauth2_client", "redirect_uri", existing_type=sa.VARCHAR(), nullable=False, schema=staticschema + ) + op.alter_column( + "oauth2_client", "state_required", existing_type=sa.BOOLEAN(), nullable=False, schema=staticschema + ) + op.alter_column( + "oauth2_client", "pkce_required", existing_type=sa.BOOLEAN(), nullable=False, schema=staticschema + ) + # URL and creation are required + op.alter_column("shorturl", "url", existing_type=sa.VARCHAR(), nullable=False, schema=staticschema) + op.alter_column( + "shorturl", "creation", existing_type=postgresql.TIMESTAMP(), nullable=False, schema=staticschema + ) + # Add default value to nb_hits + op.execute(f"UPDATE {staticschema}.shorturl SET nb_hits = 0 WHERE nb_hits IS NULL") + op.alter_column("shorturl", "nb_hits", existing_type=sa.INTEGER(), nullable=False, schema=staticschema) + # Add default value to is_password_changed and deactivated + op.execute( + f'UPDATE {staticschema}."user" SET is_password_changed = false WHERE is_password_changed IS NULL' + ) + op.alter_column( + "user", "is_password_changed", existing_type=sa.BOOLEAN(), nullable=False, schema=staticschema + ) + op.execute(f'UPDATE {staticschema}."user" SET deactivated = false WHERE deactivated IS NULL') + op.alter_column("user", "deactivated", existing_type=sa.BOOLEAN(), nullable=False, schema=staticschema) + + +def downgrade() -> None: + """Downgrade.""" + staticschema = config["schema_static"] + + op.alter_column("user", "deactivated", existing_type=sa.BOOLEAN(), nullable=True, schema=staticschema) + op.alter_column( + "user", "is_password_changed", existing_type=sa.BOOLEAN(), nullable=True, schema=staticschema + ) + op.alter_column("shorturl", "nb_hits", existing_type=sa.INTEGER(), nullable=True, schema=staticschema) + op.alter_column( + "shorturl", "creation", existing_type=postgresql.TIMESTAMP(), nullable=True, schema=staticschema + ) + op.alter_column("shorturl", "url", existing_type=sa.VARCHAR(), nullable=True, schema=staticschema) + op.alter_column( + "oauth2_client", "pkce_required", existing_type=sa.BOOLEAN(), nullable=True, schema=staticschema + ) + op.alter_column( + "oauth2_client", "state_required", existing_type=sa.BOOLEAN(), nullable=True, schema=staticschema + ) + op.alter_column( + "oauth2_client", "redirect_uri", existing_type=sa.VARCHAR(), nullable=True, schema=staticschema + ) + op.alter_column("oauth2_client", "secret", existing_type=sa.VARCHAR(), nullable=True, schema=staticschema) + op.alter_column( + "oauth2_client", "client_id", existing_type=sa.VARCHAR(), nullable=True, schema=staticschema + ) + op.alter_column( + "oauth2_bearertoken", + "expire_at", + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + schema=staticschema, + ) + op.alter_column( + "oauth2_bearertoken", + "refresh_token", + existing_type=sa.VARCHAR(length=100), + nullable=True, + schema=staticschema, + ) + op.alter_column( + "oauth2_bearertoken", + "access_token", + existing_type=sa.VARCHAR(length=100), + nullable=True, + schema=staticschema, + ) + op.alter_column( + "oauth2_authorizationcode", + "expire_at", + existing_type=postgresql.TIMESTAMP(timezone=True), + nullable=True, + schema=staticschema, + ) + op.alter_column( + "oauth2_authorizationcode", + "redirect_uri", + existing_type=sa.VARCHAR(), + nullable=True, + schema=staticschema, + ) diff --git a/commons/c2cgeoportal_commons/models/main.py b/commons/c2cgeoportal_commons/models/main.py index f7bc0d49f6..979c7effd9 100644 --- a/commons/c2cgeoportal_commons/models/main.py +++ b/commons/c2cgeoportal_commons/models/main.py @@ -110,11 +110,15 @@ class FullTextSearch(GeoInterface, Base): # type: ignore """The tsearch table representation.""" __tablename__ = "tsearch" - __table_args__ = (Index("tsearch_ts_idx", "ts", postgresql_using="gin"), {"schema": _schema}) + __table_args__ = ( + Index("tsearch_search_index", "ts", "public", "role_id", "interface_id", "lang"), + Index("tsearch_ts_idx", "ts", postgresql_using="gin"), + {"schema": _schema}, + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) label: Mapped[str] = mapped_column(Unicode) - layer_name: Mapped[str] = mapped_column(Unicode) + layer_name: Mapped[str] = mapped_column(Unicode, nullable=True) role_id: Mapped[int] = mapped_column( Integer, ForeignKey(_schema + ".role.id", ondelete="CASCADE"), nullable=True ) @@ -126,7 +130,7 @@ class FullTextSearch(GeoInterface, Base): # type: ignore lang: Mapped[str] = mapped_column(String(2), nullable=True) public: Mapped[bool] = mapped_column(Boolean, server_default="true") ts = mapped_column(TsVector) - the_geom = mapped_column(Geometry("GEOMETRY", srid=_srid)) + the_geom = mapped_column(Geometry("GEOMETRY", srid=_srid, spatial_index=False)) params = mapped_column(JSONEncodedDict, nullable=True) actions = mapped_column(JSONEncodedDict, nullable=True) from_theme: Mapped[bool] = mapped_column(Boolean, server_default="false") @@ -261,7 +265,7 @@ class Role(Base): # type: ignore }, ) extent = mapped_column( - Geometry("POLYGON", srid=_srid), + Geometry("POLYGON", srid=_srid, spatial_index=False), info={ "colanderalchemy": { "title": _("Extent"), @@ -398,7 +402,7 @@ class LayergroupTreeitem(Base): # type: ignore description: Mapped[Optional[str]] = mapped_column(Unicode, info={"colanderalchemy": {"exclude": True}}) treegroup_id: Mapped[int] = mapped_column( Integer, - ForeignKey(_schema + ".treegroup.id"), + ForeignKey(_schema + ".treegroup.id", name="treegroup_id_fkey"), nullable=False, info={"colanderalchemy": {"exclude": True}}, ) @@ -415,7 +419,7 @@ class LayergroupTreeitem(Base): # type: ignore ) treeitem_id: Mapped[int] = mapped_column( Integer, - ForeignKey(_schema + ".treeitem.id"), + ForeignKey(_schema + ".treeitem.id", ondelete="CASCADE"), nullable=False, info={"colanderalchemy": {"widget": HiddenWidget()}}, ) @@ -457,7 +461,11 @@ class TreeGroup(TreeItem): __table_args__ = {"schema": _schema} __mapper_args__ = {"polymorphic_identity": "treegroup"} # type: ignore[dict-item] # needed for _identity_class - id: Mapped[int] = mapped_column(Integer, ForeignKey(_schema + ".treeitem.id"), primary_key=True) + id: Mapped[int] = mapped_column( + Integer, + ForeignKey(_schema + ".treeitem.id", ondelete="CASCADE", name="treegroup_id_fkey"), + primary_key=True, + ) def _get_children(self) -> list[TreeItem]: return [c.treeitem for c in self.children_relation] @@ -526,7 +534,7 @@ class LayerGroup(TreeGroup): id: Mapped[int] = mapped_column( Integer, - ForeignKey(_schema + ".treegroup.id"), + ForeignKey(_schema + ".treegroup.id", ondelete="CASCADE"), primary_key=True, info={"colanderalchemy": {"missing": drop, "widget": HiddenWidget()}}, ) @@ -556,7 +564,7 @@ class Theme(TreeGroup): id: Mapped[int] = mapped_column( Integer, - ForeignKey(_schema + ".treegroup.id"), + ForeignKey(_schema + ".treegroup.id", ondelete="CASCADE"), primary_key=True, info={"colanderalchemy": {"missing": drop, "widget": HiddenWidget()}}, ) @@ -576,6 +584,7 @@ class Theme(TreeGroup): ) icon: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Icon"), @@ -633,7 +642,7 @@ class Layer(TreeItem): id: Mapped[int] = mapped_column( Integer, - ForeignKey(_schema + ".treeitem.id"), + ForeignKey(_schema + ".treeitem.id", ondelete="CASCADE"), primary_key=True, info={"colanderalchemy": {"widget": HiddenWidget()}}, ) @@ -658,6 +667,7 @@ class Layer(TreeItem): ) exclude_properties: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Exclude properties"), @@ -941,6 +951,7 @@ class LayerWMS(DimensionLayer): ) valid: Mapped[bool] = mapped_column( Boolean, + nullable=True, info={ "colanderalchemy": { "title": _("Valid"), @@ -953,6 +964,7 @@ class LayerWMS(DimensionLayer): ) invalid_reason: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Reason why I am not valid"), @@ -1086,7 +1098,7 @@ class LayerWMTS(DimensionLayer): id: Mapped[int] = mapped_column( Integer, - ForeignKey(_schema + ".layer.id"), + ForeignKey(_schema + ".layer.id", ondelete="CASCADE"), primary_key=True, info={"colanderalchemy": {"missing": None, "widget": HiddenWidget()}}, ) @@ -1114,6 +1126,7 @@ class LayerWMTS(DimensionLayer): ) style: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Style"), @@ -1125,6 +1138,7 @@ class LayerWMTS(DimensionLayer): ) matrix_set: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Matrix set"), @@ -1197,6 +1211,68 @@ def get_default(dbsession: Session) -> Optional[DimensionLayer]: ) +class LayerCOG(Layer): + """The Cloud Optimized GeoTIFF layer table representation.""" + + __tablename__ = "layer_cog" + __table_args__ = {"schema": _schema} + __colanderalchemy_config__ = { + "title": _("COG Layer"), + "plural": _("COG Layers"), + "description": c2cgeoportal_commons.lib.literal.Literal( + _( + """ +
+

Definition of a COG Layer.

+

Note: The layers named cog-defaults contains the values + used when we create a new COG layer.

+
+ """ + ) + ), + } + __c2cgeoform_config__ = {"duplicate": True} + __mapper_args__ = {"polymorphic_identity": "l_cog"} # type: ignore[dict-item] + + id: Mapped[int] = mapped_column( + Integer, + ForeignKey(_schema + ".layer.id"), + primary_key=True, + info={"colanderalchemy": {"missing": None, "widget": HiddenWidget()}}, + ) + url: Mapped[str] = mapped_column( + Unicode, + nullable=False, + info={ + "colanderalchemy": { + "title": _("URL"), + "description": c2cgeoportal_commons.lib.literal.Literal( + _( + """ +
+

Definition of a COG Layer.

+

Note: The layers named cog-defaults contains the values + used when we create a new COG layer.

+
+
+ """ + ) + ), + "column": 2, + } + }, + ) + + @staticmethod + def get_default(dbsession: Session) -> Optional[Layer]: + return dbsession.query(LayerCOG).filter(LayerCOG.name == "cog-defaults").one_or_none() + + def url_description(self, request: pyramid.request.Request) -> str: + errors: set[str] = set() + url = get_url2(self.name, self.url, request, errors) + return url.url() if url else "\n".join(errors) + + class LayerVectorTiles(DimensionLayer): """The layer_vectortiles table representation.""" @@ -1577,6 +1653,7 @@ class Metadata(Base): # type: ignore ) value: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Value"), @@ -1599,7 +1676,7 @@ class Metadata(Base): # type: ignore item_id: Mapped[int] = mapped_column( "item_id", Integer, - ForeignKey(_schema + ".treeitem.id"), + ForeignKey(_schema + ".treeitem.id", ondelete="CASCADE"), nullable=False, info={"colanderalchemy": {"exclude": True}, "c2cgeoform": {"duplicate": False}}, ) @@ -1684,6 +1761,7 @@ class Dimension(Base): # type: ignore ) value: Mapped[str] = mapped_column( Unicode, + nullable=True, info={ "colanderalchemy": { "title": _("Value"), diff --git a/commons/c2cgeoportal_commons/models/static.py b/commons/c2cgeoportal_commons/models/static.py index e516e7ee53..6dcbd6f9d1 100644 --- a/commons/c2cgeoportal_commons/models/static.py +++ b/commons/c2cgeoportal_commons/models/static.py @@ -174,7 +174,8 @@ class User(Base): # type: ignore ) settings_role_id: Mapped[int] = mapped_column( - Integer, + Integer(), + nullable=True, info={ "colanderalchemy": { "title": _("Settings from role"), @@ -219,6 +220,7 @@ class User(Base): # type: ignore last_login: Mapped[datetime] = mapped_column( DateTime(timezone=True), + nullable=True, info={ "colanderalchemy": { "title": _("Last login"), @@ -343,9 +345,9 @@ class Shorturl(Base): # type: ignore id: Mapped[int] = mapped_column(Integer, primary_key=True) url: Mapped[str] = mapped_column(Unicode) ref: Mapped[str] = mapped_column(String(20), index=True, unique=True, nullable=False) - creator_email: Mapped[str] = mapped_column(Unicode(200)) + creator_email: Mapped[str] = mapped_column(Unicode(200), nullable=True) creation: Mapped[datetime] = mapped_column(DateTime) - last_hit: Mapped[datetime] = mapped_column(DateTime) + last_hit: Mapped[datetime] = mapped_column(DateTime, nullable=True) nb_hits: Mapped[int] = mapped_column(Integer) @@ -443,7 +445,7 @@ class OAuth2BearerToken(Base): # type: ignore access_token: Mapped[str] = mapped_column(Unicode(100), unique=True) refresh_token: Mapped[str] = mapped_column(Unicode(100), unique=True) expire_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) # in one hour - state = mapped_column(String) + state = mapped_column(String, nullable=True) class OAuth2AuthorizationCode(Base): # type: ignore @@ -466,10 +468,10 @@ class OAuth2AuthorizationCode(Base): # type: ignore ) user = relationship(User) redirect_uri: Mapped[str] = mapped_column(Unicode) - code: Mapped[str] = mapped_column(Unicode(100), unique=True) - state: Mapped[Optional[str]] = mapped_column(String) - challenge: Mapped[str] = mapped_column(String(128)) - challenge_method: Mapped[str] = mapped_column(String(6)) + code: Mapped[str] = mapped_column(Unicode(100), unique=True, nullable=True) + state: Mapped[Optional[str]] = mapped_column(String, nullable=True) + challenge: Mapped[str] = mapped_column(String(128), nullable=True) + challenge_method: Mapped[str] = mapped_column(String(6), nullable=True) expire_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) # in 10 minutes diff --git a/doc/integrator/admin_interface.rst b/doc/integrator/admin_interface.rst index af53e62c99..a0a9752cbb 100644 --- a/doc/integrator/admin_interface.rst +++ b/doc/integrator/admin_interface.rst @@ -10,6 +10,13 @@ Configure the admin interface You can activate or deactivate (tabs, modules, models or tables) in administration interface using configuration key ``exclude_pages`` and ``include_pages``. +Defaults values +--------------- + +You can define the default values used for creating new layer by creating a layer with the name: +``wms-defaults``, ``wmts-defaults``, ``vector-tiles-defaults`` or ``cog-defaults``. + + Include and excludes tabs ------------------------- diff --git a/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_CHANGELOG.txt b/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_CHANGELOG.txt index f58031fff6..bccb373c80 100644 --- a/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_CHANGELOG.txt +++ b/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_CHANGELOG.txt @@ -24,6 +24,14 @@ Information 2. We replace checks (formatting) done by `c2cciutils` by `pre-commit` hooks. This will me more standard and transparent for the project. +3. The COG layers where added, but are disabled by default. + To enable them, you need to add the following configuration in the `vars.yaml` file: + ```yaml + vars: + admin_interface: + exclude_pages: [] + ``` + ============= Version 2.8.0 ============= diff --git a/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_vars.yaml b/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_vars.yaml index d904b6fee0..f57d410318 100644 --- a/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_vars.yaml +++ b/geoportal/c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_vars.yaml @@ -509,6 +509,9 @@ vars: admin_interface: layer_tree_max_nodes: 1000 + exclude_pages: + - layers-cog + # The list of available variable names for the `Metadatas` form. available_metadata: # TreeItem @@ -566,6 +569,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: isLegendExpanded type: boolean description: > @@ -573,6 +577,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: legendRule description: > The WMS "RULE" parameter used to display the icon in the layer tree. "Short version" of @@ -586,6 +591,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: hiDPILegendImages type: json description: > @@ -595,6 +601,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: iconUrl type: url description: > @@ -602,6 +609,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: metadataUrl type: url description: > @@ -609,6 +617,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - layergroup - name: disclaimer translate: True @@ -623,6 +632,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: identifierAttributeField description: > The field used in the "display query window" as feature title. @@ -636,6 +646,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: minResolution type: float description: > @@ -644,6 +655,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: opacity type: float description: > @@ -651,6 +663,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: thumbnail type: url description: > @@ -658,6 +671,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: timeAttribute description: > The name of the time attribute. For WMS(-T) layers. @@ -700,6 +714,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: editingAttributesOrder type: list description: > @@ -728,6 +743,7 @@ vars: relevant_for: - layer_wms - layer_wmts + - layer_cog - name: editingSelectionAttribute description: > The field used in the selector tooltip to name the layer when multiple ones are selected on the map. @@ -753,6 +769,7 @@ vars: The corresponding OGC server for a WMTS layer. relevant_for: - layer_wmts + - layer_cog - name: wmsLayers description: > Corresponding WMS/WFS layers (comma separated) that will be used instead of the WMTS layer @@ -761,6 +778,7 @@ vars: See also printLayers and queryLayers metadata for more granularity. relevant_for: - layer_wmts + - layer_cog - name: queryLayers description: > Corresponding WFS layers (comma separated) that will be used instead of the WMTS layer @@ -768,6 +786,7 @@ vars: An OGCServer metadata must be set with the name of the OGC server. relevant_for: - layer_wmts + - layer_cog - name: printLayers description: > Corresponding WMS layers (comma separated) that will be used instead of the WMTS layer @@ -777,18 +796,21 @@ vars: An OGCServer metadata must be set with the name of the OGC server. relevant_for: - layer_wmts + - layer_cog - name: maxQueryResolution type: float description: > The max resolution where the layer is queryable. relevant_for: - layer_wmts + - layer_cog - name: minQueryResolution type: float description: > The min resolution where the layer is queryable. relevant_for: - layer_wmts + - layer_cog - name: timeout type: float description: The fetch capabilities timeout in seconds. Use `300` by default. diff --git a/geoportal/c2cgeoportal_geoportal/views/theme.py b/geoportal/c2cgeoportal_geoportal/views/theme.py index 764231d949..e76872fe14 100644 --- a/geoportal/c2cgeoportal_geoportal/views/theme.py +++ b/geoportal/c2cgeoportal_geoportal/views/theme.py @@ -438,6 +438,10 @@ async def _layer( layer_info["type"] = "VectorTiles" self._vectortiles_layers(layer_info, layer, errors) + elif isinstance(layer, main.LayerCOG): + layer_info["type"] = "COG" + self._cog_layers(layer_info, layer, errors) + return None if errors else layer_info, errors @staticmethod @@ -575,12 +579,18 @@ def _fill_wmts(self, layer_theme: dict[str, Any], layer: main.Layer, errors: set layer_theme["layer"] = layer.layer layer_theme["imageType"] = layer.image_type - def _vectortiles_layers(self, layer_theme: dict[str, Any], layer: main.Layer, errors: set[str]) -> None: + def _vectortiles_layers( + self, layer_theme: dict[str, Any], layer: main.LayerVectorTiles, errors: set[str] + ) -> None: style = get_url2(f"The VectorTiles layer '{layer.name}'", layer.style, self.request, errors=errors) layer_theme["style"] = style.url() if style is not None else None if layer.xyz: layer_theme["xyz"] = layer.xyz + def _cog_layers(self, layer_theme: dict[str, Any], layer: main.LayerCOG, errors: set[str]) -> None: + url = get_url2(f"The COG layer '{layer.name}'", layer.url, self.request, errors=errors) + layer_theme["url"] = url.url() if url is not None else None + @staticmethod def _layer_included(tree_item: main.TreeItem) -> bool: return isinstance(tree_item, main.Layer) diff --git a/geoportal/tests/functional/test_lingua_extractor_themes.py b/geoportal/tests/functional/test_lingua_extractor_themes.py index aa65114958..f22adf1157 100644 --- a/geoportal/tests/functional/test_lingua_extractor_themes.py +++ b/geoportal/tests/functional/test_lingua_extractor_themes.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, Camptocamp SA +# Copyright (c) 2020-2024, Camptocamp SA # All rights reserved. # Redistribution and use in source and binary forms, with or without @@ -124,6 +124,7 @@ def test_extract_full_text_search(self, dbsession, transact): del transact fts = main.FullTextSearch() + fts.label = "label" fts.layer_name = "some_layer_name" fts.actions = [{"action": "add_layer", "data": "another_layer_name"}] dbsession.add(fts)