diff --git a/.docker/Dockerfile b/.docker/Dockerfile deleted file mode 100644 index a7880a6..0000000 --- a/.docker/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG QGIS_TEST_VERSION=latest -FROM qgis/qgis:${QGIS_TEST_VERSION} -MAINTAINER Matthias Kuhn - -RUN apt-get update \ - && apt-get install -y \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install pytest - -ENV LANG=C.UTF-8 - -WORKDIR / \ No newline at end of file diff --git a/.docker/docker-compose.gh.yml b/.docker/docker-compose.gh.yml deleted file mode 100644 index a5a9f87..0000000 --- a/.docker/docker-compose.gh.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '3' -services: - qgis: - build: - context: .. - dockerfile: ./.docker/Dockerfile - args: - QGIS_TEST_VERSION: ${QGIS_TEST_VERSION} - tty: true - volumes: - - ${GITHUB_WORKSPACE}:/usr/src \ No newline at end of file diff --git a/.docker/run-docker-tests.sh b/.docker/run-docker-tests.sh deleted file mode 100755 index 73e6fb9..0000000 --- a/.docker/run-docker-tests.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -#*************************************************************************** -# ------------------- -# begin : 2017-08-24 -# git sha : :%H$ -# copyright : (C) 2017 by OPENGIS.ch -# email : info@opengis.ch -#*************************************************************************** -# -#*************************************************************************** -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU General Public License as published by * -#* the Free Software Foundation; either version 2 of the License, or * -#* (at your option) any later version. * -#* * -#*************************************************************************** - -set -e -pushd /usr/src -DEFAULT_PARAMS='-v' -xvfb-run pytest ${@:-`echo $DEFAULT_PARAMS`} $1 -popd \ No newline at end of file diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index c746ee7..9f1e5f8 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - qgis_version: [release-3_16, release-3_22, latest] + qgis_version: [focal-3.16, focal-3.22, latest] env: QGIS_TEST_VERSION: ${{ matrix.qgis_version }} steps: @@ -26,8 +26,8 @@ jobs: with: submodules: recursive - name: Test on QGIS - run: docker-compose -f .docker/docker-compose.gh.yml run qgis /usr/src/.docker/run-docker-tests.sh - + run: docker run -v ${GITHUB_WORKSPACE}:/usr/src -w /usr/src opengisch/qgis:${QGIS_TEST_VERSION} sh -c 'xvfb-run pytest-3' + release: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI if: startsWith(github.ref, 'refs/tags/v') diff --git a/.gitignore b/.gitignore index 2808f30..54f0972 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ __pycache__ *.pyc .cache .vscode -**/__pycache__ \ No newline at end of file +**/__pycache__ diff --git a/README.md b/README.md index a906b5f..3de8517 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,302 @@ # toppingmaker -Python package to create QGIS project configuration toppings +Package to create parameterized QGIS projects and dump it into a YAML structure. + +## Installation +``` +pip install toppingmaker +``` + +## Structure + +``` +toppingmaker +├── exportsettings.py +├── projecttopping.py +├── target.py +└── utils.py +``` + +## User Manual + +Having a QGIS Project with some layers: + +![QGIS Project Layertree](assets/qgis_project_layertree.png) + +### Import the modules + +```py +from qgis.core import QgsProject() +from toppingmaker import ProjectTopping, ExportSettings, Target +``` + +### Create a `ProjectTopping` and parse the QGIS Project + +```py +project = QgsProject() +project_topping = ProjectTopping() +project_topping.parse_project(project) +``` + +This parses the project, but does not yet write the files (only the style and definition file to a temp folder). The QgsProject object is not used anymore. + +### Create the `Target` +To write the files we need to define a `Target` object. The target defines where to store the topping files (YAML, style, definition etc.). + +```py +target = Target(projectname="freddys_qgis_project", main_dir="/home/fred/repo/", sub_dir="freddys_qgis_topping", pathresover = None) +``` + +### Generate the Files +```py +project_topping.generate_files(target) +``` + +The structure looks like this: + +``` +repo +└── freddys_qgis_topping + └── projecttopping + └── freddys_qgis_project.yaml +``` + +And the YAML looks like this: + +```yaml +layerorder: [] +layertree: +- Street: + checked: true + expanded: true +- Park: + checked: false + expanded: true +- Building: + checked: true + expanded: true +- Info Layers: + checked: true + child-nodes: + - AssetItem: + checked: true + expanded: true + - InternalProject: + checked: true + expanded: true + expanded: true + group: true +- Background: + checked: true + child-nodes: + - Landeskarten (grau): + checked: true + expanded: true + expanded: true + group: true +``` + +The structure is exported. But not any additional files. For that, we need to parse the `ExportSettings` to the `ProjectTopping`. + +### Create the `ExportSettings`: + +Use `QMLSTYLE` for the export of the qml stylefile. +Use `DEFINITION` to export the qlr definition file. +USE `SOURCE` to store the source in the YAML tree. + +The QgsLayerTreeNode or the layername can be used as key. + +```py +export_settings = ExportSettings() +export_settings.set_setting_values( + type = ExportSettings.ToppingType.QMLSTYLE, node = None, name = "Street", export = True +) +export_settings.set_setting_values( + type = ExportSettings.ToppingType.SOURCE, node = None, name = "Park", export = True +) +export_settings.set_setting_values( + type = ExportSettings.ToppingType.DEFINITION, node = None, name = "Info Layers", export = True +) +export_settings.set_setting_values( + type = ExportSettings.ToppingType.SOURCE, node = None, name = "Landeskarten (grau)", export = True +) +``` + +Additionally you can pass category flags `QgsMapLayer.StyleCategories` to define what categories needs to be included in the QML Stylefile: + +```py +category_flags = QgsMapLayer.StyleCategory.AllStyleCategories + +export_settings.set_setting_values( + type = ExportSettings.ToppingType.QMLSTYLE, node = None, name = "Street", export = True, categories = category_flags +) +``` + +### Generate the Files for a `ProjectTopping` containing `ExportSetting` +When parsing the QgsProject we need to pass the `ExportSettings`: +``` + +project_topping.parse_project(project, export_settings) +project_topping.generate_files(target) +``` + +The structure looks like this: + +``` +repo +└── freddys_qgis_topping + ├── layerdefinition + │   └── freddys_qgis_project_info_layers.qlr + └── projecttopping + └── freddys_qgis_project.yaml + └── layerstyle + └── freddys_qgis_project_street.qml +``` + +And the YAML looks like this: + +```yaml +layerorder: [] +layertree: +- Street: + checked: true + expanded: true + stylefile: freddys_qgis_topping/layerstyle/freddys_qgis_project_street.qml +- Park: + checked: false + expanded: true + provider: ogr + uri: /home/freddy/qgis_projects/bakery/cityandcity.gpkg|layername=park +- Building: + checked: true + expanded: true +- Info Layers: + checked: true + definitionfile: freddys_qgis_topping/layerdefinition/freddys_qgis_project_info_layers.qlr + expanded: true + group: true +- Background: + checked: true + child-nodes: + expanded: true + group: true + - Landeskarten (grau): + checked: true + expanded: true + provider: wms + uri: contextualWMSLegend=0&crs=EPSG:2056&dpiMode=7&featureCount=10&format=image/jpeg&layers=ch.swisstopo.pixelkarte-grau&styles&url=https://wms.geo.admin.ch/?%0ASERVICE%3DWMS%0A%26VERSION%3D1.3.0%0A%26REQUEST%3DGetCapabilities +``` + +## Most important functions +### projecttopping.ProjectTopping +A project configuration resulting in a YAML file that contains: +- layertree +- layerorder +- project variables (future) +- print layout (future) +- map themes (future) + +QML style files, QLR layer definition files and the source of a layer can be linked in the YAML file and are exported to the specific folders. + +#### `parse_project( project: QgsProject, export_settings: ExportSettings = ExportSettings()` +Parses a project into the ProjectTopping structure. Means the LayerTreeNodes are loaded into the layertree variable and append the ExportSettings to each node. The CustomLayerOrder is loaded into the layerorder. The project is not kept as member variable. + +#### `generate_files(self, target: Target) -> str` +Generates all files according to the passed Target. +The target object containing the paths where to create the files and the path_resolver defining the structure of the link. + +#### `load_files(self, target: Target)` +not yet implemented + +#### `generate_project(self, target: Target) -> QgsProject` +not yet implemented + +### target.Target +If there is no subdir it will look like: +``` + + ├── projecttopping + │ └── .yaml + ├── layerstyle + │ ├── _.qml + │ └── _.qml + └── layerdefinition + └── _.qlr +``` +With subdir: +``` + + └── + ├── projecttopping + │ └── .yaml + ├── layerstyle + │ ├── _.qml + │ └── _.qml + └── layerdefinition + └── _.qlr +``` + +The `path_resolver` can be passed as a function. The default implementation lists the created toppingfiles (including the YAML) in the dict `Target.toppingfileinfo_list` with the `"path": , "type": `. + +#### `Target( projectname: str = "project", main_dir: str = None, sub_dir: str = None, path_resolver=None)` +The constructor of the target class to set up a target. +A member variable `toppingfileinfo_list = []` is defined, to store all the information according the `path_resolver`. + +### exportsettings.ExportSettings + +The requested export settings of each node in the specific dicts: +- qmlstyle_setting_nodes +- definition_setting_nodes +- source_setting_nodes + +The usual structure is using QgsLayerTreeNode as key and then export True/False + +```py +{ + : { export: False } + : { export: True } +} +``` + +Alternatively the layername can be used as key. In ProjectTopping it first looks up the node and if not available the name. +Using the node is much more consistent, since one can use layers with the same name, but for nodes you need the project already in advance. +With name you can use prepared settings to pass (before the project exists) e.g. in automated workflows. +```py +{ + "Node1": { export: False } + "Node2": { export: True } +} +``` + +For some settings we have additional info. Like in qmlstyle_nodes . These are Flags, and can be constructed manually as well. +```py +qmlstyle_nodes = +{ + : { export: False } + : { export: True, categories: } +} +``` + +#### `set_setting_values( type: ToppingType, node: Union[QgsLayerTreeLayer, QgsLayerTreeGroup] = None, name: str = None, export=True categories=None, ) -> bool` + +Set the specific types concerning the enumerations: +```py +class ToppingType(Enum): + QMLSTYLE = 1 + DEFINITION = 2 + SOURCE = 3 + +``` + +## Infos for Devs + +### Code style + +Is enforced with pre-commit. To use, make: +``` +pip install pre-commit +pre-commit install +``` +And to run it over all the files (with infile changes): +``` +pre-commit run --color=always --all-file +``` diff --git a/assets/qgis_project_layertree.png b/assets/qgis_project_layertree.png new file mode 100644 index 0000000..2da8ea6 Binary files /dev/null and b/assets/qgis_project_layertree.png differ diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..64ddc6e --- /dev/null +++ b/tests/README.md @@ -0,0 +1,18 @@ +# Running the tests + +To run the tests inside the same environment as they are executed on gh workflow, +you need a [Docker](https://www.docker.com/) installation. This will also launch an extra container +with a database, so your own postgres installation is not affected at all. + +To run the tests, go to the main directory of the project and do + +```sh +export QGIS_TEST_VERSION=latest # See https://hub.docker.com/r/qgis/qgis/tags/ +export GITHUB_WORKSPACE=$PWD # only for local execution +docker run -v ${GITHUB_WORKSPACE}:/usr/src -w /usr/src opengisch/qgis:${QGIS_TEST_VERSION} sh -c 'xvfb-run pytest-3' +``` + +In one line, removing all containers. +```sh +QGIS_TEST_VERSION=latest GITHUB_WORKSPACE=$PWD docker run -v ${GITHUB_WORKSPACE}:/usr/src -w /usr/src opengisch/qgis:${QGIS_TEST_VERSION} sh -c 'xvfb-run pytest-3' +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_toppingmaker.py b/tests/test_toppingmaker.py new file mode 100644 index 0000000..f7eb337 --- /dev/null +++ b/tests/test_toppingmaker.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + ------------------- + begin : 2022-07-17 + git sha : :%H$ + copyright : (C) 2022 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +import datetime +import logging +import os +import tempfile + +import yaml +from qgis.core import QgsProject, QgsVectorLayer +from qgis.testing import unittest + +from toppingmaker import ExportSettings, ProjectTopping, Target + + +class ToppingMakerTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Run before all tests.""" + cls.basetestpath = tempfile.mkdtemp() + cls.projecttopping_test_path = os.path.join(cls.basetestpath, "projecttopping") + + def test_target(self): + maindir = os.path.join(self.projecttopping_test_path, "freddys_repository") + subdir = "freddys_projects/this_specific_project" + filedirs = ["projecttopping", "layerstyle", "layerdefinition", "andanotherone"] + target = Target("freddys", maindir, subdir) + count = 0 + for filedir in filedirs: + # filedir_path should create the dir + path, _ = target.filedir_path(filedir) + assert os.path.isdir(path) + count += 1 + assert count == 4 + + def test_parse_project(self): + """ + "Big Group": + group: True + child-nodes: + - "Layer One": + checked: True + - "Medium Group": + group: True + child-nodes: + - "Layer Two": + - "Small Group: + - "Layer Three": + - "Layer Four": + - "Layer Five": + "All of em": + group: True + child-nodes: + - "Layer One": + checked: False + - "Layer Two": + - "Layer Three": + checked: False + - "Layer Four": + - "Layer Five": + """ + project, _ = self._make_project_and_export_settings() + layers = project.layerTreeRoot().findLayers() + self.assertEqual(len(layers), 10) + + project_topping = ProjectTopping() + project_topping.parse_project(project) + + checked_groups = [] + for item in project_topping.layertree.items: + if item.name == "Big Group": + assert len(item.items) == 2 + checked_groups.append("Big Group") + for item in item.items: + if item.name == "Medium Group": + assert len(item.items) == 3 + checked_groups.append("Medium Group") + for item in item.items: + if item.name == "Small Group": + assert len(item.items) == 2 + checked_groups.append("Small Group") + assert checked_groups == ["Big Group", "Medium Group", "Small Group"] + + def test_generate_files(self): + project, export_settings = self._make_project_and_export_settings() + layers = project.layerTreeRoot().findLayers() + self.assertEqual(len(layers), 10) + + project_topping = ProjectTopping() + project_topping.parse_project(project, export_settings) + + checked_groups = [] + for item in project_topping.layertree.items: + if item.name == "Big Group": + assert len(item.items) == 2 + checked_groups.append("Big Group") + for item in item.items: + if item.name == "Medium Group": + assert len(item.items) == 3 + checked_groups.append("Medium Group") + for item in item.items: + if item.name == "Small Group": + assert len(item.items) == 2 + checked_groups.append("Small Group") + assert checked_groups == ["Big Group", "Medium Group", "Small Group"] + + maindir = os.path.join(self.projecttopping_test_path, "freddys_repository") + subdir = "freddys_projects/this_specific_project" + + target = Target("freddys", maindir, subdir) + + projecttopping_file_path = os.path.join( + target.main_dir, project_topping.generate_files(target) + ) + + # check projecttopping_file + foundAllofEm = False + foundLayerOne = False + foundLayerTwo = False + + with open(projecttopping_file_path, "r") as yamlfile: + projecttopping_data = yaml.safe_load(yamlfile) + assert "layertree" in projecttopping_data + assert projecttopping_data["layertree"] + for node in projecttopping_data["layertree"]: + if "All of em" in node: + foundAllofEm = True + assert "child-nodes" in node["All of em"] + for childnode in node["All of em"]["child-nodes"]: + if "Layer One" in childnode: + foundLayerOne = True + assert "checked" in childnode["Layer One"] + assert not childnode["Layer One"]["checked"] + if "Layer Two" in childnode: + foundLayerTwo = True + assert "checked" in childnode["Layer Two"] + assert childnode["Layer Two"]["checked"] + assert foundAllofEm + assert foundLayerOne + assert foundLayerTwo + + # check toppingfiles + + # there should be exported 6 files (see _make_project_and_export_settings) + # stylefiles: + # "Layer One" + # "Layer Three" + # "Layer Five" + # + # definitionfiles: + # "Layer Three" + # "Layer Four" + # "Layer Five" + + countchecked = 0 + + # there should be 13 toppingfiles: one project topping, and 2 x 6 toppingfiles of the layers (since the layers are multiple times in the tree) + assert len(target.toppingfileinfo_list) == 13 + + for toppingfileinfo in target.toppingfileinfo_list: + assert "path" in toppingfileinfo + assert "type" in toppingfileinfo + + if ( + toppingfileinfo["path"] + == "freddys_projects/this_specific_project/layerstyle/freddys_layer_one.qml" + ): + countchecked += 1 + if ( + toppingfileinfo["path"] + == "freddys_projects/this_specific_project/layerstyle/freddys_layer_three.qml" + ): + countchecked += 1 + if ( + toppingfileinfo["path"] + == "freddys_projects/this_specific_project/layerstyle/freddys_layer_five.qml" + ): + countchecked += 1 + if ( + toppingfileinfo["path"] + == "freddys_projects/this_specific_project/layerdefinition/freddys_layer_three.qlr" + ): + countchecked += 1 + if ( + toppingfileinfo["path"] + == "freddys_projects/this_specific_project/layerdefinition/freddys_layer_four.qlr" + ): + countchecked += 1 + if ( + toppingfileinfo["path"] + == "freddys_projects/this_specific_project/layerdefinition/freddys_layer_five.qlr" + ): + countchecked += 1 + + assert countchecked == 12 + + def test_custom_path_resolver(self): + # load QGIS project into structure + project_topping = ProjectTopping() + project, export_settings = self._make_project_and_export_settings() + project_topping.parse_project(project, export_settings) + + # create target with path resolver + maindir = os.path.join(self.projecttopping_test_path, "freddys_repository") + subdir = "freddys_projects/this_specific_project" + + target = Target("freddys", maindir, subdir, custom_path_resolver) + + project_topping.generate_files(target) + + # there should be exported 6 files (see _make_project_and_export_settings) + # stylefiles: + # "Layer One" + # "Layer Three" + # "Layer Five" + # + # definitionfiles: + # "Layer Three" + # "Layer Four" + # "Layer Five" + + countchecked = 0 + for toppingfileinfo in target.toppingfileinfo_list: + assert "id" in toppingfileinfo + assert "path" in toppingfileinfo + assert "type" in toppingfileinfo + assert "version" in toppingfileinfo + + if toppingfileinfo["id"] == "layerstyle_freddys_layer_one.qml_001": + countchecked += 1 + if toppingfileinfo["id"] == "layerstyle_freddys_layer_three.qml_001": + countchecked += 1 + if toppingfileinfo["id"] == "layerstyle_freddys_layer_five.qml_001": + countchecked += 1 + if toppingfileinfo["id"] == "layerdefinition_freddys_layer_three.qlr_001": + countchecked += 1 + if toppingfileinfo["id"] == "layerdefinition_freddys_layer_four.qlr_001": + countchecked += 1 + if toppingfileinfo["id"] == "layerdefinition_freddys_layer_five.qlr_001": + countchecked += 1 + + assert countchecked == 6 + + def _make_project_and_export_settings(self): + project = QgsProject() + project.removeAllMapLayers() + + l1 = QgsVectorLayer( + "point?crs=epsg:4326&field=id:integer", "Layer One", "memory" + ) + assert l1.isValid() + l2 = QgsVectorLayer( + "point?crs=epsg:4326&field=id:integer", "Layer Two", "memory" + ) + assert l2.isValid() + l3 = QgsVectorLayer( + "point?crs=epsg:4326&field=id:integer", "Layer Three", "memory" + ) + assert l3.isValid() + l4 = QgsVectorLayer( + "point?crs=epsg:4326&field=id:integer", "Layer Four", "memory" + ) + assert l4.isValid() + l5 = QgsVectorLayer( + "point?crs=epsg:4326&field=id:integer", "Layer Five", "memory" + ) + assert l5.isValid() + + project.addMapLayer(l1, False) + project.addMapLayer(l2, False) + project.addMapLayer(l3, False) + project.addMapLayer(l4, False) + project.addMapLayer(l5, False) + + biggroup = project.layerTreeRoot().addGroup("Big Group") + biggroup.addLayer(l1) + mediumgroup = biggroup.addGroup("Medium Group") + mediumgroup.addLayer(l2) + smallgroup = mediumgroup.addGroup("Small Group") + smallgroup.addLayer(l3) + smallgroup.addLayer(l4) + mediumgroup.addLayer(l5) + allofemgroup = project.layerTreeRoot().addGroup("All of em") + node1 = allofemgroup.addLayer(l1) + node1.setItemVisibilityChecked(False) + allofemgroup.addLayer(l2) + node3 = allofemgroup.addLayer(l3) + node3.setItemVisibilityChecked(False) + allofemgroup.addLayer(l4) + allofemgroup.addLayer(l5) + + export_settings = ExportSettings() + export_settings.set_setting_values( + ExportSettings.ToppingType.QMLSTYLE, None, "Layer One", True + ) + export_settings.set_setting_values( + ExportSettings.ToppingType.QMLSTYLE, None, "Layer Three", True + ) + export_settings.set_setting_values( + ExportSettings.ToppingType.QMLSTYLE, None, "Layer Five", True + ) + + export_settings.set_setting_values( + ExportSettings.ToppingType.DEFINITION, None, "Layer Three", True + ) + export_settings.set_setting_values( + ExportSettings.ToppingType.DEFINITION, None, "Layer Four", True + ) + export_settings.set_setting_values( + ExportSettings.ToppingType.DEFINITION, None, "Layer Five", True + ) + + export_settings.set_setting_values( + ExportSettings.ToppingType.SOURCE, None, "Layer One", True + ) + export_settings.set_setting_values( + ExportSettings.ToppingType.SOURCE, None, "Layer Two", True + ) + export_settings.set_setting_values( + ExportSettings.ToppingType.SOURCE, None, "Layer Three", True + ) + + print(export_settings.qmlstyle_setting_nodes) + print(export_settings.definition_setting_nodes) + print(export_settings.source_setting_nodes) + return project, export_settings + + def print_info(self, text): + logging.info(text) + + def print_error(self, text): + logging.error(text) + + +def custom_path_resolver(target: Target, name, type): + _, relative_filedir_path = target.filedir_path(type) + id = unique_id_in_target_scope(target, f"{type}_{name}_001") + path = os.path.join(relative_filedir_path, name) + type = type + version = datetime.datetime.now().strftime("%Y-%m-%d") + toppingfile = {"id": id, "path": path, "type": type, "version": version} + target.toppingfileinfo_list.append(toppingfile) + return path + + +def unique_id_in_target_scope(target: Target, id): + for toppingfileinfo in target.toppingfileinfo_list: + if "id" in toppingfileinfo and toppingfileinfo["id"] == id: + iterator = int(id[-3:]) + iterator += 1 + id = f"{id[:-3]}{iterator:03}" + return unique_id_in_target_scope(target, id) + return id diff --git a/toppingmaker/__init__.py b/toppingmaker/__init__.py new file mode 100644 index 0000000..1c68e38 --- /dev/null +++ b/toppingmaker/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + ------------------- + begin : 2022-07-17 + git sha : :%H$ + copyright : (C) 2022 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" +from .exportsettings import ExportSettings +from .projecttopping import ProjectTopping +from .target import Target diff --git a/toppingmaker/exportsettings.py b/toppingmaker/exportsettings.py new file mode 100644 index 0000000..e931469 --- /dev/null +++ b/toppingmaker/exportsettings.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + ------------------- + begin : 2022-07-17 + git sha : :%H$ + copyright : (C) 2022 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" +from enum import Enum +from typing import Union + +from qgis.core import QgsLayerTreeGroup, QgsLayerTreeLayer + + +class ExportSettings(object): + """ + The requested export settings of each node in the specific dicts: + - qmlstyle_setting_nodes + - definition_setting_nodes + - source_setting_nodes + + The usual structure is using QgsLayerTreeNode as key and then export True/False + { + : { export: False } + : { export: True } + } + + But alternatively the layername can be used as key. In ProjectTopping it first looks up the node and if not available looking up the name. + Using the node is much more consistent, since one can use layers with the same name, but for nodes you need the project already in advance. + With name you can use prepared settings to pass (before the project exists) e.g. in automated workflows. + { + "Node1": { export: False } + "Node2": { export: True } + } + + For some settings we have additional info. Like in qmlstyle_nodes . These are Flags, and can be constructed manually as well. + qmlstyle_nodes = + { + : { export: False } + : { export: True, categories: } + } + """ + + class ToppingType(Enum): + QMLSTYLE = 1 + DEFINITION = 2 + SOURCE = 3 + + def __init__(self): + self.qmlstyle_setting_nodes = {} + self.definition_setting_nodes = {} + self.source_setting_nodes = {} + + def set_setting_values( + self, + type: ToppingType, + node: Union[QgsLayerTreeLayer, QgsLayerTreeGroup] = None, + name: str = None, + export=True, + categories=None, + ) -> bool: + """ + Appends the values (export, categories) to an existing setting + """ + setting_nodes = self._setting_nodes(type) + setting = self._get_setting(setting_nodes, node, name) + setting["export"] = export + if categories: + setting["categories"] = categories + return self._set_setting(setting_nodes, setting, node, name) + + def get_setting( + self, + type: ToppingType, + node: Union[QgsLayerTreeLayer, QgsLayerTreeGroup] = None, + name: str = None, + ) -> dict(): + """ + Returns an existing or an empty setting dict + """ + setting_nodes = self._setting_nodes(type) + return self._get_setting(setting_nodes, node, name) + + def _setting_nodes(self, type: ToppingType): + if type == ExportSettings.ToppingType.QMLSTYLE: + return self.qmlstyle_setting_nodes + if type == ExportSettings.ToppingType.DEFINITION: + return self.definition_setting_nodes + if type == ExportSettings.ToppingType.SOURCE: + return self.source_setting_nodes + + def _get_setting(self, setting_nodes, node=None, name=None): + setting = {} + if node: + setting = setting_nodes.get(node, {}) + if not setting: + setting = setting_nodes.get(name, {}) + return setting + + def _set_setting(self, setting_nodes, setting, node=None, name=None) -> bool: + if node: + setting_nodes[node] = setting + return True + if name: + setting_nodes[name] = setting + return True + return False diff --git a/toppingmaker/projecttopping.py b/toppingmaker/projecttopping.py new file mode 100644 index 0000000..42450bf --- /dev/null +++ b/toppingmaker/projecttopping.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + ------------------- + begin : 2022-07-17 + git sha : :%H$ + copyright : (C) 2022 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +import os +from typing import Union + +import yaml +from qgis.core import ( + Qgis, + QgsLayerDefinition, + QgsLayerTreeGroup, + QgsLayerTreeLayer, + QgsMapLayer, + QgsProject, +) +from qgis.PyQt.QtCore import QObject, pyqtSignal + +from .exportsettings import ExportSettings +from .target import Target +from .utils import slugify + + +class ProjectTopping(QObject): + """ + A project configuration resulting in a YAML file that contains: + - layertree + - layerorder + - project variables (future) + - print layout (future) + - map themes (future) + QML style files, QLR layer definition files and the source of a layer can be linked in the YAML file and are exported to the specific folders. + """ + + stdout = pyqtSignal(str, int) + + PROJECTTOPPING_TYPE = "projecttopping" + LAYERDEFINITION_TYPE = "layerdefinition" + LAYERSTYLE_TYPE = "layerstyle" + + class TreeItemProperties(object): + """ + The properties of a node (tree item) + """ + + def __init__(self): + # if the node is a group + self.group = False + # if the node is visible + self.checked = True + # if the node is expanded + self.expanded = True + # if the (layer) node shows feature count + self.featurecount = False + # if the (group) node handles mutually-exclusive + self.mutually_exclusive = False + # if the (group) node handles mutually-exclusive, the index of the checked child + self.mutually_exclusive_child = -1 + # the layers provider to create it from source + self.provider = None + # the layers uri to create it from source + self.uri = None + # the style file - if None then not requested + self.qmlstylefile = None + # the definition file - if None then not requested + self.definitionfile = None + + class LayerTreeItem(object): + """ + A tree item of the layer tree. Every item contains the properties of a layer and according the ExportSettings passed on parsing the QGIS project. + """ + + def __init__(self): + self.items = [] + self.name = None + self.properties = ProjectTopping.TreeItemProperties() + self.temporary_toppingfile_dir = os.path.expanduser("~/.temp_topping_files") + + def make_item( + self, + node: Union[QgsLayerTreeLayer, QgsLayerTreeGroup], + export_settings: ExportSettings, + ): + # properties for every kind of nodes + self.name = node.name() + self.properties.checked = node.itemVisibilityChecked() + self.properties.expanded = node.isExpanded() + + if isinstance(node, QgsLayerTreeLayer): + self.properties.featurecount = node.customProperty("showFeatureCount") + source_setting = export_settings.get_setting( + ExportSettings.ToppingType.SOURCE, node, node.name() + ) + if source_setting.get("export", False): + if node.layer().dataProvider(): + if node.layer().dataProvider(): + self.properties.provider = ( + node.layer().dataProvider().name() + ) + self.properties.uri = ( + QgsProject.instance() + .pathResolver() + .writePath(node.layer().publicSource()) + ) + qml_setting = export_settings.get_setting( + ExportSettings.ToppingType.QMLSTYLE, node, node.name() + ) + if qml_setting.get("export", False): + self.properties.qmlstylefile = self._temporary_qmlstylefile( + node, + QgsMapLayer.StyleCategory( + qml_setting.get( + "categories", + QgsMapLayer.StyleCategory.AllStyleCategories, + ) + ), + ) + elif isinstance(node, QgsLayerTreeGroup): + # it's a group + self.properties.group = True + self.properties.mutually_exclusive = node.isMutuallyExclusive() + + index = 0 + for child in node.children(): + item = ProjectTopping.LayerTreeItem() + item.make_item(child, export_settings) + # set the first checked item as mutually exclusive child + if ( + self.properties.mutually_exclusive + and self.properties.mutually_exclusive_child == -1 + ): + if item.properties.checked: + self.properties.mutually_exclusive_child = index + self.items.append(item) + index += 1 + else: + print( + f"Here with {node.name()} we have the problem with the LayerTreeNode (it recognizes on QgsLayerTreeLayer QgsLayerTreeNode instead. Similar to https://github.com/opengisch/QgisModelBaker/pull/514 - this needs a fix - maybe in QGIS" + ) + return + + definition_setting = export_settings.get_setting( + ExportSettings.ToppingType.DEFINITION, node, node.name() + ) + if definition_setting.get("export", False): + self.properties.definitionfile = self._temporary_definitionfile(node) + + def _temporary_definitionfile( + self, node: Union[QgsLayerTreeLayer, QgsLayerTreeGroup] + ): + filename_slug = f"{slugify(self.name)}.qlr" + os.makedirs(self.temporary_toppingfile_dir, exist_ok=True) + temporary_toppingfile_path = os.path.join( + self.temporary_toppingfile_dir, filename_slug + ) + QgsLayerDefinition.exportLayerDefinition(temporary_toppingfile_path, [node]) + return temporary_toppingfile_path + + def _temporary_qmlstylefile( + self, + node: QgsLayerTreeLayer, + categories: QgsMapLayer.StyleCategories = QgsMapLayer.StyleCategory.AllStyleCategories, + ): + filename_slug = f"{slugify(self.name)}.qml" + os.makedirs(self.temporary_toppingfile_dir, exist_ok=True) + temporary_toppingfile_path = os.path.join( + self.temporary_toppingfile_dir, filename_slug + ) + node.layer().saveNamedStyle(temporary_toppingfile_path, categories) + return temporary_toppingfile_path + + def __init__(self): + QObject.__init__(self) + self.layertree = self.LayerTreeItem() + self.layerorder = [] + + def parse_project( + self, project: QgsProject, export_settings: ExportSettings = ExportSettings() + ): + """ + Parses a project into the ProjectTopping structure. Means the LayerTreeNodes are loaded into the layertree variable and append the ExportSettings to each node. The CustomLayerOrder is loaded into the layerorder. The project is not keeped as member variable. + + :param QgsProject project: the project to parse. + :param ExportSettings settings: defining if the node needs a source or style / definitionfiles. + """ + root = project.layerTreeRoot() + if root: + self.layertree.make_item(project.layerTreeRoot(), export_settings) + self.layerorder = ( + root.customLayerOrder() if root.hasCustomLayerOrder() else [] + ) + self.stdout.emit( + self.tr("QGIS project parsed with export settings."), Qgis.Info + ) + else: + self.stdout.emit( + self.tr("Could not parse the QGIS project..."), Qgis.Warning + ) + return False + return True + + def generate_files(self, target: Target) -> str: + """ + Generates all files according to the passed Target. + + :param Target target: the target object containing the paths where to create the files and the path_resolver defining the structure of the link. + """ + # generate projecttopping as a dict + projecttopping_dict = self._projecttopping_dict(target) + + # write the yaml + projecttopping_slug = f"{slugify(target.projectname)}.yaml" + absolute_filedir_path, relative_filedir_path = target.filedir_path( + ProjectTopping.PROJECTTOPPING_TYPE + ) + with open( + os.path.join(absolute_filedir_path, projecttopping_slug), "w" + ) as projecttopping_yamlfile: + yaml.dump(projecttopping_dict, projecttopping_yamlfile) + self.stdout.emit( + self.tr("Project Topping written to YAML file: {}").format( + projecttopping_yamlfile + ), + Qgis.Info, + ) + return target.path_resolver( + target, projecttopping_slug, ProjectTopping.PROJECTTOPPING_TYPE + ) + + def load_files(self, target: Target): + """ + - [ ] Not yet implemented. + """ + raise NotImplementedError + + def generate_project(self, target: Target) -> QgsProject: + """ + - [ ] Not yet implemented. + """ + return QgsProject() + + def _projecttopping_dict(self, target: Target): + """ + Creates the layertree as a dict. + Creates the layerorder as a list. + And it generates and stores the toppingfiles according th the Target. + """ + projecttopping_dict = {} + projecttopping_dict["layertree"] = self._item_dict_list( + target, self.layertree.items + ) + projecttopping_dict["layerorder"] = [layer.name() for layer in self.layerorder] + return projecttopping_dict + + def _item_dict_list(self, target: Target, items): + item_dict_list = [] + print([item.name for item in items]) + for item in items: + item_dict = self._create_item_dict(target, item) + item_dict_list.append(item_dict) + return item_dict_list + + def _create_item_dict(self, target: Target, item: LayerTreeItem): + item_dict = {} + item_properties_dict = {} + + if item.properties.group: + item_properties_dict["group"] = True + if item.properties.mutually_exclusive: + item_properties_dict["mutually-exclusive"] = True + item_properties_dict[ + "mutually-exclusive-child" + ] = item.properties.mutually_exclusive_child + else: + if item.properties.featurecount: + item_properties_dict["featurecount"] = True + if item.properties.qmlstylefile: + item_properties_dict["qmlstylefile"] = target.toppingfile_link( + ProjectTopping.LAYERSTYLE_TYPE, item.properties.qmlstylefile + ) + if item.properties.provider and item.properties.uri: + item_properties_dict["provider"] = item.properties.provider + item_properties_dict["uri"] = item.properties.uri + + item_properties_dict["checked"] = item.properties.checked + item_properties_dict["expanded"] = item.properties.expanded + + if item.properties.definitionfile: + item_properties_dict["definitionfile"] = target.toppingfile_link( + ProjectTopping.LAYERDEFINITION_TYPE, + item.properties.definitionfile, + ) + + if item.items: + child_item_dict_list = self._item_dict_list(target, item.items) + item_properties_dict["child-nodes"] = child_item_dict_list + + item_dict[item.name] = item_properties_dict + return item_dict diff --git a/toppingmaker/target.py b/toppingmaker/target.py new file mode 100644 index 0000000..413e4a9 --- /dev/null +++ b/toppingmaker/target.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + ------------------- + begin : 2022-07-17 + git sha : :%H$ + copyright : (C) 2022 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" +import os +import shutil + +from .utils import slugify + + +class Target(object): + """ + The target defines where to store the topping files (YAML, style, definition etc.) + + If there is no subdir it will look like: + + ├── projecttopping + │ └── .yaml + ├── layerstyle + │ ├── _.qml + │ └── _.qml + └── layerdefinition + │ └── _.qlr + + With subdir: + + ├── + │ ├── projecttopping + │ │ └── .yaml + │ ├── layerstyle + │ │ ├── _.qml + │ │ └── _.qml + │ └── layerdefinition + │ │ └── _.qlr + """ + + def __init__( + self, + projectname: str = "project", + main_dir: str = None, + sub_dir: str = None, + path_resolver=None, + ): + self.projectname = projectname + self.main_dir = main_dir + self.sub_dir = sub_dir + self.path_resolver = path_resolver + + if not path_resolver: + self.path_resolver = self.default_path_resolver + + self.toppingfileinfo_list = [] + + def filedir_path(self, file_dir): + relative_path = os.path.join(self.sub_dir, file_dir) + absolute_path = os.path.join(self.main_dir, relative_path) + if not os.path.exists(absolute_path): + os.makedirs(absolute_path) + return absolute_path, relative_path + + def toppingfile_link(self, type: str, path: str): + filename_slug = f"{slugify(self.projectname)}_{os.path.basename(path)}" + absolute_filedir_path, relative_filedir_path = self.filedir_path(type) + shutil.copy( + path, + os.path.join(absolute_filedir_path, filename_slug), + ) + return self.path_resolver(self, filename_slug, type) + + @staticmethod + def default_path_resolver(target, name, type): + _, relative_filedir_path = target.filedir_path(type) + + toppingfile = {"path": os.path.join(relative_filedir_path, name), "type": type} + target.toppingfileinfo_list.append(toppingfile) + + return os.path.join(relative_filedir_path, name) diff --git a/toppingmaker/utils.py b/toppingmaker/utils.py new file mode 100644 index 0000000..c95df12 --- /dev/null +++ b/toppingmaker/utils.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + ------------------- + begin : 2022-07-17 + git sha : :%H$ + copyright : (C) 2022 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" +import re +import unicodedata + + +def slugify(text: str) -> str: + if not text: + return text + slug = unicodedata.normalize("NFKD", text) + slug = re.sub(r"[^a-zA-Z0-9]+", "_", slug).strip("_") + slug = re.sub(r"[-]+", "_", slug) + slug = slug.lower() + return slug