Update doc creation, Fix content (#1854)

* first updates

* add docs

* add first bunch of work

* final fix of doc refactoring

* fix lint issues

* add makefile2graph explanations

* Update doc/source/makefile.rst

remove leading whitespace to ensure correct HTML formatting

* remove file from versioning

---------

Co-authored-by: Julian <[email protected]>

.gitignore

@@ -27,6 +27,14 @@ __pycache__/
 /doc/source/contrib/print_proxy.rst
 /doc/source/contrib/stats.rst
 /doc/source/api.rst
+/doc/source/contrib/print_proxy/index.rst
+/doc/source/contrib/print_proxy/mapfish-print.rst
+/doc/source/contrib/print_proxy/xml2pdf.rst
+/doc/source/contrib/data_sources/standard/models-main.rst
+/doc/source/contrib/data_sources/standard/models-theme.rst
+/doc/source/contrib/data_sources/standard/models.rst
+/doc/source/contrib/data_sources/swisstopo/index.rst
+/doc/source/contrib/data_sources/interlis_2_3/index.rst
 /doc/build/
 
 # Standard configuration

Makefile

@@ -336,7 +336,7 @@ docker-clean-all:
 	docker compose run --rm oereb-make clean-all
 
 .PHONY: check
-check: git-attributes lint test
+check: git-attributes lint tests
 
 .PHONY: doc-latex
 doc-latex: ${VENV_ROOT}/requirements-timestamp
@@ -349,7 +349,7 @@ doc-html: ${VENV_ROOT}/requirements-timestamp
 	$(VENV_BIN)/sphinx-build -b html doc/source doc/build/html
 
 .PHONY: updates
-updates: $(PIP_REQUIREMENTS)
+updates: ${VENV_ROOT}/requirements-timestamp
 	$(VENV_BIN)/pip list --outdated
 
 .PHONY: serve-dev

doc/source/conf.py

@@ -17,19 +17,12 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-from mako.template import Template
-import sys
-import inspect
 # sys.path.insert(0, os.path.abspath('.'))
 import re
-import types
 import subprocess
 import sphinx_rtd_theme
 from pyramid_oereb.core.config import Config
 Config._config = {'srid': -1, 'app_schema': {'name': 'pyramid_oereb_main'}}
-from pyramid_oereb.contrib.data_sources.standard.sources.plr import StandardThemeConfigParser  # noqa E402
-import pyramid_oereb.contrib.data_sources.standard  # noqa E402
-import pyramid_oereb.contrib.data_sources.standard.models.main  # noqa E402
 
 
 # -- General configuration ------------------------------------------------
@@ -65,73 +58,30 @@
         '../../.venv/bin/mako-render' if os.path.exists('../../.venv/bin/mako-render') else 'mako-render',
         'core/sources.rst.mako']).decode('utf-8'))
 
-with open('standard/sources.rst', 'w') as sources:
-    sources.write(subprocess.check_output([
-        '../../.venv/bin/mako-render' if os.path.exists('../../.venv/bin/mako-render') else 'mako-render',
-        'standard/sources.rst.mako']).decode('utf-8'))
-
-with open('contrib/sources.rst', 'w') as sources:
-    sources.write(subprocess.check_output([
-        '../../.venv/bin/mako-render' if os.path.exists('../../.venv/bin/mako-render') else 'mako-render',
-        'contrib/sources.rst.mako']).decode('utf-8'))
-
-with open('contrib/print_proxy.rst', 'w') as sources:
-    sources.write(subprocess.check_output([
-        '../../.venv/bin/mako-render' if os.path.exists('../../.venv/bin/mako-render') else 'mako-render',
-        'contrib/print_proxy.rst.mako']).decode('utf-8'))
-
 with open('contrib/stats.rst', 'w') as sources:
     sources.write(subprocess.check_output([
         '../../.venv/bin/mako-render' if os.path.exists('../../.venv/bin/mako-render') else 'mako-render',
         'contrib/stats.rst.mako']).decode('utf-8'))
+target_paths = [
+    'contrib/data_sources/standard/index.rst',
+    'contrib/data_sources/standard/models.rst',
+    'contrib/data_sources/standard/models-main.rst',
+    'contrib/data_sources/standard/models-theme.rst',
+    'contrib/data_sources/standard/sources.rst',
+    'contrib/data_sources/interlis_2_3/index.rst',
+    'contrib/data_sources/oereblex/index.rst',
+    'contrib/data_sources/swisstopo/index.rst',
+    'contrib/data_sources/index.rst',
+    'contrib/print_proxy/index.rst',
+    'contrib/print_proxy/mapfish-print.rst',
+    'contrib/print_proxy/xml2pdf.rst'
+]
 
-module_file_names = []
-module_name = 'pyramid_oereb.contrib.data_sources.standard.models.main'
-file_name = 'pyramid_oereb_contrib_data_sources_standard_models_main'
-module_file_names.append(file_name)
-main_classes = []
-for name, obj in inspect.getmembers(pyramid_oereb.contrib.data_sources.standard.models.main):
-    if inspect.isclass(obj) and obj.__module__ == module_name:
-        main_classes.append(name)
-with open('standard/models/{name}.rst'.format(name=file_name), 'w') as sources:
-    template = Template(filename='standard/models/models.rst.mako')
-    sources.write(template.render(**{'module_name': module_name, 'classes': main_classes}))
-
-conf_parser = StandardThemeConfigParser(source={
-    "class": 'pyramid_oereb.contrib.data_sources.standard.sources.plr.DatabaseSource',
-    "params": {
-        "model_factory": "pyramid_oereb.contrib.data_sources.standard.models.theme.model_factory_string_pk",
-        "schema_name": "land_use_plans"
-    }
-})
-
-models = conf_parser.get_models()
-modelnames = [modelname for modelname in dir(models) if modelname[0].isupper() and modelname != 'Base']
-
-# create fake python module pyramid_oereb.contrib.data_sources.standard.factory_models
-factory_models = types.ModuleType('FactoryModels', 'Virtual module for factory generated classes')
-factory_models.__name__ = 'factory_models'
-factory_models.__mro__ = []
-factory_models.__module__ = 'pyramid_oereb.contrib.data_sources.standard'
-for modelname in modelnames:
-    model = models.__getattribute__(modelname)
-    model.__name__ = modelname
-    model.__module__ = 'pyramid_oereb.contrib.data_sources.standard.factory_models'
-    factory_models.__dict__.update({modelname: model})
-
-pyramid_oereb.contrib.data_sources.standard.__dict__.update({'factory_models': factory_models})
-sys.modules['pyramid_oereb.contrib.data_sources.standard.factory_models'] = factory_models
-
-module_name = 'pyramid_oereb.contrib.data_sources.standard.factory_models'
-file_name = module_name.replace('.', '_').lower()
-module_file_names.append(file_name)
-with open('standard/models/{name}.rst'.format(name=file_name), 'w') as sources:
-    template = Template(filename='standard/models/factory_models.rst.mako')
-    sources.write(template.render(**{'module_name': module_name, 'classes': modelnames}))
-
-with open('standard/models/index.rst', 'w') as sources:
-    template = Template(filename='standard/models/index.rst.mako')
-    sources.write(template.render(**{'module_file_names': module_file_names}))
+for target_path in target_paths:
+    with open(target_path, 'w') as sources:
+        sources.write(subprocess.check_output([
+            '../../.venv/bin/mako-render' if os.path.exists('../../.venv/bin/mako-render') else 'mako-render',
+            f'{target_path}.mako']).decode('utf-8'))
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['doc/_buildtemplates']

doc/source/configuration.rst

@@ -109,48 +109,6 @@ structure. This method is recommended if you are using an existing database supp
 already containing all the necessary data but in a different structure. In this case you should check, if it
 is possible to transform the data by extending the existing models with a mapping to fit your structure.
 
-The easiest example is a simple mapping of table and column names, if you use a different language. Using the
-possibilities of SQLAlchemy, you could extend the existing
-:ref:`api-pyramid_oereb-standard-models-motorways_building_lines-office` like this:
-
-.. code-block:: python
-
-   from pyramid_oereb.lib.standard.models import motorways_building_lines
-
-   class Office(motorways_building_lines.Office):
-       """
-       The bucket to fill in all the offices you need to reference from public law restriction,
-       document, geometry.
-
-       Attributes:
-           id (int): The identifier. This is used in the database only and must not be set manually.
-               If you don't like it - don't care about.
-           name (dict): The multilingual name of the office.
-           office_at_web (str): A web accessible url to a presentation of this office.
-           uid (str): The uid of this office from https
-           line1 (str): The first address line for this office.
-           line2 (str): The second address line for this office.
-           street (str): The streets name of the offices address.
-           number (str): The number on street.
-           postal_code (int): The ZIP-code.
-           city (str): The name of the city.
-       """
-       __table_args__ = {'schema': 'baulinien_nationalstrassen'}
-       __tablename__ = 'amt'
-       id = sa.Column('oid', sa.Integer, primary_key=True)
-       office_at_web = sa.Column('amt_im_web', sa.String, nullable=True)
-       line1 = sa.Column('zeile1', sa.String, nullable=True)
-       line2 = sa.Column('zeile2', sa.String, nullable=True)
-       street = sa.Column('strasse', sa.String, nullable=True)
-       number = sa.Column('hausnr', sa.String, nullable=True)
-       postal_code = sa.Column('plz', sa.Integer, nullable=True)
-       city = sa.Column('ort', sa.String, nullable=True)
-
-       (...)
-
-The only thing, you have to care about, if you want to stay using the standard sources, is to keep the class
-name, the names of the properties and their data types.
-
 After extending the models, do not forget to change the models module in the configuration of the topic's
 source.
 
@@ -204,11 +162,11 @@ restrictions on implementing a custom source:
 
    1.  The source has to implement the method `read()` with the arguments used in its base source. For
        example, your custom real estate source has to accept the arguments defined in
-       :ref:`api-pyramid_oereb-lib-sources-real_estate-realestatebasesource`.
+       :ref:`api-pyramid_oereb-contrib-data_sources-standard-sources-real_estate-databasesource`.
 
    2.  The method `read()` has to add records of the corresponding type to the source' records list. Every
        source has list property called `records`. In case of a real estate source, the method `read()` has to
-       create one or more instances of the :ref:`api-pyramid_oereb-lib-records-real_estate-realestaterecord`
+       create one or more instances of the :ref:`api-pyramid_oereb-core-records-real_estate-realestaterecord`
        and add them to this list.
 
 This way, you should be able to create sources for nearly every possible data source.

doc/source/contrib/data_sources/index.rst

@@ -0,0 +1,29 @@
+.. _contrib-data-sources:
+
+Data Sources
+------------
+
+``pyramid_oereb`` provides a plugable system to offer an extendable system. The most
+interesting part of that is probably the extension of what data sources should be
+connected to the core system of ``pyramid_oereb``.
+
+As of now ``pyramid_oereb`` offers the following data sources (db structures) you can store
+your data to:
+
+* :ref:`contrib-data-sources-standard`
+* oereblex
+* interlis 2.3
+
+
+
+In addition there is a source to use with the address localisation:
+
+* swisstopo
+
+.. toctree::
+   :hidden:
+
+   standard/index
+   oereblex/index
+   interlis_2_3/index
+   swisstopo/index

doc/source/contrib/data_sources/index.rst.mako

@@ -0,0 +1,29 @@
+.. _contrib-data-sources:
+
+Data Sources
+------------
+
+``pyramid_oereb`` provides a plugable system to offer an extendable system. The most
+interesting part of that is probably the extension of what data sources should be
+connected to the core system of ``pyramid_oereb``.
+
+As of now ``pyramid_oereb`` offers the following data sources (db structures) you can store
+your data to:
+
+* :ref:`contrib-data-sources-standard`
+* oereblex
+* interlis 2.3
+
+
+
+In addition there is a source to use with the address localisation:
+
+* swisstopo
+
+.. toctree::
+   :hidden:
+
+   standard/index
+   oereblex/index
+   interlis_2_3/index
+   swisstopo/index

doc/source/contrib/data_sources/interlis_2_3/index.rst.mako

@@ -0,0 +1,9 @@
+.. _contrib-data-sources-interlis-23:
+
+INTERLIS 2.3
+============
+
+
+
+.. toctree::
+   :hidden:

doc/source/contrib/data_sources/oereblex/index.rst

@@ -0,0 +1,8 @@
+.. _contrib-data-sources-oereblex:
+
+ÖREBlex
+=======
+
+
+.. toctree::
+   :hidden:

doc/source/contrib/data_sources/oereblex/index.rst.mako

@@ -0,0 +1,8 @@
+.. _contrib-data-sources-oereblex:
+
+ÖREBlex
+=======
+
+
+.. toctree::
+   :hidden:

doc/source/contrib/data_sources/standard/index.rst.mako

@@ -0,0 +1,43 @@
+.. _contrib-data-sources-standard:
+
+Standard
+^^^^^^^^
+
+The standard data source is a binding to a database following the definitions of
+`OeREBKRMkvs_V2_0 <http://models.geo.admin.ch/V_D/OeREB/OeREBKRMkvs_V2_0.ili>`__
+and `OeREBKRMtrsfr_V2_0 <http://models.geo.admin.ch/V_D/OeREB/OeREBKRMtrsfr_V2_0.ili>`__
+but adds some magic sugar for convenience.
+
+Exemplary schema of a standard database:
+
+ .. image:: ../../../../images/standard_database_schema_example.png
+   :scale: 20 %
+   :align: center
+
+This structure is defined as models for SQL-Alchemy ORM and as a fitting set of
+DataBaseSource-Adapters to hook it into the core of ``pyramid_oereb``. The models
+defined in this contribution package describing 2 different structures. The ``main``
+definitions are used by ``pyramid_oereb`` application itself and contain the application
+configuration as suggested in
+`OeREBKRMkvs_V2_0 <http://models.geo.admin.ch/V_D/OeREB/OeREBKRMkvs_V2_0.ili>`__.
+The ``theme`` definitions define a single ÖREB-Theme as it is suggested by
+`OeREBKRMtrsfr_V2_0 <http://models.geo.admin.ch/V_D/OeREB/OeREBKRMtrsfr_V2_0.ili>`__.
+Please be aware that for performance reasons the schematic mapping of SQL-Alchemy ORM
+definitions and the linked INTERLIS models is not 1:1.
+
+The definition of main and theme schema share some classes. These are provided via
+factory methods at modul level. These methods can be found here :ref:`contrib-data-sources-standard-models`
+
+* :ref:`contrib-data-sources-standard-models`
+* :ref:`contrib-data-sources-standard-models-main`
+* :ref:`contrib-data-sources-standard-models-theme`
+* :ref:`contrib-data-sources-standard-sources`
+
+
+.. toctree::
+   :hidden:
+
+   models
+   models-main
+   models-theme
+   sources

doc/source/contrib/sources.rst.mako → doc/source/contrib/data_sources/standard/models-main.rst.mako

@@ -1,20 +1,23 @@
-.. _contrib-sources:
+.. _contrib-data-sources-standard-models-main:
 
-Sources
--------
+ORM Main schema
+```````````````
 
-<%! import glob, inspect, re, sys %>
+<%!
+import glob, inspect, re, sys
+from pyramid_oereb.core.config import Config
+Config._config = {'srid': -1, 'app_schema': {'name': 'pyramid_oereb_main'}}
+%>
 <%
 modules = [m for m in sys.modules.keys() if m.startswith('pyramid_oereb')]
-files = glob.glob('../../pyramid_oereb/contrib/data_sources/oereblex/sources/*.py')
-files += glob.glob('../../pyramid_oereb/contrib/data_sources/swisstopo/*.py')
-files += glob.glob('../../pyramid_oereb/contrib/data_sources/interlis_2_3/sources/*.py')
+files = glob.glob('../../pyramid_oereb/contrib/data_sources/standard/models/main.py')
 modules = [
     re.sub(r'\.__init__', '', f[6:-3].replace("/", ".")) for f in files
 ]
-
 modules.sort()
+
 delete_modules = []
+
 for i, module in enumerate(modules):
     try:
         __import__(module)
@@ -39,16 +42,16 @@ underline = ['^', '`', '\'', '.', '~', '*']
 
 .. automodule:: ${module}
 
-
 %for cls in classes[module]:
 .. _api-${module.replace('.', '-').lower()}-${cls.lower()}:
 
 *${cls}*
-${re.sub('.', underline[0], 'Class   ' + cls)}
+${underline[2]*len("*"+cls+"*")}
 
 .. autoclass:: ${module}.${cls}
    :members:
    :inherited-members:
+   :show-inheritance:
 
    .. automethod:: __init__