Add configuration schemas

docs/source/changes/90.config_schema.yaml

@@ -0,0 +1,16 @@
+category: changed
+summary: "YAML configuration validation/conversion via schemas"
+description: |
+  Plugins for YAML configurations can define a schema for their configuration section.
+  Schemas can validate and optionally convert the configuration to match the plugin,
+  and promptly report any failures.
+
+  Schemas are defined via :py:func:`cobald.daemon.plugins.constraints`,
+  just like any other plugin settings.
+  The only currently supported schema type is a :py:mod:`pydantic`
+  :py:class:`~pydantic.BaseModel`.
+issues:
+- 75
+pull requests:
+- 77
+version: 0.12.1

setup.py

@@ -58,6 +58,7 @@
             "entrypoints",
             "toposort",
             "typing_extensions",
+            "pydantic",
         ],
         extras_require={
             "docs": ["sphinx", "sphinxcontrib-tikz", "sphinx_rtd_theme"],

src/cobald/daemon/config/mapping.py

@@ -145,16 +145,9 @@ def load(cls, entry_point: EntryPoint) -> "SectionPlugin":
         """
         Load a plugin from a pre-parsed entry point
 
-        Parses the following options:
-
-        ``required``
-            If present implies ``required=True``.
-
-        ``before=other``
-            This plugin must be processed before ``other``.
-
-        ``after=other``
-            This plugin must be processed after ``other``.
+        The entry point name is used as the configuration ``section`` to digest
+        by the entry point target. Additional metadata may be provided by decorating
+        the target implementation with :py:func:`cobald.daemon.plugins.constraints`.
         """
         digest = entry_point.load()
         requirements = getattr(digest, "__requirements__", PluginRequirements())
@@ -206,6 +199,8 @@ def load_configuration(
                     where="root", what="missing section %r" % plugin.section
                 )
         else:
+            if plugin.requirements.schema is not None:
+                section_data = plugin.requirements.schema(section_data)
             # invoke the plugin and store possible output
             # to avoid it being garbage collected
             plugin_content = plugin.digest(section_data)

src/cobald/daemon/core/config.py

@@ -1,3 +1,23 @@
+"""
+The configuration machinery to load and apply the configuration based on plugins
+
+Configuration is geared towards digesting structured mappings/YAML,
+with plugins for digesting either entire sections or individual nodes.
+
+* *Constructor Plugins* are callables that receive part of a YAML during loading.
+* *Section Plugins* are callables that receive a top-level section of the
+  configuration after loading.
+
+The plugins are loaded from Python
+`entry points <https://packaging.python.org/specifications/entry-points/>`_
+which point at callables inside normal Python packages
+(the section ``"logging"`` being the only exception).
+The machinery is implemented by :py:func:`~.load`,
+which in turn uses helpers to load plugins of each type.
+
+The core part of cobald's functionality, a controller >> pool pipeline,
+is itself a section plugin as :py:func:`~.load_pipeline`.
+"""
 import os
 from contextlib import contextmanager
 from typing import Type, Tuple, Dict, Set
@@ -16,15 +36,16 @@
 
 
 class COBalDLoader(SafeLoader):
-    """Loader with access to COBalD configuration constructors"""
+    """YAML loader with access to COBalD configuration constructors"""
 
 
+# Loading of plugins from entry_points
 def add_constructor_plugins(entry_point_group: str, loader: Type[BaseLoader]) -> None:
     """
     Add PyYAML constructors from an entry point group to a loader
 
     :param loader: the PyYAML loader which uses the plugins
-    :param entry_point_group: entry point group to search
+    :param entry_point_group: name of entry point group to search
 
     .. note::
 
@@ -48,10 +69,10 @@ def add_constructor_plugins(entry_point_group: str, loader: Type[BaseLoader]) ->
 
 def load_section_plugins(entry_point_group: str) -> Tuple[SectionPlugin]:
     """
-    Load configuration plugins from an entry point group
+    Load configuration section plugins from an entry point group
 
-    :param entry_point_group: entry point group to search
-    :return: all loaded plugins
+    :param entry_point_group: name of entry point group to search
+    :return: all loaded plugins sorted by before/after dependencies
     """
     plugins: Dict[str, SectionPlugin] = {
         plugin.section: plugin
@@ -70,6 +91,7 @@ def load_section_plugins(entry_point_group: str) -> Tuple[SectionPlugin]:
     )
 
 
+# The high-level config machinery implementation itself
 @contextmanager
 def load(config_path: str):
     """
@@ -97,6 +119,7 @@ def load(config_path: str):
     yield
 
 
+# The plugin for loading a cobald pipeline
 @plugin_constraints(required=True)
 def load_pipeline(content: list):
     """

src/cobald/daemon/plugins.py

@@ -1,7 +1,9 @@
 """
 Tools and helpers to declare plugins
 """
-from typing import Iterable, FrozenSet, TypeVar
+from typing import Iterable, FrozenSet, TypeVar, Optional, Type
+
+from pydantic import BaseModel
 
 
 T = TypeVar("T")
@@ -10,36 +12,44 @@
 class PluginRequirements:
     """Requirements of a :py:class:`~.SectionPlugin`"""
 
-    __slots__ = "required", "before", "after"
+    __slots__ = "required", "before", "after", "schema"
 
     def __init__(
         self,
         required: bool = False,
         before: FrozenSet[str] = frozenset(),
         after: FrozenSet[str] = frozenset(),
+        schema: Optional[Type[BaseModel]] = None,
     ):
         self.required = required
         self.before = before
         self.after = after
+        self.schema = schema
 
     def __repr__(self):
         return (
             f"{self.__class__.__name__}"
             f"(required={self.required},"
             f" before={self.before},"
-            f" after={self.after})"
+            f" after={self.after}),"
+            f" schema={self.schema})"
         )
 
 
 def constraints(
-    *, before: Iterable[str] = (), after: Iterable[str] = (), required: bool = False
+    *,
+    before: Iterable[str] = (),
+    after: Iterable[str] = (),
+    required: bool = False,
+    schema: Optional[Type[BaseModel]] = None,
 ):
     """
-    Mark a callable as a plugin with constraints
+    Mark a callable as a configuration section plugin with constraints
 
     :param before: other plugins that must execute before this one
     :param after: other plugins that must execute after this one
     :param required: whether it is an error if the plugin does not apply
+    :param schema: schema for validation of the section
 
     .. note::
 
@@ -49,7 +59,10 @@ def constraints(
 
     def section_wrapper(plugin: T) -> T:
         plugin.__requirements__ = PluginRequirements(
-            required=required, before=frozenset(before), after=frozenset(after)
+            required=required,
+            before=frozenset(before),
+            after=frozenset(after),
+            schema=schema,
         )
         return plugin