From d60f0ccbac3d1cbbe2c54cb873307db57b235b75 Mon Sep 17 00:00:00 2001 From: Max Fischer Date: Thu, 25 Mar 2021 17:42:02 +0100 Subject: [PATCH 1/4] added documentation to config/plugin backend --- src/cobald/daemon/config/mapping.py | 13 +++--------- src/cobald/daemon/core/config.py | 33 ++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/cobald/daemon/config/mapping.py b/src/cobald/daemon/config/mapping.py index 5452f8e1..d0bb5157 100644 --- a/src/cobald/daemon/config/mapping.py +++ b/src/cobald/daemon/config/mapping.py @@ -145,16 +145,9 @@ def load(cls, entry_point: EntryPoint) -> "SectionPlugin": """ Load a plugin from a pre-parsed entry point - Parses the following options: - - ``required`` - If present implies ``required=True``. - - ``before=other`` - This plugin must be processed before ``other``. - - ``after=other`` - This plugin must be processed after ``other``. + The entry point name is used as the configuration ``section`` to digest + by the entry point target. Additional metadata may be provided by decorating + the target implementation with :py:func:`cobald.daemon.plugins.constraints`. """ digest = entry_point.load() requirements = getattr(digest, "__requirements__", PluginRequirements()) diff --git a/src/cobald/daemon/core/config.py b/src/cobald/daemon/core/config.py index 93de0e53..afcfa911 100644 --- a/src/cobald/daemon/core/config.py +++ b/src/cobald/daemon/core/config.py @@ -1,3 +1,23 @@ +""" +The configuration machinery to load and apply the configuration based on plugins + +Configuration is geared towards digesting structured mappings/YAML, +with plugins for digesting either entire sections or individual nodes. + +* *Constructor Plugins* are callables that receive part of a YAML during loading. +* *Section Plugins* are callables that receive a top-level section of the + configuration after loading. + +The plugins are loaded from Python +`entry points `_ +which point at callables inside normal Python packages +(the section ``"logging"`` being the only exception). +The machinery is implemented by :py:func:`~.load`, +which in turn uses helpers to load plugins of each type. + +The core part of cobald's functionality, a controller >> pool pipeline, +is itself a section plugin as :py:func:`~.load_pipeline`. +""" import os from contextlib import contextmanager from typing import Type, Tuple, Dict, Set @@ -16,15 +36,16 @@ class COBalDLoader(SafeLoader): - """Loader with access to COBalD configuration constructors""" + """YAML loader with access to COBalD configuration constructors""" +# Loading of plugins from entry_points def add_constructor_plugins(entry_point_group: str, loader: Type[BaseLoader]) -> None: """ Add PyYAML constructors from an entry point group to a loader :param loader: the PyYAML loader which uses the plugins - :param entry_point_group: entry point group to search + :param entry_point_group: name of entry point group to search .. note:: @@ -48,10 +69,10 @@ def add_constructor_plugins(entry_point_group: str, loader: Type[BaseLoader]) -> def load_section_plugins(entry_point_group: str) -> Tuple[SectionPlugin]: """ - Load configuration plugins from an entry point group + Load configuration section plugins from an entry point group - :param entry_point_group: entry point group to search - :return: all loaded plugins + :param entry_point_group: name of entry point group to search + :return: all loaded plugins sorted by before/after dependencies """ plugins: Dict[str, SectionPlugin] = { plugin.section: plugin @@ -70,6 +91,7 @@ def load_section_plugins(entry_point_group: str) -> Tuple[SectionPlugin]: ) +# The high-level config machinery implementation itself @contextmanager def load(config_path: str): """ @@ -97,6 +119,7 @@ def load(config_path: str): yield +# The plugin for loading a cobald pipeline @plugin_constraints(required=True) def load_pipeline(content: list): """ From 41d611c1b5eee1a7dcd9a750b1df8eb7295cd383 Mon Sep 17 00:00:00 2001 From: Max Fischer Date: Thu, 25 Mar 2021 18:17:10 +0100 Subject: [PATCH 2/4] constraints decorator supports validation schemas --- setup.py | 1 + src/cobald/daemon/plugins.py | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 1bd2f84f..983e6e3a 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ "entrypoints", "toposort", "typing_extensions", + "pydantic", ], extras_require={ "docs": ["sphinx", "sphinxcontrib-tikz", "sphinx_rtd_theme"], diff --git a/src/cobald/daemon/plugins.py b/src/cobald/daemon/plugins.py index 91030d99..ea29456e 100644 --- a/src/cobald/daemon/plugins.py +++ b/src/cobald/daemon/plugins.py @@ -1,7 +1,9 @@ """ Tools and helpers to declare plugins """ -from typing import Iterable, FrozenSet, TypeVar +from typing import Iterable, FrozenSet, TypeVar, Optional, Type + +from pydantic import BaseModel T = TypeVar("T") @@ -10,36 +12,44 @@ class PluginRequirements: """Requirements of a :py:class:`~.SectionPlugin`""" - __slots__ = "required", "before", "after" + __slots__ = "required", "before", "after", "schema" def __init__( self, required: bool = False, before: FrozenSet[str] = frozenset(), after: FrozenSet[str] = frozenset(), + schema: Optional[Type[BaseModel]] = None, ): self.required = required self.before = before self.after = after + self.schema = schema def __repr__(self): return ( f"{self.__class__.__name__}" f"(required={self.required}," f" before={self.before}," - f" after={self.after})" + f" after={self.after})," + f" schema={self.schema})" ) def constraints( - *, before: Iterable[str] = (), after: Iterable[str] = (), required: bool = False + *, + before: Iterable[str] = (), + after: Iterable[str] = (), + required: bool = False, + schema: Optional[Type[BaseModel]] = None, ): """ - Mark a callable as a plugin with constraints + Mark a callable as a configuration section plugin with constraints :param before: other plugins that must execute before this one :param after: other plugins that must execute after this one :param required: whether it is an error if the plugin does not apply + :param schema: schema for validation of the section .. note:: @@ -49,7 +59,10 @@ def constraints( def section_wrapper(plugin: T) -> T: plugin.__requirements__ = PluginRequirements( - required=required, before=frozenset(before), after=frozenset(after) + required=required, + before=frozenset(before), + after=frozenset(after), + schema=schema, ) return plugin From eea194da0781b54d89bcb653dc67cda7c4152f62 Mon Sep 17 00:00:00 2001 From: Max Fischer Date: Thu, 6 May 2021 15:05:01 +0200 Subject: [PATCH 3/4] applying plugin schema if available --- src/cobald/daemon/config/mapping.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cobald/daemon/config/mapping.py b/src/cobald/daemon/config/mapping.py index d0bb5157..117f5148 100644 --- a/src/cobald/daemon/config/mapping.py +++ b/src/cobald/daemon/config/mapping.py @@ -199,6 +199,8 @@ def load_configuration( where="root", what="missing section %r" % plugin.section ) else: + if plugin.requirements.schema is not None: + section_data = plugin.requirements.schema(section_data) # invoke the plugin and store possible output # to avoid it being garbage collected plugin_content = plugin.digest(section_data) From 53dd68dc95336c145ab421a88e103801f472e562 Mon Sep 17 00:00:00 2001 From: Max Fischer Date: Thu, 6 May 2021 15:18:28 +0200 Subject: [PATCH 4/4] added change fragment for documentation --- docs/source/changes/90.config_schema.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/source/changes/90.config_schema.yaml diff --git a/docs/source/changes/90.config_schema.yaml b/docs/source/changes/90.config_schema.yaml new file mode 100644 index 00000000..4e503128 --- /dev/null +++ b/docs/source/changes/90.config_schema.yaml @@ -0,0 +1,16 @@ +category: changed +summary: "YAML configuration validation/conversion via schemas" +description: | + Plugins for YAML configurations can define a schema for their configuration section. + Schemas can validate and optionally convert the configuration to match the plugin, + and promptly report any failures. + + Schemas are defined via :py:func:`cobald.daemon.plugins.constraints`, + just like any other plugin settings. + The only currently supported schema type is a :py:mod:`pydantic` + :py:class:`~pydantic.BaseModel`. +issues: +- 75 +pull requests: +- 77 +version: 0.12.1