From d75ecb68a17291d75493946c311cf428746a4f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Mon, 28 Dec 2020 22:55:54 +0100 Subject: [PATCH 1/7] Update homepage --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 80fc516..16a586b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ packages = [ { include = "recap" } ] readme = "README.md" -homepage = "https://github.com/georgw777/recap" +homepage = "http://recap.readthedocs.io" repository = "https://github.com/georgw777/recap" keywords = ["flake8", "markdown", "lint"] classifiers = [ From c2b629e94874cc04a47400bc785cab1dad1cdc42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Mon, 28 Dec 2020 23:35:16 +0100 Subject: [PATCH 2/7] Update badges --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1661e0d..9405c2b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # recap -![build](https://github.com/georgw777/recap/workflows/build/badge.svg) -![PyPI](https://img.shields.io/pypi/v/recap) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/recap) -![GitHub](https://img.shields.io/github/license/georgw777/recap) +[![build](https://github.com/georgw777/recap/workflows/build/badge.svg)](https://github.com/georgw777/recap/actions?query=workflow%3Abuild) +[![PyPI](https://img.shields.io/pypi/v/recap)](https://pypi.org/project/recap) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/recap)](https://pypi.org/project/recap) +[![Licence](https://img.shields.io/github/license/georgw777/recap)](https://github.com/georgw777/recap/blob/master/LICENSE) +[![Documentation Status](https://readthedocs.org/projects/recap/badge/?version=latest)](http://recap.readthedocs.io/?badge=latest) _recap_ is a tool for providing _REproducible Configurations for Any Project_. From b5c93988745d4e607aa6ea04eb4612735c4112b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Wed, 30 Dec 2020 11:41:44 +0100 Subject: [PATCH 3/7] Handle relative paths in _BASE_ key --- recap/config.py | 6 +++++- recap/path_manager.py | 10 ++++++++-- tests/resources/data_type_override.yaml | 3 +++ tests/resources/data_type_override_base.yaml | 1 + tests/resources/inherit_base.yaml | 2 ++ tests/resources/inherit_override.yaml | 3 +++ tests/test_config.py | 17 +++++++++++++++++ tests/test_path_manager.py | 16 ++++++++++++++++ 8 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 tests/resources/data_type_override.yaml create mode 100644 tests/resources/data_type_override_base.yaml create mode 100644 tests/resources/inherit_base.yaml create mode 100644 tests/resources/inherit_override.yaml create mode 100644 tests/test_config.py diff --git a/recap/config.py b/recap/config.py index d718e2c..bd0a171 100644 --- a/recap/config.py +++ b/recap/config.py @@ -41,7 +41,11 @@ def load_yaml_with_base(cls, filename: os.PathLike) -> "CfgNode": with uri.open("r") as f: cfg = cls.load_cfg(f) if BASE_KEY in cfg: - base_cfg = cls.load_yaml_with_base(cfg[BASE_KEY]) + base_uri = URI(cfg[BASE_KEY]) + # If the URI of the base cfg is relative, it should be relative to the current YAML file + if not base_uri.is_absolute(): + base_uri = uri.parent / base_uri + base_cfg = cls.load_yaml_with_base(base_uri) del cfg[BASE_KEY] base_cfg.merge_from_other_cfg(cfg) return base_cfg diff --git a/recap/path_manager.py b/recap/path_manager.py index 2450337..c73b40e 100644 --- a/recap/path_manager.py +++ b/recap/path_manager.py @@ -24,6 +24,7 @@ def splitroot(self, part, sep=_PosixFlavour.sep): root = "" return drive, root, path else: + self.has_drv = False return super().splitroot(part, sep=sep) @@ -73,7 +74,8 @@ def resolve(self, path: os.PathLike) -> Path: path = _URIBase(path) if path.scheme: if path.scheme not in self._handlers: - raise NotImplementedError(f"No handler is registered for scheme {path.scheme}") + raise NotImplementedError( + f"No handler is registered for scheme {path.scheme}") return self._handlers[path.scheme](path) else: return Path(path.path) @@ -119,7 +121,8 @@ def __enter__(self, *args, **kwargs): return self._instance.__enter__(*args, **kwargs) def __exit__(self, *args, **kwargs): - return self._instance.__exit__(*args, **kwargs) + return self._instance.__exit__(*args, **kwargs) + #: The public path manager instance. PathManager: PathManagerBase = PathManagerProxy(PathManagerBase()) @@ -138,6 +141,9 @@ def _init(self, *args, **kwargs): def __str__(self) -> str: return str(self._local_path) + def is_absolute(self) -> bool: + return self._local_path.is_absolute() + class PathTranslator(abc.ABC): """Abstract class representing a path translator that can translate a specific type of URIs to local paths. diff --git a/tests/resources/data_type_override.yaml b/tests/resources/data_type_override.yaml new file mode 100644 index 0000000..520c9b7 --- /dev/null +++ b/tests/resources/data_type_override.yaml @@ -0,0 +1,3 @@ +_BASE_: ./data_type_override_base.yaml + +VALUE: my string diff --git a/tests/resources/data_type_override_base.yaml b/tests/resources/data_type_override_base.yaml new file mode 100644 index 0000000..a24b714 --- /dev/null +++ b/tests/resources/data_type_override_base.yaml @@ -0,0 +1 @@ +VALUE: 1234 diff --git a/tests/resources/inherit_base.yaml b/tests/resources/inherit_base.yaml new file mode 100644 index 0000000..2aa0dcd --- /dev/null +++ b/tests/resources/inherit_base.yaml @@ -0,0 +1,2 @@ +TEST: 1 +XYZ: abc diff --git a/tests/resources/inherit_override.yaml b/tests/resources/inherit_override.yaml new file mode 100644 index 0000000..aa52d88 --- /dev/null +++ b/tests/resources/inherit_override.yaml @@ -0,0 +1,3 @@ +_BASE_: inherit_base.yaml + +TEST: 2 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..5564adc --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,17 @@ +from pathlib import Path +import pytest + +from recap import CfgNode as CN + +RESOURCES = Path(__file__).parent / "resources" + + +def test_inherit_yaml(): + cfg = CN.load_yaml_with_base(RESOURCES / "inherit_override.yaml") + assert cfg.TEST == 2 + assert cfg.XYZ == "abc" + + +def test_data_type_override(): + with pytest.raises(ValueError): + cfg = CN.load_yaml_with_base(RESOURCES / "data_type_override.yaml") diff --git a/tests/test_path_manager.py b/tests/test_path_manager.py index 97a96b8..f8f99c3 100644 --- a/tests/test_path_manager.py +++ b/tests/test_path_manager.py @@ -44,3 +44,19 @@ def abc(path: URI) -> Path: return Path("/def") / path.path assert str(manager.resolve("abc://d/e")) == str(Path("/def/d/e")) + + +def test_absolute_path(): + assert URI("/a").is_absolute() + + +def test_relative_path(): + assert not URI("a").is_absolute() + + +def test_absolute_uri(): + with PathManagerBase(): + test_dir = Path("/a/b/test") + register_translator("test", test_dir) + uri = URI("test://a/b") + assert uri.is_absolute() From 89330c0809eba4baab48d50c15d39dc4850f1a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Wed, 30 Dec 2020 11:55:15 +0100 Subject: [PATCH 4/7] Make the PathManagerProxy private --- recap/path_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recap/path_manager.py b/recap/path_manager.py index c73b40e..6cca349 100644 --- a/recap/path_manager.py +++ b/recap/path_manager.py @@ -104,7 +104,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): PathManager._instance = self._previous_path_managers.pop() -class PathManagerProxy(PathManagerBase): +class _PathManagerProxy(PathManagerBase): """Proxy class for the main path manager instance. """ @@ -125,7 +125,7 @@ def __exit__(self, *args, **kwargs): #: The public path manager instance. -PathManager: PathManagerBase = PathManagerProxy(PathManagerBase()) +PathManager: PathManagerBase = _PathManagerProxy(PathManagerBase()) class URI(_URIBase): From fce77deb04a538dfc06c306c9ca7089c75f12498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Wed, 30 Dec 2020 12:18:08 +0100 Subject: [PATCH 5/7] Use wrapt to perform object proxying --- poetry.lock | 4 +- pyproject.toml | 1 + recap/path_manager.py | 75 ++++++++++++++------------------------ tests/test_path_manager.py | 9 +++++ 4 files changed, 40 insertions(+), 49 deletions(-) diff --git a/poetry.lock b/poetry.lock index aecc5d2..77eb8e1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -523,7 +523,7 @@ secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "cer socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] -category = "dev" +category = "main" description = "Module for decorators, wrappers and monkey patching." name = "wrapt" optional = false @@ -558,7 +558,7 @@ testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pyt docs = ["sphinx", "sphinx-rtd-theme", "m2r2"] [metadata] -content-hash = "d43ca2eaba4f190ebdee3a56b2f851f5f40f2c0f4bd6f4e76bd039cf0a38a73b" +content-hash = "dca5bc0b0ce84c75a966a58531442b70de528a1dc5a334fc47af6312efedf1c4" lock-version = "1.0" python-versions = "^3.6" diff --git a/pyproject.toml b/pyproject.toml index 16a586b..ae77630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ pyyaml = "^5.3.1" sphinx = { version = "^3.4.1", optional = true } sphinx-rtd-theme = { version = "^0.5.0", optional = true } m2r2 = { version = "^0.2.7", optional = true } +wrapt = "^1.12.1" [tool.poetry.dev-dependencies] autopep8 = "^1.5.4" diff --git a/recap/path_manager.py b/recap/path_manager.py index 6cca349..d0dfbbd 100644 --- a/recap/path_manager.py +++ b/recap/path_manager.py @@ -6,6 +6,7 @@ from contextlib import contextmanager import re import os +import wrapt logger = logging.getLogger(__name__) @@ -50,6 +51,25 @@ def __repr__(self) -> str: return "{}({!r})".format(self.__class__.__name__, s) +class PathTranslator(abc.ABC): + """Abstract class representing a path translator that can translate a specific type of URIs to local paths. + """ + + def __call__(self, uri: "URI") -> Path: + """Translate a URI to a local path. + + Usually, this involves looking at uri.path. + + Args: + uri (URI): the URI + + Returns: + Path: the corresponding local path + """ + + raise NotImplementedError + + class PathManagerBase: """Base class for a path manager. @@ -80,7 +100,7 @@ def resolve(self, path: os.PathLike) -> Path: else: return Path(path.path) - def register_handler(self, scheme: str) -> callable: + def register_handler(self, scheme: str) -> Callable[[PathTranslator], PathTranslator]: """Decorator to register a path handler for a given URI scheme. Args: @@ -90,42 +110,22 @@ def register_handler(self, scheme: str) -> callable: callable: the decorated function """ - def decorator(func: Callable[[os.PathLike], Path]): - self._handlers[scheme] = func + def decorator(translator: PathTranslator) -> PathTranslator: + self._handlers[scheme] = translator logger.debug(f"Registered path handler for scheme {scheme}") - return func + return translator return decorator def __enter__(self): - self._previous_path_managers.append(PathManager._instance) - PathManager._instance = self + self._previous_path_managers.append(PathManager.__wrapped__) + PathManager.__wrapped__ = self def __exit__(self, exc_type, exc_val, exc_tb): - PathManager._instance = self._previous_path_managers.pop() - - -class _PathManagerProxy(PathManagerBase): - """Proxy class for the main path manager instance. - """ - - def __init__(self, instance: PathManagerBase): - self._instance = instance - - def resolve(self, *args, **kwargs): - return self._instance.resolve(*args, **kwargs) - - def register_handler(self, *args, **kwargs): - return self._instance.register_handler(*args, **kwargs) - - def __enter__(self, *args, **kwargs): - return self._instance.__enter__(*args, **kwargs) - - def __exit__(self, *args, **kwargs): - return self._instance.__exit__(*args, **kwargs) + PathManager.__wrapped__ = self._previous_path_managers.pop() #: The public path manager instance. -PathManager: PathManagerBase = _PathManagerProxy(PathManagerBase()) +PathManager: PathManagerBase = wrapt.ObjectProxy(PathManagerBase()) class URI(_URIBase): @@ -145,25 +145,6 @@ def is_absolute(self) -> bool: return self._local_path.is_absolute() -class PathTranslator(abc.ABC): - """Abstract class representing a path translator that can translate a specific type of URIs to local paths. - """ - - def __call__(self, uri: URI) -> Path: - """Translate a URI to a local path. - - Usually, this involves looking at uri.path. - - Args: - uri (URI): the URI - - Returns: - Path: the corresponding local path - """ - - raise NotImplementedError - - def register_translator(scheme: str, path: Path): """Convenience method to register a path translator that forwards a URI scheme to a local path. diff --git a/tests/test_path_manager.py b/tests/test_path_manager.py index f8f99c3..e0a7c3d 100644 --- a/tests/test_path_manager.py +++ b/tests/test_path_manager.py @@ -11,6 +11,15 @@ def test_uri_concat(): assert str(a) == "/path/a/b" +def test_path_manager_context(): + assert len(PathManager._handlers) == 0 + with PathManagerBase(): + test_dir = Path("/a/b/test") + register_translator("test", test_dir) + assert len(PathManager._handlers) == 1 + assert len(PathManager._handlers) == 0 + + def test_virtual_uri(): with PathManagerBase(): test_dir = Path("/a/b/test") From eb28d327f5f603f4fab0464eb8fa3b78ff3a417b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Wed, 30 Dec 2020 12:42:58 +0100 Subject: [PATCH 6/7] Improve documentation --- docs/source/index.rst | 4 ++-- docs/source/modules.rst | 7 ------- docs/source/recap.rst | 13 +++++-------- recap/__init__.py | 3 +++ recap/config.py | 3 +++ recap/path_manager.py | 9 ++++++--- 6 files changed, 19 insertions(+), 20 deletions(-) delete mode 100644 docs/source/modules.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 56cbca6..a41b994 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,7 +1,7 @@ .. mdinclude:: ../../README.md .. toctree:: - :maxdepth: 2 + :maxdepth: 4 :caption: Contents: - modules + recap diff --git a/docs/source/modules.rst b/docs/source/modules.rst deleted file mode 100644 index 9a9586e..0000000 --- a/docs/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -recap -===== - -.. toctree:: - :maxdepth: 4 - - recap diff --git a/docs/source/recap.rst b/docs/source/recap.rst index 87ead7b..a858f28 100644 --- a/docs/source/recap.rst +++ b/docs/source/recap.rst @@ -1,6 +1,11 @@ recap package ============= +.. automodule:: recap + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -19,11 +24,3 @@ recap.path\_manager module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: recap - :members: - :undoc-members: - :show-inheritance: diff --git a/recap/__init__.py b/recap/__init__.py index 156a51b..e3037b8 100644 --- a/recap/__init__.py +++ b/recap/__init__.py @@ -1,3 +1,6 @@ +"""The recap package. The most important classes, :class:`~recap.config.CfgNode` and :class:`~recap.path_manager.URI` are available as top-level exports. +""" + from .__version__ import __version__ from .config import CfgNode from .path_manager import URI diff --git a/recap/config.py b/recap/config.py index bd0a171..b2b21af 100644 --- a/recap/config.py +++ b/recap/config.py @@ -1,3 +1,6 @@ +"""Configuration management by extending :class:`yacs.config.CfgNode`. +""" + import os from typing import Dict, Any from pathlib import Path diff --git a/recap/path_manager.py b/recap/path_manager.py index d0dfbbd..4433fa0 100644 --- a/recap/path_manager.py +++ b/recap/path_manager.py @@ -1,3 +1,6 @@ +"""Module for conveniently managing paths through the :class:`URI` class which is fully compatible with :class:`pathlib.Path`. +""" + from pathlib import Path, PurePath, _PosixFlavour from typing import Callable import logging @@ -73,7 +76,7 @@ def __call__(self, uri: "URI") -> Path: class PathManagerBase: """Base class for a path manager. - This class simultaneously acts as a context manager for the currently active path manager of the URI class. + This class simultaneously acts as a context manager for the currently active path manager of the :class:`URI` class. """ def __init__(self): @@ -81,7 +84,7 @@ def __init__(self): self._previous_path_managers = [] def resolve(self, path: os.PathLike) -> Path: - """Resolve a path (which might be a URI) to a local path. + """Resolve a path (which might be a :class:`pathlib.Path`) to a local path. Args: path (os.PathLike): the path @@ -131,7 +134,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class URI(_URIBase): """A class representing a recap URI that is lazily evaluated to a local path when it is used. - It is fully compatible with pathlib.Path. + It is fully compatible with :class:`pathlib.Path`. """ def _init(self, *args, **kwargs): From 25f59a6d88fc3f94a562f1bb710b5a49a26c2025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20W=C3=B6lflein?= Date: Wed, 30 Dec 2020 12:43:36 +0100 Subject: [PATCH 7/7] Bump version --- pyproject.toml | 2 +- recap/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae77630..f53cad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "recap" -version = "0.1.4" +version = "0.1.5" description = "Reproducible configurations for any project" authors = ["Georg Wölflein "] license = "Apache-2.0" diff --git a/recap/__version__.py b/recap/__version__.py index bbab024..1276d02 100644 --- a/recap/__version__.py +++ b/recap/__version__.py @@ -1 +1 @@ -__version__ = "0.1.4" +__version__ = "0.1.5"