From a65ad39e7700dd9c01fe61599a6e5c2f0eff27a0 Mon Sep 17 00:00:00 2001 From: Jonas Hensel <111120040+jhe-iqbis@users.noreply.github.com> Date: Fri, 12 Aug 2022 13:15:39 +0200 Subject: [PATCH 1/6] implemented command line argument for loading custom validators --- yamale/command_line.py | 6 ++++++ yamale/validators/validators.py | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/yamale/command_line.py b/yamale/command_line.py index 66c99aa..2acdfe6 100644 --- a/yamale/command_line.py +++ b/yamale/command_line.py @@ -117,9 +117,15 @@ def main(): help='number of CPUs to use. Default is 4.') parser.add_argument('-p', '--parser', default='pyyaml', help='YAML library to load files. Choices are "ruamel" or "pyyaml" (default).') + parser.add_argument('-i', '--include', default=None, + help='Path of Python library to load for custom validators.') parser.add_argument('--no-strict', action='store_true', help='Disable strict mode, unexpected elements in the data will be accepted.') args = parser.parse_args() + if args.include is not None: + import importlib.machinery + importlib.machinery.SourceFileLoader('yamalevalidators', args.include).load_module() + yamale.validators.validators.update_default_validators() try: _router(args.path, args.schema, args.cpu_num, args.parser, not args.no_strict) except (SyntaxError, NameError, TypeError, ValueError) as e: diff --git a/yamale/validators/validators.py b/yamale/validators/validators.py index e5fc4e4..589b8b8 100644 --- a/yamale/validators/validators.py +++ b/yamale/validators/validators.py @@ -234,7 +234,9 @@ def __init__(self, *args, **kwargs): DefaultValidators = {} -for v in util.get_subclasses(Validator): - # Allow validator nodes to contain either tags or actual name - DefaultValidators[v.tag] = v - DefaultValidators[v.__name__] = v +def update_default_validators(): + for v in util.get_subclasses(Validator): + # Allow validator nodes to contain either tags or actual name + DefaultValidators[v.tag] = v + DefaultValidators[v.__name__] = v +update_default_validators() From fb56134ef18e53431f391b580ad02ec0a4c21a0f Mon Sep 17 00:00:00 2001 From: Jonas Hensel <111120040+jhe-iqbis@users.noreply.github.com> Date: Fri, 12 Aug 2022 13:20:25 +0200 Subject: [PATCH 2/6] Made package executable by adding `__main__.py` used by `python -m yamale`. --- yamale/__main__.py | 16 ++++++++++++++++ yamale/command_line.py | 4 ++++ 2 files changed, 20 insertions(+) create mode 100644 yamale/__main__.py diff --git a/yamale/__main__.py b/yamale/__main__.py new file mode 100644 index 0000000..288ba59 --- /dev/null +++ b/yamale/__main__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + Validate yaml files and check them against their schemas. Designed to be used outside of Vagrant. + + Just install Yamale: + pip install yamale + And run with: + yamale + OR + python -m yamale +""" +from .command_line import main + +main() diff --git a/yamale/command_line.py b/yamale/command_line.py index 2acdfe6..0579fbc 100644 --- a/yamale/command_line.py +++ b/yamale/command_line.py @@ -6,6 +6,10 @@ Just install Yamale: pip install yamale + And run with: + yamale + OR + python -m yamale """ import argparse From cbf46f0b726e39f738398ebced9e6a5ac69aa2ab Mon Sep 17 00:00:00 2001 From: Jonas Hensel <111120040+jhe-iqbis@users.noreply.github.com> Date: Fri, 12 Aug 2022 14:55:11 +0200 Subject: [PATCH 3/6] Added different examples in `examples/yamalevalidators.py` for the new `-i` argument. --- examples/yamalevalidators.py | 82 ++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 examples/yamalevalidators.py diff --git a/examples/yamalevalidators.py b/examples/yamalevalidators.py new file mode 100644 index 0000000..d57a1fe --- /dev/null +++ b/examples/yamalevalidators.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +import math +import re +from yamale.validators.base import Validator +from yamale.validators import constraints as con +from yamale import util + + +class MultipleOf(con.Constraint): + fail = '%s is not a multiple of %s' + + def __init__(self, value_type, kwargs): + self.keywords = {'mul': value_type, 'prec': float} + super(MultipleOf, self).__init__(value_type, kwargs) + + def _is_valid(self, value): + return value % self.mul <= self.prec + + def _fail(self, value): + return self.fail % (value, self.min) + + +class PowerOf(con.Constraint): + fail = '%s is not a power of %s' + + def __init__(self, value_type, kwargs): + self.keywords = {'pow': value_type, 'prec': float} + super(PowerOf, self).__init__(value_type, kwargs) + + def _is_valid(self, value): + return abs(math.log(value, self.pow) % 1 <= self.prec + + def _fail(self, value): + return self.fail % (value, self.min) + + +class ExtendedNumber(Validator): + """Extended number/float validator""" + value_type = float + tag = 'xnum' + constraints = [con.Min, con.Max, MultipleOf, PowerOf] + + def _is_valid(self, value): + return isinstance(value, (int, float)) and not isinstance(value, bool) + +# OR: yamale.validators.validators.Number.constraints.extend([MultipleOf, PowerOf]) + + +class ExtendedInteger(Validator): + """Extended integer validator""" + value_type = int + tag = 'xint' + constraints = [con.Min, con.Max, MultipleOf, PowerOf] + + def _is_valid(self, value): + return isinstance(value, int) and not isinstance(value, bool) + +# OR: yamale.validators.validators.Integer.constraints.extend([MultipleOf, PowerOf]) + + +class NumberRegex(Validator): + """Regular expression validator with number support""" + tag = 'num_regex' + _regex_flags = {'ignore_case': re.I, 'multiline': re.M, 'dotall': re.S} + + def __init__(self, *args, **kwargs): + self.regex_name = kwargs.pop('name', None) + + flags = 0 + for k, v in util.get_iter(self._regex_flags): + flags |= v if kwargs.pop(k, False) else 0 + + self.regexes = [re.compile(arg, flags) + for arg in args if util.isstr(arg)] + super(NumberRegex, self).__init__(*args, **kwargs) + + def _is_valid(self, value): + return any(r.match(str(value)) for r in self.regexes) + + def get_name(self): + return self.regex_name or self.tag + " match" From 577d8ea993289c80ec5c82588a829a6ffec22c03 Mon Sep 17 00:00:00 2001 From: Jonas Hensel <111120040+jhe-iqbis@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:28:02 +0200 Subject: [PATCH 4/6] Improved command line argument for loading custom validators. --- yamale/command_line.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yamale/command_line.py b/yamale/command_line.py index 0579fbc..bb073c3 100644 --- a/yamale/command_line.py +++ b/yamale/command_line.py @@ -102,8 +102,12 @@ def _validate_dir(root, schema_name, cpus, parser, strict): raise ValueError('\n----\n'.join(set(error_messages))) -def _router(root, schema_name, cpus, parser, strict=True): +def _router(root, schema_name, cpus, parser, strict=True, include=None): root = os.path.abspath(root) + if include is not None: + from importlib.machinery import SourceFileLoader + SourceFileLoader(os.path.basename(include), include).load_module() + yamale.validators.validators.update_default_validators() if os.path.isfile(root): _validate_single(root, schema_name, parser, strict) else: @@ -126,12 +130,8 @@ def main(): parser.add_argument('--no-strict', action='store_true', help='Disable strict mode, unexpected elements in the data will be accepted.') args = parser.parse_args() - if args.include is not None: - import importlib.machinery - importlib.machinery.SourceFileLoader('yamalevalidators', args.include).load_module() - yamale.validators.validators.update_default_validators() try: - _router(args.path, args.schema, args.cpu_num, args.parser, not args.no_strict) + _router(args.path, args.schema, args.cpu_num, args.parser, not args.no_strict, args.include) except (SyntaxError, NameError, TypeError, ValueError) as e: print('Validation failed!\n%s' % str(e)) exit(1) From 4056bba459884112f25549cb960547d85722f6d9 Mon Sep 17 00:00:00 2001 From: Jonas Hensel <111120040+jhe-iqbis@users.noreply.github.com> Date: Fri, 12 Aug 2022 16:30:11 +0200 Subject: [PATCH 5/6] Created test for the new `-i` command line argument. --- .../schema_needs_include.yaml | 1 + .../command_line_fixtures/yamalevalidators.py | 18 ++++++++++++++++++ yamale/tests/test_command_line.py | 13 ++++++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 yamale/tests/command_line_fixtures/schema_needs_include.yaml create mode 100644 yamale/tests/command_line_fixtures/yamalevalidators.py diff --git a/yamale/tests/command_line_fixtures/schema_needs_include.yaml b/yamale/tests/command_line_fixtures/schema_needs_include.yaml new file mode 100644 index 0000000..ecff3c1 --- /dev/null +++ b/yamale/tests/command_line_fixtures/schema_needs_include.yaml @@ -0,0 +1 @@ +map: map(oneof('hello', 'bye'), int()) diff --git a/yamale/tests/command_line_fixtures/yamalevalidators.py b/yamale/tests/command_line_fixtures/yamalevalidators.py new file mode 100644 index 0000000..2d0b362 --- /dev/null +++ b/yamale/tests/command_line_fixtures/yamalevalidators.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from yamale.validators.base import Validator + + +class OneOf(Validator): + """One of given values validator""" + tag = 'oneof' + + def __init__(self, *args, **kwargs): + super(OneOf, self).__init__(*args, **kwargs) + self.values = args + + def _is_valid(self, value): + return value in self.values + + def fail(self, value): + return '\'%s\' is not in %s' % (value, self.values) diff --git a/yamale/tests/test_command_line.py b/yamale/tests/test_command_line.py index ed940ae..d7142be 100644 --- a/yamale/tests/test_command_line.py +++ b/yamale/tests/test_command_line.py @@ -33,21 +33,28 @@ def test_good_yaml(parser): command_line._router( 'yamale/tests/command_line_fixtures/yamls/good.yaml', 'schema.yaml', 1, parser) - + @pytest.mark.parametrize('parser', parsers) def test_good_relative_yaml(parser): command_line._router( 'yamale/tests/command_line_fixtures/yamls/good.yaml', '../schema_dir/external.yaml', 1, parser) - + @pytest.mark.parametrize('parser', parsers) def test_external_glob_schema(parser): command_line._router( 'yamale/tests/command_line_fixtures/yamls/good.yaml', os.path.join(dir_path, 'command_line_fixtures/schema_dir/ex*.yaml'), 1, parser) - + + +@pytest.mark.parametrize('parser', parsers) +def test_include(parser): + command_line._router( + 'yamale/tests/command_line_fixtures/yamls/good.yaml', + 'schema_needs_include.yaml', 1, parser, include='yamale/tests/command_line_fixtures/yamalevalidators.py') + def test_empty_schema_file(): with pytest.raises(ValueError, match='is an empty file!'): From 09b74794c7b887b55e23ef647b1b61058b36819d Mon Sep 17 00:00:00 2001 From: Jonas Hensel <111120040+jhe-iqbis@users.noreply.github.com> Date: Fri, 12 Aug 2022 16:49:58 +0200 Subject: [PATCH 6/6] Added error handling for `-i` file not found. --- yamale/command_line.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yamale/command_line.py b/yamale/command_line.py index bb073c3..06f7d86 100644 --- a/yamale/command_line.py +++ b/yamale/command_line.py @@ -105,6 +105,8 @@ def _validate_dir(root, schema_name, cpus, parser, strict): def _router(root, schema_name, cpus, parser, strict=True, include=None): root = os.path.abspath(root) if include is not None: + if not os.path.isfile(include): + raise ValueError("Python include file '{}' not found.".format(include)) from importlib.machinery import SourceFileLoader SourceFileLoader(os.path.basename(include), include).load_module() yamale.validators.validators.update_default_validators() @@ -126,7 +128,7 @@ def main(): parser.add_argument('-p', '--parser', default='pyyaml', help='YAML library to load files. Choices are "ruamel" or "pyyaml" (default).') parser.add_argument('-i', '--include', default=None, - help='Path of Python library to load for custom validators.') + help='File path of Python library to load for custom validators.') parser.add_argument('--no-strict', action='store_true', help='Disable strict mode, unexpected elements in the data will be accepted.') args = parser.parse_args()