diff --git a/.travis.yml b/.travis.yml index 41f2dd7..d9076f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ before_install: - sudo apt-get update -qq - sudo apt-get install -qq sloccount - pip install -r requirements.txt - - pip install pip-tools coviolations_app flake8 coverage coveralls + - pip install pip-tools coviolations_app flake8 coverage coveralls testfixtures script: python setup.py nosetests --with-coverage --cover-package=xylem > test_result notifications: email: false diff --git a/docs/apidoc/xylem.rst b/docs/apidoc/xylem.rst index 06b833a..b64aa1f 100644 --- a/docs/apidoc/xylem.rst +++ b/docs/apidoc/xylem.rst @@ -15,6 +15,30 @@ Subpackages Submodules ---------- +xylem.arguments module +---------------------- + +.. automodule:: xylem.arguments + :members: + :undoc-members: + :show-inheritance: + +xylem.config module +------------------- + +.. automodule:: xylem.config + :members: + :undoc-members: + :show-inheritance: + +xylem.config_utils module +------------------------- + +.. automodule:: xylem.config_utils + :members: + :undoc-members: + :show-inheritance: + xylem.exception module ---------------------- diff --git a/docs/apidoc/xylem.sources.rst b/docs/apidoc/xylem.sources.rst index dcd87ea..761d160 100644 --- a/docs/apidoc/xylem.sources.rst +++ b/docs/apidoc/xylem.sources.rst @@ -20,14 +20,6 @@ xylem.sources.impl module :undoc-members: :show-inheritance: -xylem.sources.rules_dict module -------------------------------- - -.. automodule:: xylem.sources.rules_dict - :members: - :undoc-members: - :show-inheritance: - Module contents --------------- diff --git a/docs/apidoc/xylem.specs.plugins.rst b/docs/apidoc/xylem.specs.plugins.rst new file mode 100644 index 0000000..c700498 --- /dev/null +++ b/docs/apidoc/xylem.specs.plugins.rst @@ -0,0 +1,22 @@ +xylem.specs.plugins package +=========================== + +Submodules +---------- + +xylem.specs.plugins.rules module +-------------------------------- + +.. automodule:: xylem.specs.plugins.rules + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: xylem.specs.plugins + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/apidoc/xylem.specs.rst b/docs/apidoc/xylem.specs.rst index b5043fb..b5775db 100644 --- a/docs/apidoc/xylem.specs.rst +++ b/docs/apidoc/xylem.specs.rst @@ -1,6 +1,13 @@ xylem.specs package =================== +Subpackages +----------- + +.. toctree:: + + xylem.specs.plugins + Submodules ---------- @@ -12,10 +19,10 @@ xylem.specs.impl module :undoc-members: :show-inheritance: -xylem.specs.rules module ------------------------- +xylem.specs.rules_dict module +----------------------------- -.. automodule:: xylem.specs.rules +.. automodule:: xylem.specs.rules_dict :members: :undoc-members: :show-inheritance: diff --git a/docs/xylem_api.rst b/docs/xylem_api.rst index 66c326c..3c2790f 100644 --- a/docs/xylem_api.rst +++ b/docs/xylem_api.rst @@ -23,6 +23,7 @@ Database .. automodule:: xylem.update :members: :undoc-members: + :noindex: Indices and tables ------------------ diff --git a/setup.py b/setup.py index 5d277d2..dcf2e6c 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # http://www.jeffknupp.com/blog/2013/08/16/open-sourcing-a-python-project-the-right-way/ # see: http://reinout.vanrees.org/weblog/2009/12/17/managing-dependencies.html -tests_require = ['nose', 'flake8', 'mock', 'coverage'] +tests_require = ['nose', 'flake8', 'mock', 'coverage', 'testfixtures'] setup( name='xylem', diff --git a/test/unit_tests/config/__init__.py b/test/unit_tests/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/config/test_config.py b/test/unit_tests/config/test_config.py new file mode 100644 index 0000000..c1bec2d --- /dev/null +++ b/test/unit_tests/config/test_config.py @@ -0,0 +1,19 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for config module.""" + +from __future__ import unicode_literals + +# TODO diff --git a/test/unit_tests/config_utils/__init__.py b/test/unit_tests/config_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_tests/config_utils/test_config_api.py b/test/unit_tests/config_utils/test_config_api.py new file mode 100644 index 0000000..fb5b690 --- /dev/null +++ b/test/unit_tests/config_utils/test_config_api.py @@ -0,0 +1,19 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test high level API for config_utils.""" + +from __future__ import unicode_literals + +# TODO diff --git a/test/unit_tests/config_utils/test_config_description.py b/test/unit_tests/config_utils/test_config_description.py new file mode 100644 index 0000000..347f731 --- /dev/null +++ b/test/unit_tests/config_utils/test_config_description.py @@ -0,0 +1,19 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test creating and using ConfigDescription.""" + +from __future__ import unicode_literals + +# TODO diff --git a/test/unit_tests/config_utils/test_config_dict.py b/test/unit_tests/config_utils/test_config_dict.py new file mode 100644 index 0000000..7263a04 --- /dev/null +++ b/test/unit_tests/config_utils/test_config_dict.py @@ -0,0 +1,19 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test functions manipulating config dicts.""" + +from __future__ import unicode_literals + +# TODO diff --git a/test/unit_tests/config_utils/test_config_types.py b/test/unit_tests/config_utils/test_config_types.py new file mode 100644 index 0000000..2881c1e --- /dev/null +++ b/test/unit_tests/config_utils/test_config_types.py @@ -0,0 +1,433 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the config types.""" + +from __future__ import unicode_literals +from __future__ import print_function + +import unittest +import mock + +from testfixtures import compare + +from xylem.config_utils import Any +from xylem.config_utils import Boolean +from xylem.config_utils import String +from xylem.config_utils import Path +from xylem.config_utils import List +from xylem.config_utils import Dict +from xylem.config_utils import MergingDict +from xylem.config_utils import ConfigValueError +from xylem.config_utils import maybe_instantiate +from xylem.config_utils import expand_input_path +from xylem.config_utils import UNSET_YAML +from xylem.config_utils import UNSET_COMMAND_LINE + +from xylem.text_utils import text_type + + +def same_pair(v): + return (v, v) + + +class ConfigTypesTestCase(unittest.TestCase): + + def _test_type(self, type_, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify): + + t = maybe_instantiate(type_) + + # test unset value + unset = t.unset_value() + compare(unset, t.from_yaml(UNSET_YAML)) + compare(unset, t.from_command_line(UNSET_COMMAND_LINE)) + t.verify(unset) + self.assertTrue(t.is_unset(unset)) + self.assertFalse(t.is_unset(not unset)) + + # test command line multiple + self.assertIsInstance(t.command_line_multiple(), bool) + + # test command_line_parsing_help + self.assertIsInstance(t.command_line_parsing_help(), (type(None), text_type)) + + # test merging + compare(unset, t.merge(unset, unset)) + def verify_merge(input): + compare(input, t.merge(input, unset)) + compare(input, t.merge(unset, input)) + + # test parsing from command line + for input, result in command_line: + try: + with mock.patch.object(t, 'verify', side_effect=t.verify) as verify_mock: + compare(result, t.from_command_line(input)) + if result != unset: + self.assertGreaterEqual(verify_mock.call_count, 1) + verify_merge(result) + except ConfigValueError: + print("from_command_line raised unexpected: `{}`".format(input)) + raise + for input in fail_command_line: + with self.assertRaises(ConfigValueError): + result = t.from_command_line(input) + print("from_command_line did not raise: `{}`; result: `{}`".format(input, result)) + + # test parsing from yaml structure + for input, result in yaml: + try: + with mock.patch.object(t, 'verify', side_effect=t.verify) as verify_mock: + compare(result, t.from_yaml(input)) + if result != unset: + self.assertGreaterEqual(verify_mock.call_count, 1) + verify_merge(result) + except ConfigValueError: + print("from_yaml raised unexpected: `{}`".format(input)) + raise + for input in fail_yaml: + with self.assertRaises(ConfigValueError): + result = t.from_yaml(input) + print("from_yaml did not raise: `{}`; result: `{}`".format(input, result)) + + # test verification + for input in verify: + try: + t.verify(input) + verify_merge(input) + except ConfigValueError: + print("verify raised unexpected: `{}`".format(input)) + raise + for input in fail_verify: + with self.assertRaises(ConfigValueError): + t.verify(input) + print("verify did not raise: `{}`".format(input)) + + def test_any(self): + command_line = [ + same_pair("foo bar"), + ("{foo: bar, quux: quax}", dict(foo="bar",quux="quax")), + ("[1,2,3]", [1, 2, 3]), + same_pair("1,2,3"), + ("", None), + same_pair(None), + ] + fail_command_line = [ + "foo: bar, quux: quax", + "[ , [", + "[}", + "*&^%%&*(", + ] + yaml = [ + same_pair("foo"), + same_pair([1, 2]), + same_pair([[]]), + same_pair({}), + same_pair(None), + ] + fail_yaml = [ + ] + verify = [ + False, + True, + None, + [1,2], + [1,[2, "foo"]], + {}, + {"foo": "bar", 2: {3: 42}}, + 3.141, + "foobar", + ] + fail_verify = [ + ] + self._test_type(Any, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + + def test_bool(self): + command_line = [ + ("true", True), + ("TRUE", True), + ("yes", True), + ("YES", True), + ("false", False), + ("FALSE", False), + ("no", False), + ("NO", False), + ("", None), + same_pair(None), + ] + fail_command_line = [ + "yEs", + "nO", + "truE", + "faLse", + "foo bar", + "{foo: bar, quux: quax}", + "[1,2,3]", + "1,2,3", + "foo: bar", + "[ , [", + "[}", + "*&^%%&*(", + ] + yaml = [ + same_pair(None), + same_pair(False), + same_pair(True), + ] + fail_yaml = [ + [1,2], + [1,[2, "foo"]], + {}, + {"foo": "bar", 2: {3: 42}}, + 3.141, + "foobar", + ] + verify = [ + False, + True, + None, + ] + fail_verify = [ + [1,2], + [1,[2, "foo"]], + {}, + {"foo": "bar", 2: {3: 42}}, + 3.141, + "foobar", + ] + self._test_type(Boolean, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + + def test_string(self): + command_line = [ + same_pair("foo"), + same_pair("[foo,bar]"), + same_pair("yes"), + same_pair("FALSE"), + same_pair("{foo: bar, quux: quax}"), + same_pair("[1,2,3]"), + same_pair("1,2,3"), + same_pair("foo: bar"), + same_pair("[ , ["), + same_pair("[}"), + same_pair("*&^%%&*("), + same_pair("\n"), + same_pair(" "), + same_pair(""), + same_pair(None), + ] + fail_command_line = [ + ] + yaml = command_line + fail_yaml = [ + ["foo", "bar"], + True, + [1,2], + [1,[2, "foo"]], + {}, + {"foo": "bar", 2: {3: 42}}, + 3.141, + ] + verify = [x[0] for x in command_line] + fail_verify = fail_yaml + self._test_type(String, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + + def test_path(self): + def expand_pair(path): + return (path, expand_input_path(path)) + command_line = [ + expand_pair("~"), + expand_pair("~/foo/bar"), + expand_pair("/foo"), + expand_pair("/foo/bar/"), + expand_pair("/"), + expand_pair("."), + expand_pair(""), + same_pair(None), + ] + fail_command_line = [ + ] + yaml = command_line + fail_yaml = [ + ["foo", "bar"], + True, + [1,2], + [1,[2, "foo"]], + {}, + {"foo": "bar", 2: {3: 42}}, + 3.141, + ] + verify = [x[1] for x in command_line] + [x[1] for x in yaml] + fail_verify = fail_yaml + self._test_type(Path, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + + def test_list(self): + command_line = [ + ("[1,2,3]", [1, 2, 3]), + (["[1,2,3]", "4"], [1, 2, 3, 4]), + ("{}, [], foo", [{}, [], "foo"]), + ("1,2,3", [1, 2, 3]), + ("foo: bar, quux: quax" ,[{"foo": "bar"}, {"quux": "quax"}]), + ("{foo: bar, quux: quax}" ,[{"foo": "bar", "quux": "quax"}]), + ("foo bar", ["foo bar"]), + ("yes", [True]), + ("", []), + same_pair(None), + ] + fail_command_line = [ + "[ , [", + "[}", + "*&^%%&*(", + ] + yaml = [ + same_pair([1, 2]), + same_pair([1, "foo"]), + same_pair([[]]), + same_pair(None), + ] + fail_yaml = [ + "foo", + {}, + 1, + True, + ] + verify = [x[1] for x in command_line] + [x[1] for x in yaml] + fail_verify = fail_yaml + t = List() + self._test_type(t, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + compare([1,2], t.merge([1,2], [3,4])) + compare([3,4], t.merge(None, [3,4])) + compare([], t.merge([], [3,4])) + + def test_list_of_strings(self): + command_line = [ + ("[foo bar, baz]", ["foo bar", "baz"]), + (["", "foo", "bar,baz", "[]"], ["foo", "bar", "baz"]), + ("foo bar", ["foo bar"]), + ("", []), + same_pair(None), + ] + fail_command_line = [ + "[1,2,3]", + ["[1,2,3]", "4"], + "{}, [], foo", + "1,2,3", + "foo: bar, quux: quax", + "{foo: bar, quux: quax}", + "yes", + "[ , [", + "[}", + "*&^%%&*(", + ] + yaml = [ + same_pair(["foo", "bar"]), + same_pair([]), + same_pair(None), + ] + fail_yaml = [ + [1, "foo"], +# FIXME: we want the following to fail, but currently with the way unset works, it does not +# [None], + "foo", + {}, + 1, + True, + ] + verify = [x[1] for x in command_line] + [x[1] for x in yaml] + fail_verify = fail_yaml + self._test_type(List(String()), command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + + def test_dict(self): + command_line = [ + ("foo: bar, quux: quax", {"foo": "bar", "quux": "quax"}), + ("{foo: bar, quux: quax}", {"foo": "bar", "quux": "quax"}), + ("foo: 2", {"foo": 2}), + ("foo:", {"foo": None}), + ("foo bar:", {"foo bar": None}), + (["foo:", "", "bar: 2"], {"foo": None, "bar": 2}), + ("", {}), + same_pair(None), + ] + fail_command_line = [ + ["foo: 1", "foo: 2"], + "{1:2}", + "1,2,3", + "[1,2,3]", + "[ , [", + "[}", + "*&^%%&*(", + ] + yaml = [ + same_pair({"foo": "bar"}), + same_pair({"foo": "bar", "baz": 2}), + same_pair({}), + same_pair(None), + ] + fail_yaml = [ + False, + True, + [1,2], + [1,[2, "foo"]], + {"foo": "bar", 2: {3: 42}}, + 3.141, + "foobar", + ] + verify = [x[1] for x in command_line] + [x[1] for x in yaml] + fail_verify = fail_yaml + t = Dict() + self._test_type(t, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + compare({"1": 2}, t.merge({"1": 2}, {"3": 4})) + compare({"3": 4}, t.merge(None, {"3": 4})) + compare({}, t.merge({}, {"3": 4})) + + def test_mergingdict_of_list_of_string(self): + command_line = [ + ("foo: [bar, baz]", {"foo": ["bar", "baz"]}), + ("foo: []", {"foo": []}), + ("", {}), + (None, {}), + ] + fail_command_line = [ + "{foo: bar, quux: quax}", + ["foo: []", "foo: []"], +# FIXME: we want the following to fail, but currently with the way unset works, it does not +# "foo:", + "{1:2}", + "1,2,3", + "[1,2,3]", + "[ , [", + "[}", + "*&^%%&*(", + ] + yaml = [ + same_pair({"foo": ["bar"]}), + same_pair({"foo": ["bar"], "baz": []}), + same_pair({}), + (None, {}), + ] + fail_yaml = [ + {"foo": ["bar"], "baz": [2]}, + False, + True, + [1,2], + [1,[2, "foo"]], + {"foo": "bar", 2: {3: 42}}, + 3.141, + "foobar", + ] + verify = [x[1] for x in command_line] + [x[1] for x in yaml] + fail_verify = fail_yaml + t = MergingDict(List(String)) + self._test_type(t, command_line, fail_command_line, yaml, fail_yaml, verify, fail_verify) + compare({"1": 2, "3": 4}, t.merge({"1": 2}, {"3": 4})) + compare({"1": 2}, t.merge({"1": 2}, {})) + compare({"3": 4}, t.merge({}, {"3": 4})) diff --git a/test/unit_tests/config_utils/test_config_utils.py b/test/unit_tests/config_utils/test_config_utils.py new file mode 100644 index 0000000..fea974c --- /dev/null +++ b/test/unit_tests/config_utils/test_config_utils.py @@ -0,0 +1,19 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for utility functions in config_utils.""" + +from __future__ import unicode_literals + +# TODO diff --git a/test/unit_tests/specs/test_merge_rules.py b/test/unit_tests/specs/test_merge_rules.py index a59a5e2..eb4717a 100644 --- a/test/unit_tests/specs/test_merge_rules.py +++ b/test/unit_tests/specs/test_merge_rules.py @@ -25,7 +25,7 @@ from xylem.specs.rules_dict import merge_rules from xylem.specs.rules_dict import verify_rules_dict -from xylem.util import load_yaml +from xylem.yaml_utils import load_yaml _default_installers = { diff --git a/test/unit_tests/specs/test_rules.py b/test/unit_tests/specs/test_rules.py index 7929d1e..8a56207 100644 --- a/test/unit_tests/specs/test_rules.py +++ b/test/unit_tests/specs/test_rules.py @@ -21,7 +21,7 @@ from xylem.specs.plugins.rules import expand_rules from xylem.specs.plugins.rules import compact_rules -from xylem.util import load_yaml +from xylem.yaml_utils import load_yaml _default_installers = dict( diff --git a/tox.ini b/tox.ini index c78660a..33d38f2 100644 --- a/tox.ini +++ b/tox.ini @@ -13,4 +13,5 @@ deps = nose flake8 mock + testfixtures #changedir = {envtmpdir} diff --git a/xylem/__init__.py b/xylem/__init__.py index bc3e770..9c750dd 100644 --- a/xylem/__init__.py +++ b/xylem/__init__.py @@ -22,5 +22,3 @@ __version__ = 'unset' except (ImportError, OSError): __version__ = 'unset' - -DEFAULT_PREFIX = "/" diff --git a/xylem/arguments.py b/xylem/arguments.py new file mode 100644 index 0000000..b8f3879 --- /dev/null +++ b/xylem/arguments.py @@ -0,0 +1,107 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Handling of command line arguments shared by multiple xylem commands. + +This includes arguments related to the configuration (see +:mod:`xylem.config`). +""" + +from __future__ import unicode_literals + +import argparse +import pydoc +import os +from six import StringIO + +from .config import add_global_config_arguments +from .config import handle_global_config_arguments + +from .util import enable_pdb +from .log_utils import enable_verbose +from .log_utils import enable_debug +from .terminal_color import disable_ANSI_colors + + +class PagerHelpAction(argparse._HelpAction): + + """Custom help action that uses pydoc.pager. + + On ttys this presents a scrollable output akin to ``less``. + """ + + def __init__(self, *args, **kwargs): + super(PagerHelpAction, self).__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + output = StringIO() + parser.print_help(output) + text = output.getvalue() + output.close() + pydoc.pager(text) + parser.exit() + + +def add_global_arguments(parser): + """Add a 'global' argparse group and add comon arguments. + + See :func:`handle_global_arguments` + + :param parser: argparse parser to which the group is added + """ + from xylem import __version__ + group = parser.add_argument_group( + 'global arguments', description="""By default xylem operates + with system-wide configuration of sources and cache. The path + under which those are found can be set with the XYLEM_PREFIX + environment variable (or `--prefix` argument). To instead use + sources and directories in the user's home directory, make use + of the `--user- sources` argument. An alternative mode of + configuration is by setting the XYLEM_DIR environment variable + (or `--xylem-dir` argument). With that, only config files in + that directory are used and sources/cache folder is local to the + XYLEM_DIR. This can be used by third party tools to utilize + xylem with config/sources/cache in a temporary directory.""") + add = group.add_argument + add("-h", "--help", action=PagerHelpAction, + help="show this help message and exit") + add('--version', action='version', version=__version__, + help="print xylem version and exit") + add('-v', '--verbose', action='store_true', + help="verbose console output") + add('-d', '--debug', action='store_true', + help="""enable debug messages (overwrites the XYLEM_DEBUG + environment variable)""") + add('--pdb', action='store_true', + help=argparse.SUPPRESS,) + add('--no-color', action='store_true', + help=argparse.SUPPRESS) + add_global_config_arguments(group) + return parser + + +def handle_global_arguments(args): + """Handle common arguments + + See :func:`add_global_arguments` + + :param argparse.Namespace args: parsed arguments + """ + enable_debug(args.debug or 'XYLEM_DEBUG' in os.environ) + enable_pdb(args.pdb or 'XYLEM_PDB' in os.environ) + if args.verbose: + enable_verbose() + if args.no_color: + disable_ANSI_colors() + handle_global_config_arguments(args) diff --git a/xylem/commands/_compact_rules_file.py b/xylem/commands/_compact_rules_file.py index 3c840a1..5eeabcb 100644 --- a/xylem/commands/_compact_rules_file.py +++ b/xylem/commands/_compact_rules_file.py @@ -14,14 +14,13 @@ from __future__ import unicode_literals -import argparse import sys import os from ..specs.rules_dict import verify_rules_dict -from ..specs.rules import expand_rules -from ..specs.rules import compact_rules +from ..specs.plugins.rules import expand_rules +from ..specs.plugins.rules import compact_rules from ..os_support import OSSupport from ..log_utils import info @@ -29,13 +28,16 @@ from ..text_utils import to_str from ..text_utils import to_bytes -from ..util import load_yaml -from ..util import dump_yaml -from ..util import add_global_arguments -from ..util import handle_global_arguments +from ..yaml_utils import load_yaml +from ..yaml_utils import dump_yaml + +from .main import command_handle_args + DESCRIPTION = """\ -Compact a rules file. +Parse a rules file and print it in canonical compact notation. + +This is a utility command intended for rules file maintenance. """ @@ -47,15 +49,12 @@ def prepare_arguments(parser): "compacted file.") +def prepare_config(description): + pass + + def main(args=None): - if args is None: - parser = argparse.ArgumentParser( - description=DESCRIPTION - ) - prepare_arguments(parser) - add_global_arguments(parser) - args = parser.parse_args() - handle_global_arguments(args) + args = command_handle_args(args, definition) try: # prepare arguments filepath = os.path.abspath(os.path.expanduser(args.rules_file)) @@ -87,5 +86,6 @@ def main(args=None): title='_compact_rules_file', description=DESCRIPTION, main=main, - prepare_arguments=prepare_arguments + prepare_arguments=prepare_arguments, + prepare_config=prepare_config ) diff --git a/xylem/commands/lookup.py b/xylem/commands/lookup.py index ac7ff50..4e7d15e 100644 --- a/xylem/commands/lookup.py +++ b/xylem/commands/lookup.py @@ -14,20 +14,18 @@ from __future__ import unicode_literals -import argparse import sys from ..log_utils import info - +from ..config import get_config from ..lookup import lookup - from ..installers import InstallerContext - -from ..util import add_global_arguments -from ..util import handle_global_arguments -from ..util import dump_yaml +from ..yaml_utils import dump_yaml from ..terminal_color import ansi +from .main import command_handle_args + + DESCRIPTION = """\ Lookup all rules for a xylem key. """ @@ -35,44 +33,27 @@ def prepare_arguments(parser): parser.add_argument('xylem_key', nargs="+") - parser.add_argument( - '--os', - help="Override detected operating system with os:version pair.") -def parse_os_tuple(os_arg): - if ':' in os_arg: - os_tuple = tuple(os_arg.split(':', 1)) - else: - os_tuple = os_arg, '' - return os_tuple +def prepare_config(description): + pass def main(args=None): - if args is None: - parser = argparse.ArgumentParser( - description=DESCRIPTION - ) - prepare_arguments(parser) - add_global_arguments(parser) - args = parser.parse_args() - handle_global_arguments(args) + args = command_handle_args(args, definition) + config = get_config() try: - os_tuple = None - if args.os: - os_tuple = parse_os_tuple(args.os) - - ic = InstallerContext(os_override=os_tuple) - + # TODO: handle multiple keys in one go + ic = InstallerContext(config) for key in args.xylem_key: - result = lookup( - key, prefix=args.prefix, os_override=ic, compact=True) - info("Rules for '{0}' on '{1}':\n{2}". + result = lookup(key, compact=True, config=config, + installer_context=ic) + info("Rules for '{}' on '{}':\n{}". format(ansi('cyanf') + key + ansi('reset'), ansi('cyanf') + ic.get_os_string() + ansi('reset'), ansi('yellowf') + dump_yaml(result)[:-1])) except (KeyboardInterrupt, EOFError): - # Note: @William: why EOFError here? + # Note: @William why EOFError here? info('') sys.exit(1) @@ -82,5 +63,6 @@ def main(args=None): title='lookup', description=DESCRIPTION, main=main, - prepare_arguments=prepare_arguments + prepare_arguments=prepare_arguments, + prepare_config=prepare_config ) diff --git a/xylem/commands/main.py b/xylem/commands/main.py index 5a2bf84..91f40a0 100644 --- a/xylem/commands/main.py +++ b/xylem/commands/main.py @@ -18,11 +18,20 @@ import sys import pkg_resources -from xylem.log_utils import error -from xylem.log_utils import info +from ..log_utils import error +from ..log_utils import info +from ..log_utils import ansi + +from ..arguments import add_global_arguments +from ..arguments import handle_global_arguments + +from ..config import get_config_description +from ..config import add_config_arguments +from ..config import handle_config_arguments +from ..config_utils import ConfigHelpFormatter + +from ..text_utils import type_name -from xylem.util import add_global_arguments -from xylem.util import handle_global_arguments XYLEM_CMDS_GROUP = 'xylem.commands' @@ -34,45 +43,65 @@ def list_commands(): return commands -def load_command_description(command_name): +def load_command_definition(command_name): for entry_point in pkg_resources.iter_entry_points(group=XYLEM_CMDS_GROUP): if entry_point.name == command_name: - desc = entry_point.load() - if not isinstance(desc, dict): + defi = entry_point.load() + if not isinstance(defi, dict): error("Invalid entry point: '{0}', expected dict got '{1}'" - .format(entry_point, type(desc))) + .format(entry_point, type_name(defi))) return None - return desc + return defi + + +def create_command_parser(command_defi, constructor, parser_title=False): + args = [str(command_defi['title'])] if parser_title else [] + parser = constructor(*args, + description=command_defi['description'], + add_help=False, + formatter_class=ConfigHelpFormatter) + parser = command_defi['prepare_arguments'](parser) or parser + parser.set_defaults(func=command_defi['main']) + command_defi['prepare_config'](get_config_description()) + add_config_arguments(parser) + add_global_arguments(parser) + + +def command_handle_args(args, definition): + """Helper for commands.""" + if args is None: + parser = create_command_parser(definition, argparse.ArgumentParser) + args = parser.parse_args() + handle_global_arguments(args) + handle_config_arguments(args) + return args def create_subparsers(parser, cmds): - descs = [] + defis = [] for cmd in list(cmds): - desc = load_command_description(cmd) - if desc is None: + defi = load_command_definition(cmd) + if defi is None: info("Skipping invalid command '{0}'".format(cmd)) del cmds[cmds.index(cmd)] continue - descs.append(desc) - if not descs or not cmds: - add_global_arguments(parser) + defis.append(defi) + if not defis or not cmds: return - metavar = '[' + ' | '.join(cmds) + ']' + public_cmds = [c for c in cmds if not c.startswith("_")] + metavar = '[' + ' | '.join(public_cmds) + ']' subparser = parser.add_subparsers( title='commands', metavar=metavar, - description='Call `xylem -h` for help on a specific ' - 'command.', + description="""Call `xylem -h` for help on a specific + command.""", dest='cmd' ) - for desc in descs: - cmd_parser = subparser.add_parser(desc['title'], - description=desc['description']) - cmd_parser = desc['prepare_arguments'](cmd_parser) or cmd_parser - cmd_parser.set_defaults(func=desc['main']) - add_global_arguments(cmd_parser) + for defi in defis: + create_command_parser(defi, subparser.add_parser, parser_title=True) +# FIXME: Merge with help message somehow and decide when to print def print_usage(): info("xylem is a package manager abstraction tool.") info("") @@ -90,23 +119,28 @@ def print_usage(): def main(sysargs=None): parser = argparse.ArgumentParser( description="xylem is a package manager abstraction tool.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter + add_help=False, + formatter_class=ConfigHelpFormatter ) + add_config_arguments(parser) add_global_arguments(parser) cmds = list_commands() - create_subparsers(parser, cmds) args = parser.parse_args(sysargs) handle_global_arguments(args) + handle_config_arguments(args) + # FIXME: the following logic and error handling try: args.func - except AttributeError: - print_usage() - return + result = args.func(args) +# except AttributeError: +# print_usage() +# return except (KeyboardInterrupt, EOFError): info('') sys.exit(1) - sys.exit(args.func(args) or 0) + sys.stdout.write(ansi('reset')) + sys.exit(result or 0) diff --git a/xylem/commands/resolve.py b/xylem/commands/resolve.py index fc703c0..666bb5e 100644 --- a/xylem/commands/resolve.py +++ b/xylem/commands/resolve.py @@ -14,21 +14,23 @@ from __future__ import unicode_literals -import argparse import sys +from six.moves import map + from ..log_utils import info from ..resolve import resolve from ..installers import InstallerContext -from ..util import add_global_arguments -from ..util import handle_global_arguments - from ..text_utils import to_str from ..terminal_color import ansi -from six.moves import map + +from ..config import get_config + +from .main import command_handle_args + DESCRIPTION = """\ Lookup a xylem key and resolve to unique, parsed rule based on @@ -39,8 +41,6 @@ def prepare_arguments(parser): add = parser.add_argument add('xylem_key', nargs="*") - add('--os', - help="Override detected operating system with os:version pair.") add('--show-trumped', action="store_true", help="Show all possible resolutions for key, also for " "trumped installers.") @@ -52,48 +52,28 @@ def prepare_arguments(parser): help="Show installer even if it is the default installer.") # TODO: add 'show-depends' option +# TODO: abstract a way to + -# TODO: move this to os_support package -def parse_os_tuple(os_arg): - if ':' in os_arg: - os_tuple = tuple(os_arg.split(':', 1)) - else: - os_tuple = os_arg, '' - return os_tuple +def prepare_config(description): + pass def main(args=None): - if args is None: - parser = argparse.ArgumentParser( - description=DESCRIPTION - ) - prepare_arguments(parser) - add_global_arguments(parser) - args = parser.parse_args() - handle_global_arguments(args) + args = command_handle_args(args, definition) + config = get_config() try: - os_tuple = None - if args.os: - os_tuple = parse_os_tuple(args.os) - - ic = InstallerContext(os_override=os_tuple) + ic = InstallerContext(config) default_installer_name = ic.get_default_installer_name() - - results = resolve(args.xylem_key, - prefix=args.prefix, - os_override=ic, - all_keys=args.all) - + results = resolve(args.xylem_key, all_keys=args.all, config=config, + installer_context=ic) for key, result in results: - if not args.show_trumped: result = [result[0]] # TODO: this should be done inside `resolve` - # TODO: Error if single resolution is requested, but # highest priority occurs multuple times (macports vs # homebrew) - for priority, installer_name, resolutions in result: if installer_name != default_installer_name or \ args.show_default_installer or \ @@ -105,9 +85,7 @@ def main(args=None): installer_string = "{0} : ".format(installer_name) else: installer_string = "" - resolution_string = ', '.join(map(to_str, resolutions)) - info("{0} --> {1}{2}". format(ansi("cyanf") + key + ansi("reset"), ansi("bluef") + installer_string, @@ -122,5 +100,6 @@ def main(args=None): title='resolve', description=DESCRIPTION, main=main, - prepare_arguments=prepare_arguments + prepare_arguments=prepare_arguments, + prepare_config=prepare_config ) diff --git a/xylem/commands/update.py b/xylem/commands/update.py index 266b367..427e9f9 100644 --- a/xylem/commands/update.py +++ b/xylem/commands/update.py @@ -14,42 +14,36 @@ from __future__ import unicode_literals -import argparse import sys -from xylem.log_utils import info +from ..update import update +from ..log_utils import info +from .main import command_handle_args -from xylem.update import update - -from xylem.util import add_global_arguments -from xylem.util import handle_global_arguments DESCRIPTION = """\ Updates the xylem cache according to the source config files. -If no source config files are found under the current XYLEM_PREFIX, at -/etc/xylem/sources.list.d, then the default, internal source -configs are used. The cache is always stored under the XYLEM_PREFIX in -the /var/caches/xylem directory. +Location for sources files and cache are determined from the +configuration. """ def prepare_arguments(parser): + # TODO: do we need this `--dry-run` argument? What about consistency + # with `install --simulate` parser.add_argument('-n', '--dry-run', action='store_true', default=False, - help="Shows affect of an update only") + help="shows affect of an update only") + + +def prepare_config(description): + pass def main(args=None): - if args is None: - parser = argparse.ArgumentParser( - description=DESCRIPTION - ) - prepare_arguments(parser) - add_global_arguments(parser) - args = parser.parse_args() - handle_global_arguments(args) + args = command_handle_args(args, definition) try: - update(prefix=args.prefix, dry_run=args.dry_run) + update(dry_run=args.dry_run) except (KeyboardInterrupt, EOFError): info('') sys.exit(1) @@ -60,5 +54,6 @@ def main(args=None): title='update', description=DESCRIPTION, main=main, - prepare_arguments=prepare_arguments + prepare_arguments=prepare_arguments, + prepare_config=prepare_config ) diff --git a/xylem/config.py b/xylem/config.py new file mode 100644 index 0000000..7f0a2e1 --- /dev/null +++ b/xylem/config.py @@ -0,0 +1,273 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Description of xylem configuration files using `.config_utils`. + +The rest of the code base is supposed to interact with with +configuration only via this module, not `.config_utils` directly. + +A global representation of configuration in form of a +`.config_utils.ConfigDescription` object can be accessed via +:func:`get_config_description`. The resulting configuration dictionary +can be accessed via :func:`get_config` and :func:`set_config`. +""" + +from __future__ import unicode_literals + +import os + +from six.moves import map + +from .config_utils import DEFAULT_PREFIX +from .config_utils import ConfigDescription +from .config_utils import String +from .config_utils import List +from .config_utils import Boolean +from .config_utils import Dict +from .config_utils import MergingDict +from .config_utils import Any +from .config_utils import Path +from .config_utils import load_config +from .config_utils import config_from_defaults +from .config_utils import system_config_dir +from .config_utils import system_cache_dir +from .config_utils import user_cache_dir +from .config_utils import user_config_dir +from .config_utils import add_global_config_arguments as \ + _add_global_config_arguments +from .config_utils import handle_global_config_arguments as \ + _handle_global_config_arguments +from .config_utils import handle_global_config_arguments_post as \ + _handle_global_config_arguments_post + +from .text_utils import text_type + + +def sources_dir(parent): + """Utility returning the sources.d directory given a parent folder.""" + return os.path.join(parent, SOURCES_DIR) + + +XYLEM_TOOL_NAME = "xylem" +"""Name of the xylem tool as passed to various `.config_utils` APIs""" + +SOURCES_DIR = "sources.d" +"""Sources folder inside the config directory.""" + +DEFAULT_CACHE_DIR = system_cache_dir(DEFAULT_PREFIX, XYLEM_TOOL_NAME) +"""Default system-wide cache folder.""" + +DEFAULT_SOURCES_DIR = sources_dir(system_config_dir(DEFAULT_PREFIX, + XYLEM_TOOL_NAME)) +"""Default system-wide sources folder.""" + + +def build_config_description(): + """Build the global configuration description for xylem. + + :rtype: `.config_utils.ConfigDescription` + """ + description = ConfigDescription("config.yaml") + add = description.add + add("os_override", type=String, + command_line_argument="os", + command_line_metavar="name:version", + help="""override os detection; if no ':' is present, the entire + string is interpreted as the os name and the version is + detected""") + add("os_options/features", type=List(String), + command_line_argument="os-features", + command_line_metavar='"feature1,feature2,..."', + help="""OS features to be used. The list of possible values and + the default choice is defined by the os plugin for the + selected OS.""") + add("os_options/installers", type=List(String), + command_line_argument="core-installers", + command_line_metavar='"inst1,inst2,..."', + help="""core installers to be used. The default list is defined + by the OS plugin for the selected OS. Additional + installers as defined by installer plugins may be used + on top of the core installers unless the + `use_additional_installers` option is set to False.""") + add("use_additional_installers", type=Boolean, default=True, + command_line=True, + help="""if `True`, use additional installers as defined by + installer plugins on top of the core installers""") + add("install_from", type=MergingDict(List(String)), + command_line_metavar='"inst1:[key1,key2,...] ..."', + help="""mapping installer names to list of xylem keys; + overwrites the installer priority such that the given keys are + only ever installed with the specified installer""") + add("installer_options", type=MergingDict(Dict(Any)), + command_line_metavar='"inst1:{opt1:val1,...}, ..."', + help="""options passed to installer plugins; valid options are + specific to each installer plugin""") + add("user_sources", type=Boolean, default=False, + command_line=True, + help="""if `True`, look for sources and cache in user directory + instead of system-wide locations""") + add("cache_dir", type=Path, + command_line=True, + help="""override the cache location""") + add("sources_dir", type=Path, + command_line=True, + help="""override the sources directory""") + return description + + +_config_description = build_config_description() +_config = config_from_defaults(_config_description) + + +def get_config_description(): + """Access the global configuration description for xylem. + + :rtype: `.config_utils.ConfigDescription` + """ + global _config_description + return _config_description + + +def get_config(): + """Access the global config dict as built after command line parsing. + + See :func:`set_config`. + + :rtype: dict + """ + global _config + return _config + + +def set_config(config): + """Set the global config dict. + + See :func:`get_config`. + + :param dict config: the config dict to set + """ + global _config + _config = config + + +def add_global_config_arguments(parser): + """Add global config-related command line options. + + See :func:`.config_utils.add_global_config_arguments`. + + :param parser: :mod:`argparse` argument parser; does not create a + subgroup, i.e. best pass a 'global' parser group + """ + _add_global_config_arguments(parser, XYLEM_TOOL_NAME) + + +def handle_global_config_arguments(args): + """Handle global config-related command line options before loading config. + + See :func:`.config_utils.handle_global_config_arguments`. + + :param argparse.Namespace args: arguments from command line + """ + _handle_global_config_arguments(args, XYLEM_TOOL_NAME) + + +def add_config_arguments(parser): + """Add arguments for config description in :func:`get_config_description` + + :param parser: :mod:`argparse` argument parser; an appropriate + subgroup is created + """ + description = get_config_description() + description.add_arguments(parser) + + +def handle_config_arguments(args): + """Handle xylem specific config-related command line options. + + Loads the configuration from config files and merges it + appropriately with the command line arguments. Sets the global + config with :func:`set_config`. + + :param argparse.Namespace args: arguments from command line + """ + description = get_config_description() + config = load_config(args, description, XYLEM_TOOL_NAME) + process_config(config, args) + _handle_global_config_arguments_post(args, config, XYLEM_TOOL_NAME) + set_config(config) + + +def parse_os_override(os_arg): + """Utility to parse os_override arguments. + + :param str os_arg: string of form + ``"name:version&feature1,features2,..."`` where features and + version is optional + :returns: triple ``(name, version, list_of_features)``; ``version`` + and ``list_of_features`` may be ``None`` if no ``':'`` / ``'&' + is contained in ``os_arg`` + """ + strip = text_type.strip + if '&' in os_arg: + os_tuple, features = map(strip, os_arg.split('&', 1)) + if features: + features = list(map(strip, features.split(','))) + else: + features = [] + else: + os_tuple, features = os_arg, None + if ':' in os_tuple: + os_name, os_version = map(strip, os_tuple.split(':', 1)) + else: + os_name, os_version = os_tuple.strip(), None + return (os_name, os_version, features) + + +def process_config(config, args): + """Post-process the loaded config given the command line arguments. + + Ensures that the ``cache_dir`` and ``sources_dir`` entries in the + config files have appropriate values. See the argparse description + set in :func:`.arguments.add_global_arguments` for a detailed + description how these directories are computed if not explicitly + set. + + Also parses ``"os_override"`` into (name, version) tuple and the + potential override of ``"os_options/features"``. + """ + if config.cache_dir is None: + if args.config_dir: + config.cache_dir = user_cache_dir(args.config_dir) + else: + if config.user_sources: + config.cache_dir = user_cache_dir( + user_config_dir(XYLEM_TOOL_NAME)) + else: + config.cache_dir = system_cache_dir( + args.prefix, XYLEM_TOOL_NAME) + if config.sources_dir is None: + if args.config_dir: + config.sources_dir = sources_dir(args.config_dir) + else: + if config.user_sources: + config.sources_dir = sources_dir( + user_config_dir(XYLEM_TOOL_NAME)) + else: + config.sources_dir = sources_dir( + system_config_dir(args.prefix, XYLEM_TOOL_NAME)) + if config.os_override is not None: + name, version, features = parse_os_override(config.os_override) + config.os_override = (name, version) + if features is not None: + config.os_options.features = features diff --git a/xylem/config_utils.py b/xylem/config_utils.py new file mode 100644 index 0000000..a6fd063 --- /dev/null +++ b/xylem/config_utils.py @@ -0,0 +1,1213 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to describe config files and expose some items on the command line. + +Configuration is done in YAML files containing dictionaries with string +keys and on the command line with `argparse`. `ConfigDescription` +objects can be created that describe the items config files, how they +are parsed, exposed on the command line and how values from multiple +sources are merged to the effective configuration. Single items are +described by `ConfigItem` objects, whose type is defined by `ConfigType` +objects. Implemented config types include `Any`, `Boolean`, `String`, +`Path`, `List`, `Dict`, `MergingDict`. The created configuration +dictionaries are of type `ConfigDict`, which derives from `dict` to +allow attribute access like ``config.os_override`` instead of only +``config["os_override"]``. + +High-level API functions include: + - :meth:`ConfigDescription.add` + - :meth:`ConfigDescription.add_arguments` + - :func:`add_global_config_arguments` + - :func:`handle_global_config_arguments` + - :func:`handle_global_config_arguments_post` + - :func:`load_config` + +Example usage: + +.. code-block:: python + + TOOL_NAME="foo-tool" + + # describe config structure and types + description = ConfigDescription("config.yaml") + add = description.add + add("foo", type=String, + command_line=True) + add("bar-flag", type=Boolean, default=False, + command_line_argument="bar") + add("quux/fiz", type=Path, + command_line=True, + help="`~` is correctly expanded") + add("quux/faz", type=List(String), + command_line_metavar='"faz1,faz2,..."') + add("dict_of_lists", type=MergingDict(List(String)), + command_line_metavar='"inst1:[key1,key2,...] ..."') + add("dict_of_dicts", type=MergingDict(Dict(Any)), + help="dict of dict of any value; not exposed to command line") + + # create argparse parser and add arguments for config items + parser = argparse.ArgumentParser(formatter_class=ConfigHelpFormatter) + description.add_arguments(parser) + + # add argument groups and some global config-related arguments that + # are independent from the description + group = parser.add_argument_group('global arguments') + add_global_config_arguments(group, "footool") + + # parse command line arguments + args = parser.parse_args() + + # handle global config arguments + handle_global_config_arguments(args, TOOL_NAME) + + # load config from files, merging with values from command line + config = load_config(args, description, TOOL_NAME) + + # handle global arguments that require the loaded config (e.g. + # `--print-config`) + handle_global_config_arguments_post(args, config, TOOL_NAME) + + # access `config` like a dictionary or with attribute access + print(config["quux"]["fiz"]) + print(config.quux.faz) + print(config.dict_of_lists['foo'][0:3]) +""" + +# TODO: for Boolean possibly add two mutually exclusive options --foo +# and --no-foo both with store_true instead of one argument +# taking a value + +from __future__ import unicode_literals + +import six +import os +import argparse +import textwrap +import re + +from copy import deepcopy +from yaml import YAMLError + +from .text_utils import type_name +from .text_utils import text_type +from .text_utils import to_str +from .util import raise_from +from .yaml_utils import load_yaml +from .yaml_utils import dump_yaml +from .log_utils import info +from .log_utils import debug +from .exception import XylemError + + +UNSET_YAML = None +"""Value to interpret as 'unset' in YAML file.""" + +UNSET_COMMAND_LINE = None +"""Value to interpret as 'unset' for command line arguments.""" + +DEFAULT_PREFIX = "/" +DEFAULT_USER_CONFIG_DIR_UNEXPANDED = "~/.config" +DEFAULT_USER_CONFIG_DIR = os.path.expanduser( + DEFAULT_USER_CONFIG_DIR_UNEXPANDED) +SYSTEM_CONFIG_PATH = "etc" +SYSTEM_CACHE_PATH = "var/cache" +USER_CACHE_PATH = "cache" + + +class ConfigError(XylemError): + + """Type for any errors related to config description and parsing.""" + + +class ConfigValueError(ConfigError): + + """Input from file or command line has wrong type or cannot be parsed. + + This is supposed to indicate a configuration error (rather than a + programming error). + """ + + +class ConfigDescriptionError(ConfigError): + + """ConfigDescription is invalid. + + This probably indicates a programming error. + """ + + +def system_config_dir(prefix, tool_name): + """Path to system configuration, given prefix and tool name.""" + return os.path.join(prefix, SYSTEM_CONFIG_PATH, tool_name) + + +def system_cache_dir(prefix, tool_name): + """Path to system cache, given prefix and tool name.""" + return os.path.join(prefix, SYSTEM_CACHE_PATH, tool_name) + + +def user_config_dir(tool_name): + """Path to user configuration, given tool name.""" + return os.path.join(DEFAULT_USER_CONFIG_DIR, tool_name) + + +def user_cache_dir(config_dir): + """Path to user cache dir, given the user config dir.""" + return os.path.join(config_dir, USER_CACHE_PATH) + + +def split_group(name): + """Split item name containing ``'/'`` into tuple of group/subname. + + Examples: + - an input of ``'foo/bar'`` results in ``('foo', 'bar')`` + - an input of ``'foo/bar/baz'`` results in ``('foo/bar', 'baz')`` + """ + return tuple(reversed([x[::-1] for x in name[::-1].split("/", 1)])) + + +def dashify(name): + """Replace ``_`` and ``/`` with ``-``.""" + return name.replace("_", "-").replace("/", "-") + + +def underscorify(name): + """Replace ``-`` and ``/`` with ``_``.""" + return name.replace("-", "_").replace("/", "_") + + +def strip_leading_dashdash(name): + """Remove leading ``'--'`` if present.""" + if name.startswith('--'): + return name[2:] + else: + return name + + +def expand_input_path(value): + """Expand and normalize path, unless it is ``None``.""" + if value is None: + return None + return os.path.abspath(os.path.expanduser(value)) + + +def type_error_msg(expected_type_name, value, what_for=None): + """Helper for exception error messages about wrong type.""" + if what_for: + what_for = ' for {}'.format(what_for) + else: + what_for = '' + return "Expected type '{}'{}, but got '{}' of type '{}'.".format( + expected_type_name, what_for, value, type_name(value)) + + +def process_config_file_yaml(config): + """Utility for parsing yaml config files. + + Handles empty files and makes sure config file is dictionary with + strings as keys. + + :raises ConfigValueError: if config is not ``None`` or dictionary + with string keys + """ + if config is None: + # allow empty config file + config = dict() + if not isinstance(config, dict): + raise ConfigValueError( + "Config file cannot be interpreted as dictionary. " + "Parsed as type '{}'.".format(type_name(config))) + for key in config: + if not isinstance(key, text_type): + raise ConfigValueError( + type_error_msg('text', key, what_for="config keys")) + return config + + +def load_config_file_yaml(path): + """Utility for loading yaml config files. + + Makes sure to always return a dict with strings as keys. Non- + existent or empty files result in an empty dictionary. + + :raises ConfigValueError: if config file cannot be opened, decode or + parsed + """ + try: + if os.path.isfile(path): + with open(path, 'rb') as f: + binary_data = f.read() + config = load_yaml(to_str(binary_data)) + else: + config = None + return process_config_file_yaml(config) + except EnvironmentError as e: + raise_from(ConfigValueError, + "failed to read config file `{}`".format(path), e) + except UnicodeError as e: + raise_from(ConfigValueError, + "failed to decode config file `{}`".format(path), e) + except YAMLError as e: + raise_from(ConfigValueError, + "failed to parse config file as YAML `{}`".format(path), e) + except ConfigValueError as e: + raise_from(ConfigValueError, + "config file has invalid structure `{}`".format(path), e) + + +def maybe_instantiate(class_or_object): + """Helper for :class:`ConfigType`. + + We want to allow passing uninstantiated classes ``String`` (instead + of ``String()``) but we also want to allow instantiated ones as in + ``List(String)``. I.e. the user can pass either a + :class:`ConfigType` derived class, or an instantiated + :class:`ConfigType` object. + """ + if isinstance(class_or_object, type): + return class_or_object() + else: + return class_or_object + + +class ConfigHelpFormatter(argparse.HelpFormatter): + + """Help formatter for argparse with support for line-breaks and paragraphs. + + The standard argparse formatter formats all argument help texts and + section descriptions into one block, ignoring all existing line- + breaks. With this formatter, three or more consecutive line-breaks + are interpreted as a paragraph break (replaced by ``\\n\\n``) and + within paragraphs, two consecutive line-breaks are interpreted as a + hard line-break (replaced by ``\\n``). Each blocks within a paragraph + (between the hard line-breaks) is still formatted as in the standard + help formatter. + + The formatter also supports raw pre-formatted paragraphs. If the + paragraphs starts with ``R|``, it is only re-indented appropriately, + but not otherwise formatted. + """ + + def __init__(self, *args, **kwargs): + super(ConfigHelpFormatter, self).__init__(*args, **kwargs) + self._paragraph_break_matcher = re.compile(r'\n\n\n+') + self._line_break_matcher = re.compile(r'\n\n') + self._raw_marker_matcher = re.compile(r'^\s*R\|') + + def _wrap_paragraphs(self, text, width, indent=''): + result = [] + for para in self._paragraph_break_matcher.split(text): + if self._raw_marker_matcher.match(para): + para = self._raw_marker_matcher.sub( + lambda m: ' '*(m.end()-m.start()), para) + para = textwrap.dedent(para).splitlines() + result.extend([indent + l for l in para]) + else: + for nobreak in self._line_break_matcher.split(para): + block = self._whitespace_matcher.sub(' ', nobreak).strip() + result.extend(textwrap.wrap(block, width, + initial_indent=indent, + subsequent_indent=indent)) + result.append("") + if result: + result = result[0:-1] + return result + + def _split_lines(self, text, width): + return self._wrap_paragraphs(text, width) + + def _fill_text(self, text, width, indent): + return "\n".join(self._wrap_paragraphs(text, width, indent)) + + +class ConfigDict(dict): + + """Derivative of ``dict`` to allow attribute access. + + This is used for all config dicts to allow attribute access like + ``config.os_override`` instead of ``config["os_override"]``. + """ + + def __init__(self, *args, **kwargs): + super(ConfigDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + +def ceorce_config_dict(config): + """Helper to convert a regular dictionary to a config dict. + + Calls ``ceorce_config_dict`` recursively on dict type values. Does + not copy or modify if ``config`` is already of type ``ConfigDict``. + + The resulting config dict may share structure with the input. + + :type config: `dict` or `ConfigDict` + """ + if isinstance(config, ConfigDict): + return config + if not isinstance(config, dict): + return config + result = ConfigDict() + for k, v in six.iteritems(config): + result[k] = ceorce_config_dict(v) + + +def copy_to_dict(config): + """Helper to copy a ConfigDict instance to a regular dict. + + Calls ``copy_to_dict`` recursively on values of type `ConfigDict`, + and `deepcopy` on keys and other values. + + :param ConfigDict config: (nested) config dict + """ + result = {} + for k, v in six.iteritems(config): + if isinstance(v, ConfigDict): + v = copy_to_dict(v) + else: + v = deepcopy(v) + result[deepcopy(k)] = v + return result + + +class ConfigType(object): + + """Type describing a config file entry. + + The config type provides information on how to parse config values + from yaml and command line and how to verify appropriate values. It + further describes how multiple values from a cascade of config files + (e.g. command line, user config, system config, default) are to be + merged. + + A value of ``None`` by default refers to the config item being + unset, but custom 'unset' values can be specified with + :meth:`unset_value`. + """ + + def from_yaml(self, value): + """Create config value from yaml loaded config value. + + This for example allows to expand shorthand notation into a + uniform representations, or transform yaml into custom datatypes + as needed. + + A value of `UNSET_YAML` should be interpreted as 'unset'. + + + :param value: python data structure representing the loaded yaml + from the config file. Thus the type is one of (None, bool, + str, list, dict) + :raises ConfigValueError: if ``value`` has invalid type or + structure + """ + if value == UNSET_YAML: + return self.unset_value() + return self.verify(value) + + def from_command_line(self, value): + """Create config value from value of command line parser. + + A value of `UNSET_COMMAND_LINE` should be interpreted as + 'unset'. + + The default implementation parses the input string as yaml and + then invokes :meth:`from_yaml`. + + Before returning a value, :meth:`verify` should be used to + verify the validity of the parsed value. + + :param value: value from the command line parser, which is a + string when :meth:`command_line_multiple()` is false and a + list of strings otherwise, or `UNSET_COMMAND_LINE` if the + argument was omitted on the command line. + :raises ConfigValueError: if ``value`` cannot be parsed properly + or has invalid type or structure + """ + if value == UNSET_COMMAND_LINE: + return self.unset_value() + try: + return self.from_yaml(load_yaml(value)) + except YAMLError as e: + raise_from( + ConfigValueError, "parsing command line value failed", e) + + def verify(self, value): + """Verify that value is of correct type and structure. + + This should be called by :meth:`from_yaml` or + :meth:`from_command_line` before returning the parsed value, but + can also be used independently for example when changing the + configuration value from within the program. + + An value of :meth:`unset_value` is considered valid. + + :returns: ``value`` unchanged + :raises ConfigValueError: if ``value`` is not of appropriate + type and structure + """ + return value + + def merge(self, top, bottom): + """Merge two values of this config entry. + + The default implementation replaces ``bottom`` with ``top``, + unless top is :meth`unset_value`. + + :param top: new value taking priority + :param bottom: value being overwritten or extended + """ + if not self.is_unset(top): + return top + else: + return bottom + + def unset_value(self): + """Value representing an unset config option for this type. + + Determines what value should appear in the config dict if this + option is omitted from all config files (and on the command + line) and no default value was specified. Unset value comparison + is done by equality (`==`). + + :meth:`verify` should accept this value as valid. :meth:`merge` + should properly handle this value and :meth:`from_yaml` as well + as :meth:`from_command_line` should convert + `UNSET_YAML`/`UNSET_COMMAND_LINE` to this 'unset' value. + """ + return None + + def is_unset(self, value): + """Return ``True`` if ``value`` equals :meth:`unset_value`.""" + return value == self.unset_value() + + def command_line_multiple(self): + """Determine if we should allow multiple command line options. + + Types like "List" and "Dict" can be composed of multiple command + line options. If this returns `True`, + :meth:`from_command_line` needs to handle input of type list. + """ + return False + + def command_line_default_metavar(self): + """Customize the default metavar used for command line help. + + By default, this returns None meaning argparse is setting the + default metavar. + """ + return None + + @staticmethod + def command_line_parsing_help(): + return None + + +class Any(ConfigType): + + """Config type allowing arbitrary (yaml) structures.""" + + @staticmethod + def command_line_parsing_help(): + return """any YAML structure""" + + +class String(ConfigType): + + """Config type for simple unparsed strings.""" + + def verify(self, value): + if not self.is_unset(value): + if not isinstance(value, text_type): + raise ConfigValueError(type_error_msg('string', value)) + return value + + def from_command_line(self, value): + if value == UNSET_COMMAND_LINE: + return self.unset_value() + return self.from_yaml(to_str(value)) + + def command_line_default_metavar(self): + return "STRING" + + @staticmethod + def command_line_parsing_help(): + return """string parsed as-is""" + + +class Boolean(ConfigType): + + """Config type for booleans (parsed as YAML).""" + + def verify(self, value): + if not self.is_unset(value): + if not isinstance(value, bool): + raise ConfigValueError(type_error_msg('bool', value)) + return value + + def command_line_default_metavar(self): + return "yes|no" + + +class List(ConfigType): + + """Config type for lists supporting custom element types.""" + + def __init__(self, element_type=Any): + self.element_type = maybe_instantiate(element_type) + + def from_yaml(self, value): + if value == UNSET_YAML: + return self.unset_value() + if not isinstance(value, list): + raise ConfigValueError(type_error_msg('list', value)) + return self.verify( + [self.element_type.from_yaml(elt) for elt in value]) + + def from_command_line(self, value): + if value == UNSET_COMMAND_LINE: + return self.unset_value() + + def parse_single(input): + try: + result = load_yaml(input) + except YAMLError: + result = None + if not isinstance(result, list): + # special parsing allowing lists without enclosing `[]` + try: + result = load_yaml("[" + input + "]") + except YAMLError as e: + raise_from(ConfigValueError, + "failed to parse `{}` as list".format(input), e) + if not isinstance(result, list): + raise ConfigValueError(type_error_msg("list", result)) + return result + + # list value means multiple command line arguments that should + # be concatenated + if not isinstance(value, list): + value = [value] + result = [] + for string in value: + result.extend(parse_single(string)) + return self.from_yaml(result) + + def verify(self, value): + if not self.is_unset(value): + if not isinstance(value, list): + raise ConfigValueError(type_error_msg('list', value)) + try: + for elt in value: + self.element_type.verify(elt) + except ConfigValueError as e: + raise_from(ConfigValueError, "invalid list element", e) + return value + + def command_line_multiple(self): + return True + + def command_line_default_metavar(self): + return '"item1,item2,..."' + + @staticmethod + def command_line_parsing_help(): + return """the outer '[]' can be omitted; multiple occurrences of + the same command line argument are concatenated""" + + +class Dict(ConfigType): + + """Config type for dictionaries supporting custom key and value types.""" + + def __init__(self, value_type=Any, key_type=String): + self.key_type = maybe_instantiate(key_type) + self.value_type = maybe_instantiate(value_type) + + def from_yaml(self, value): + if value == UNSET_YAML: + return self.unset_value() + if not isinstance(value, dict): + raise ConfigValueError(type_error_msg("dict", value)) + result = {} + for k, v in six.iteritems(value): + result[self.key_type.from_yaml(k)] = \ + self.value_type.from_yaml(v) + return self.verify(result) + + def from_command_line(self, value): + if value == UNSET_COMMAND_LINE: + return self.unset_value() + + def parse_single(input): + try: + result = load_yaml(input) + except YAMLError: + result = None + if not isinstance(result, dict): + # special parsing allowing lists without enclosing `{}` + try: + result = load_yaml("{" + input + "}") + except YAMLError as e: + raise_from(ConfigValueError, + "failed to parse `{}` as dict".format(input), e) + if not isinstance(result, dict): + raise ConfigValueError(type_error_msg("dict", result)) + return result + + # list value means multiple command line arguments to be merged + if not isinstance(value, list): + value = [value] + result = {} + for d in value: + d = parse_single(d) + existing_keys = set(d.keys()) & set(result.keys()) + if existing_keys: + raise ConfigValueError( + "Dicts defined via multiple command line arguments must " + "have unique keys. These keys occur more than once: `{}`.". + format(existing_keys)) + result.update(d) + return self.from_yaml(result) + + def verify(self, value): + if not self.is_unset(value): + if not isinstance(value, dict): + raise ConfigValueError(type_error_msg('dict', value)) + try: + for k, v in six.iteritems(value): + self.key_type.verify(k) + self.value_type.verify(v) + except ConfigValueError as e: + raise_from(ConfigValueError, "invalid key or value", e) + return value + + def command_line_multiple(self): + return True + + def command_line_default_metavar(self): + return '"key:value,key2:value2,..."' + + @staticmethod + def command_line_parsing_help(): + return """the outer '{}' can be omitted; multiple occurrences of + the same command line argument are merged""" + + +class MergingDict(Dict): + + """Dictionary whose entries are merged when config files are merged.""" + + def __init__(self, *args, **kwargs): + super(MergingDict, self).__init__(*args, **kwargs) + + def merge(self, top, bottom): + # merge the dictionary key by key + result = bottom.copy() + result.update(top) + return result + + def unset_value(self): + return {} + + @staticmethod + def command_line_parsing_help(): + return Dict.command_line_parsing_help() + \ + "; value is merged with the config file entries" + + +class Path(String): + + """Path that gets expanded and normalized.""" + + def from_yaml(self, value): + if value == UNSET_YAML: + return self.unset_value() + value = super(Path, self).from_yaml(value) + return self.verify(expand_input_path(value)) + + def command_line_default_metavar(self): + return "PATH" + + @staticmethod + def command_line_parsing_help(): + return """string parsed as-is; path is expanded (e.g. `~`)""" + + +class ConfigItem(object): + + """Holds meta information about one entry in a config file. + + :cvar str name: name of the config item; may contain (up to one) '/' + to specify grouping into a sub-dictionary, which is interpreted + as 'group/subname' + :cvar str help: description of the config item for the argument + parser help + :cvar ConfigType type: type of config item describing how it is + parsed and merged + :cvar default: default value to be used if unset + :cvar bool command_line: if true, this config is exposed on the + command line; ``False`` by default, but specifying any other + ``command_line_...`` arguments implies ``True`` for + ``command_line``, unless explicitly set to ``False``. + :cvar command_line_argument: command line argument name (without + leading dashdash); if not explicitly supplied, this is derived + from name by replacing all '_' and '/' with '-' + :cvar command_line_metavar: command line argument metavar; if not + explicitly supplied, the default for the type is used + """ + + def __init__(self, + name, + help=None, + type=Any, + default=None, + command_line=None, + command_line_argument=None, + command_line_metavar=None): + self.name = name + self.help = help + self.type = maybe_instantiate(type) + if default is None: + self.default = self.type.unset_value() + else: + self.default = default + if command_line is None: + command_line = any([x is not None for x in [command_line_argument, + command_line_metavar]]) + if command_line_argument: + command_line_argument = \ + strip_leading_dashdash(command_line_argument) + elif command_line: + command_line_argument = dashify(name) + self.command_line = command_line + self.command_line_argument = command_line_argument + if command_line_metavar is None: + command_line_metavar = self.type.command_line_default_metavar() + self.command_line_metavar = command_line_metavar + + @staticmethod + def is_group_name(name): + return "/" in name + + def is_group(self): + return self.is_group_name(self.name) + + def help_string(self): + """Compose help string for command line parser.""" + help = "" + if self.help: + help += self.help + "\n\n" + info = [] + if self.is_group() or self.command_line_argument != dashify(self.name): + info += ["config: `{}`".format(self.name)] + info += ["type: {}".format(type_name(self.type))] + if self.default != self.type.unset_value(): + info.append("default: {}".format(self.default)) + help += "(" + ", ".join(info) + ")" + return help + + def add_argument(self, parser): + """Add command line argument for this config item to ``parser``.""" + assert(self.command_line) + kwargs = {} + kwargs["help"] = self.help_string() + kwargs["default"] = UNSET_COMMAND_LINE + if self.type.command_line_multiple(): + kwargs["action"] = "append" + if self.command_line_metavar is not None: + kwargs["metavar"] = self.command_line_metavar + parser.add_argument('--' + self.command_line_argument, **kwargs) + + +class ConfigDescription(object): + + """Describes structure of a config file. + + It also describes which options are exposed to command line and how + to parse them. + + A configuration is a dictionary, where each item corresponds to a + entry in that dictionary with the item name as key. There is + support for one level of grouping. Item names containing a '/' are + interpreted as 'group/subname'. Groups are sub-dictionaries as + entries in the config dictionary. The subnames of the config items + constitute the keys of the group dicts. When config dicts are + merged, Group dicts are merged automatically key but key. + + :cvar str namespace: path of the config file relative to the root + config directory + :cvar list itemlist: list of all items in order (including items in + groups) + :cvar dict items: mapping names to items (including full + names for items in groups) + :cvar dict groups: mapping group names to dict of subnames to items + :cvar dict command_line_arguments: mapping command line argument + names to items + """ + + def __init__(self, namespace): + self.namespace = namespace + self.itemlist = [] + self.items = {} + self.groups = {} + self.command_line_arguments = {} + + def add(self, name, *args, **kwargs): + """Add item with given name to the description. + + Additional arguments are passed to the :class:`ConfigItem` + constructor. + + :raises ConfigDescriptionError: if the newly added config item + is invalid or conflicting with the existing items + """ + if name in self.items: + raise ConfigDescriptionError("Duplicate name '{}'".format(name)) + if name in self.groups: + raise ConfigDescriptionError( + "Name '{}' already exists as group".format(name)) + + item = ConfigItem(name, *args, **kwargs) + + if "/" in name: + group, subname = split_group(name) + if group not in self.groups: + if group in self.items: + raise ConfigDescriptionError( + "Group name '{}' already exists as non-group item.". + format(group)) + else: + self.groups[group] = {} + self.groups[group][subname] = item + + self.itemlist.append(item) + self.items[name] = item + + if item.command_line: + arg_name = item.command_line_argument + if arg_name in self.command_line_arguments: + raise ConfigDescriptionError( + "Duplicate command line argument name '{}' for config " + "item '{}' and '{}'.". + format(arg_name, item.name, + self.command_line_arguments[arg_name].name)) + self.command_line_arguments[arg_name] = item + + def add_arguments(self, parser): + """Create a group and add all command-line-enabled items to it. + + See :func:`config_from_args` on how to process the passed + arguments to a config dict. + """ + typeset = {type(i.type) for i in self.itemlist + if i.type.command_line_parsing_help()} + if typeset: + typelist = list(typeset) + typelist.sort(key=to_str) + helplist = ["{}: {}".format(to_str(t), + t.command_line_parsing_help()) + for t in typelist] + typehelp = "\n\n\nThere are some special cases and short hand " \ + "notation for parsing config arguments:\n\n* " + "\n\n* ". \ + join(helplist) + else: + typehelp = "" + subparser = parser.add_argument_group( + "config arguments", description="""The following typed + arguments correspond to entries in the config file. Command + line argument values are interpreted as YAML and override + the corresponding entries in user/system config files. """ + + typehelp) + for item in self.itemlist: + if item.command_line: + item.add_argument(subparser) + + +def add_global_config_arguments(parser, tool_name): + """Add global command line arguments related to the configuration setup. + + These options are independent from a config description. + + See :func:`handle_global_config_arguments`, + :func:`handle_global_config_arguments_post` + """ + prefix_env_var = "{}_PREFIX".format(tool_name.upper()) + config_dir_env_var = "{}_DIR".format(tool_name.upper()) + config_argument = "--{}-dir".format(tool_name) + + add = parser.add_argument + add('--print-config', action='store_true', + help="""print the effective configuration (from command line and + config files) and exit""") + add('--no-user-config', action="store_true", + help="""disable user config (system-wide config only)""") + add('--prefix', metavar=prefix_env_var, + help="""Set the FHS prefix for finding system-wide + configurations and caches. System-wide configuration files will + be looked for in '{0}' and cache files will be placed in '{1}'. + The default prefix is '{2}'. The prefix can also be set by the + {3} environment variable, but the command line option takes + precedence.""".format( + system_config_dir(prefix_env_var, tool_name), + system_cache_dir(prefix_env_var, tool_name), + DEFAULT_PREFIX, prefix_env_var)) + add(config_argument, metavar=config_dir_env_var, dest="config_dir", + help="""Specify the path of directory to use for configuration + and caches. Configuration files will be looked for in '{0}' and + caches will be placed in '{1}'. This is intended to be used by + tools that want to create a local configuration and cache + independent from the system-wide setup. If set, only the + configuration files in {0} are used instead of system and user + configuration files, i.e. {2} will be ignored. This can also + be set by the {0} environment variable, but the command line + option takes precedence.""".format( + config_dir_env_var, + user_cache_dir(config_dir_env_var), + prefix_env_var)) + + +def handle_global_config_arguments(args, tool_name): + """Handle global configuration-related command line arguments. + + This deals with expanding provided paths, as well as looking up + environment variables for arguments not passed. It updates ``args`` + accordingly. + + It is intended to be called before the effective config dict is + created. + + See :func:`add_global_config_arguments`, + :func:`handle_global_config_arguments_post` + + :param argparse.Namespace args: parsed command line arguments + :param str tool_name: name of the tool to determine config file + locations, argument names and environment variables + """ + prefix_env_var = "{}_PREFIX".format(tool_name.upper()) + config_dir_env_var = "{}_DIR".format(tool_name.upper()) + + if args.config_dir is None: + args.config_dir = os.environ.get(config_dir_env_var, None) + if args.prefix is None: + args.prefix = os.environ.get(prefix_env_var, DEFAULT_PREFIX) + + args.config_dir = expand_input_path(args.config_dir) + args.prefix = expand_input_path(args.prefix) + if args.config_dir and args.prefix: + debug("Specifed both prefix `{}` and {} dir `{}`. The prefix will be " + "ignored.".format(args.prefix, tool_name, args.config_dir)) + + +def handle_global_config_arguments_post(args, config, tool_name): + """Handle global configuration-related command line arguments with config. + + This deals with global config-related arguments that require the + loaded configuration, e.g. ``--print-config``. + + The is intended to be called after the effective config dict has + been created. + + See :func:`add_global_config_arguments`, + :func:`handle_global_config_arguments` + + :param argparse.Namespace args: parsed command line arguments + :param dict config: config dict + :param str tool_name: name of the tool to determine config file + locations, argument names and environment variables + """ + if args.print_config: + info(dump_yaml(config)) + exit(0) + + +def copy_conifg_dict(description, config): + """Utility to shallow-copy config dict.""" + result = ConfigDict(config) + for group in description.groups: + result[group] = ConfigDict(result[group]) + return result + + +def merge_configs(description, top, *more_configs): + """Merge two or more config dicts. + + The configurations are merged on a key by key basis as described by + ``description``, using the :meth:`ConfigType.merge` for each + :class:`ConfigItem`. + + ``top`` takes highest priority and the configs in ``more_configs`` + have decreasing priority. The config dicts are assumed to have been + loaded with the ``config_from_...`` functions using ``description``. + This means it is assumed that all keys and groups as described by + ``description`` exist in all input config dicts. + + :param description: description of the config dicts; the type + information of each entry is used for merging + :type description: `ConfigDescription` + :param dict top: config dict with structure defined by + ``description`` + :param more_configs: list of configuration dicts with structure + defined by ``description`` + :type more_configs: list of dicts + """ + result = copy_conifg_dict(description, top) + for bottom_config in more_configs: + for name, item in six.iteritems(description.items): + if "/" in name: + group, subname = split_group(name) + result[group][subname] = item.type.merge( + result[group][subname], + bottom_config[group][subname]) + else: + result[name] = item.type.merge(result[name], + bottom_config[name]) + return result + + +def load_config(args, description, tool_name): + """Load configuration from config files and command line arguments. + + High-level API for loading config defined by system/user config + files and config from the command line ``args``. + + Uses :func:`merge_configs` on config dicts loaded by + :func:`config_from_defaults`, :func:`config_from_file` and + :func:`config_from_args`. + + :param argparse.Namespace args: parsed command line arguments + :param description: description of the items comprising the + configuration + :type description: `ConfigDescription` + :param str tool_name: name of the tool to determine config file + location + :returns: config dict with structure as defined by ``description`` + :raises ConfigValueError: if parsing of files or arguments fails + """ + if args.config_dir is not None: + dirs = [args.config_dir] + else: + system_dir = system_config_dir(args.prefix, tool_name) + user_dir = user_config_dir(tool_name) + if args.no_user_config: + dirs = [system_dir] + else: + dirs = [user_dir, system_dir] + filenames = [os.path.join(d, description.namespace) for d in dirs] + configs = [config_from_args(args, description)] + \ + [config_from_file(f, description) for f in filenames] + \ + [config_from_defaults(description)] + return merge_configs(description, *configs) + + +def config_from_defaults(description): + """Return a config dictionary with default values for all items. + + :param description: description of the items comprising the + configuration + :type description: `ConfigDescription` + :returns: config dict with structure as defined by ``description`` + """ + result = ConfigDict() + for name, item in six.iteritems(description.items): + if "/" in name: + group, subname = split_group(name) + if group not in result: + result[group] = ConfigDict() + result[group][subname] = item.default + else: + result[name] = item.default + return result + + +def config_from_args(args, description): + """Return config dictionary from command line arguments. + + See :meth:`ConfigDescription.add_arguments` for creating compatible + command line arguments. + + :param argparse.Namespace args: parsed command line arguments + :param description: description of the items comprising the + configuration + :type description: `ConfigDescription` + :returns: config dict with structure as defined by ``description`` + :raises ConfigValueError: if parsing of arguments fails + """ + config = ConfigDict() + args_dict = vars(args) + + def process_item(target_dict, key, item): + if item.command_line: + value = item.type.from_command_line( + args_dict[underscorify(item.command_line_argument)]) + else: + value = item.type.unset_value() + target_dict[key] = value + + for group, group_dict in six.iteritems(description.groups): + config[group] = ConfigDict() + for subname, item in six.iteritems(group_dict): + process_item(config[group], subname, item) + for name, item in six.iteritems(description.items): + # only non-group items + if "/" not in name: + process_item(config, name, item) + return config + + +def config_from_file(filename, description): + """Return config dictionary from config file. + + :param str filename: path to the config file + :param description: description of the items comprising the + configuration; config dict is created according to this + description; any missing keys are created with 'unset' values + and unknown entries are ignored + :type description: `ConfigDescription` + :returns: config dict with structure as defined by ``description`` + :raises ConfigValueError: if parsing of files fails + """ + contents = load_config_file_yaml(filename) + return config_from_parsed_yaml(contents, description) + + +def config_from_parsed_yaml(data, description): + """Utility for creating config dict from parsed YAML file. + + :param dict data: dictionary (nested yaml structure) as read from + file (see :func:`load_config_file_yaml`) + :param description: config dict is created according to this + description; any missing keys are created with 'unset' values + and unknown entries are ignored + :type description: `ConfigDescription` + :returns: config dict with structure as defined by ``description`` + :raises ConfigValueError: if parsing of files fails + """ + def process_item(target_dict, source_dict, key, item): + value = source_dict.get(key, None) + value = item.type.from_yaml(value) + target_dict[key] = value + + config = ConfigDict() + for group, group_dict in six.iteritems(description.groups): + config[group] = ConfigDict() + data_group = data.get(group, {}) + for subname, item in six.iteritems(group_dict): + process_item(config[group], data_group, subname, item) + for name, item in six.iteritems(description.items): + # only handle non-group items + if "/" not in name: + process_item(config, data, name, item) + return config diff --git a/xylem/exception.py b/xylem/exception.py index eaf167b..bf9b6a9 100644 --- a/xylem/exception.py +++ b/xylem/exception.py @@ -22,12 +22,17 @@ # by multiple submodules? -class InvalidDataError(Exception): +class XylemError(Exception): + + """Common base class for all custom xylem exceptions.""" + + +class InvalidDataError(XylemError): """Data is not in valid xylem format.""" -class InvalidPluginError(Exception): +class InvalidPluginError(XylemError): """Plugin loaded from an entry point does not have the right type/data.""" @@ -45,14 +50,14 @@ class InvalidPluginError(Exception): # return self.message -class DownloadFailure(Exception): +class DownloadFailure(XylemError): """Failure downloading data for I/O or other reasons.""" pass -class InstallerNotAvailable(Exception): +class InstallerNotAvailable(XylemError): """Failure indicating a installer is not installed.""" diff --git a/xylem/installers/impl.py b/xylem/installers/impl.py index e7f9af3..e4694e8 100644 --- a/xylem/installers/impl.py +++ b/xylem/installers/impl.py @@ -16,12 +16,14 @@ import pkg_resources -from xylem.os_support import OSSupport -from xylem.exception import InvalidPluginError -from xylem.log_utils import info, warning, is_verbose -from xylem.text_utils import text_type from six.moves import map +from ..os_support import OSSupport +from ..exception import InvalidPluginError +from ..log_utils import info, warning, is_verbose +from ..text_utils import text_type +from ..config import get_config + INSTALLER_GROUP = "xylem.installers" @@ -29,6 +31,9 @@ # TODO: split out verify installer plugin function +# TODO: fix docstrings + + def load_installer_plugin(entry_point): """Load Installer plugin from entry point. @@ -88,20 +93,18 @@ class InstallerContext(object): priorities. """ - # TODO: Make parameters for passing os/version pair more uniform + def __init__(self, config=None, setup_installers=True): - def __init__(self, setup_installers=True, os_override=None): + if config is None: + config = get_config() - if isinstance(os_override, OSSupport): - self.os_support = os_override + self.os_support = OSSupport() + if config.os_override: + self.set_os_override(config.os_override) else: - self.os_support = OSSupport() - if os_override: - self.set_os_override(os_override) - else: - self.os_support.detect_os() - if is_verbose(): - info("detected OS [%s:%s]" % self.get_os_tuple()) + self.os_support.detect_os() + if is_verbose(): + info("detected OS [%s]" % self.get_os_string()) self.installer_plugins = get_installer_plugin_list() @@ -122,6 +125,8 @@ def set_os_override(self, os_tuple): if is_verbose(): info("overriding OS to [%s:%s]" % os_tuple) self.os_support.override_os(os_tuple) + if is_verbose() and os_tuple[1] is None: + info("detected OS version [%s]" % self.get_os_string()) def get_os_tuple(self): """Get the OS (name,version) tuple. diff --git a/xylem/lookup.py b/xylem/lookup.py index 6da0fe7..e6395a6 100644 --- a/xylem/lookup.py +++ b/xylem/lookup.py @@ -17,21 +17,22 @@ from .sources import SourcesContext from .sources import RulesDatabase from .installers import InstallerContext -from .specs.rules import compact_installer_dict +from .specs.plugins.rules import compact_installer_dict +from .config import get_config -def lookup(xylem_key, prefix=None, os_override=None, compact=False): +def lookup(xylem_key, compact=False, config=None, sources_context=None, + installer_context=None): - sources_context = SourcesContext(prefix=prefix) + if config is None: + config = get_config() + + sources_context = sources_context or SourcesContext(config) + ic = installer_context or InstallerContext(config) database = RulesDatabase(sources_context) database.load_from_cache() - if isinstance(os_override, InstallerContext): - ic = os_override - else: - ic = InstallerContext(os_override=os_override) - installer_dict = database.lookup(xylem_key, ic) if compact: diff --git a/xylem/os_support/impl.py b/xylem/os_support/impl.py index 667dfef..2945ff3 100644 --- a/xylem/os_support/impl.py +++ b/xylem/os_support/impl.py @@ -16,16 +16,19 @@ import pkg_resources -from xylem.log_utils import warning -from xylem.exception import InvalidPluginError from six.moves import map + +from ..log_utils import warning +from ..exception import XylemError +from ..exception import InvalidPluginError + # TODO: Document the description of how OS plugins look like (maybe in # module docstring?) - # COMMENT: @wjwwood: This is good to describe, minimally, in the module # docstring, however, the details on how to write one and what # all the ramifications and such are can go in a high level # document in the developer docs. + OS_GROUP = 'xylem.os' @@ -33,6 +36,8 @@ # are not any of the special values like any_os, any_version, # default_installer etc +# TODO: fix docstrings and error handling + def load_os_plugin(entry_point): """Load OS plugin from entry point. @@ -80,7 +85,7 @@ def get_os_plugin_list(): return os_list -class UnsupportedOSError(Exception): +class UnsupportedOSError(XylemError): """ Operating system unsupported. @@ -204,7 +209,10 @@ def __init__(self, os, version): # raise RuntimeError("Tried to override OS '{0}' with invalid " # "version '{1}'.".format(os.name(), version)) self.os = os - self.version = version + if version is None: + self.version = os.get_version() + else: + self.version = version def is_os(self): """Detection for OverrideOS is always `True`.""" diff --git a/xylem/os_support/os_detect.py b/xylem/os_support/os_detect.py index a28e07e..18e15fc 100644 --- a/xylem/os_support/os_detect.py +++ b/xylem/os_support/os_detect.py @@ -29,6 +29,9 @@ import locale import codecs +from ..exception import XylemError + + def _read_stdout(cmd): try: pop = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -55,7 +58,7 @@ def read_issue(filename="/etc/issue"): return f.read().split() return None -class OsNotDetected(Exception): +class OsNotDetected(XylemError): """ Exception to indicate failure to detect operating system. """ @@ -411,7 +414,7 @@ def register_default(os_name, os_detector): def detect_os(self, env=None): """ Detect operating system. Return value can be overridden by - the :env:`ROS_OS_OVERRIDE` environment variable. + the ``ROS_OS_OVERRIDE`` environment variable. :param env: override ``os.environ`` :returns: (os_name, os_version, os_codename), ``(str, str, str)`` diff --git a/xylem/resolve.py b/xylem/resolve.py index 30257d8..b156730 100644 --- a/xylem/resolve.py +++ b/xylem/resolve.py @@ -19,27 +19,23 @@ from .installers import InstallerContext from .log_utils import debug +from .config import get_config -def resolve(xylem_keys, prefix=None, os_override=None, all_keys=False): +def resolve(xylem_keys, all_keys=False, config=None, sources_context=None, + installer_context=None): + if config is None: + config = get_config() + sources_context = sources_context or SourcesContext(config) + ic = installer_context or InstallerContext(config) # xylem_keys can be one key or list of keys, return value accordingly - - sources_context = SourcesContext(prefix=prefix) - database = RulesDatabase(sources_context) database.load_from_cache() - if isinstance(os_override, InstallerContext): - ic = os_override - else: - ic = InstallerContext(os_override=os_override) - list_argument = isinstance(xylem_keys, list) - if not list_argument: xylem_keys = [xylem_keys] - requested_keys = list(xylem_keys) if all_keys: @@ -48,15 +44,11 @@ def resolve(xylem_keys, prefix=None, os_override=None, all_keys=False): xylem_keys = set(xylem_keys) result = [] - for key in xylem_keys: - installer_dict = database.lookup(key, ic) - if not installer_dict: raise LookupError("Could not find rule for xylem key '{0}' on " "'{1}'.".format(key, ic.get_os_string())) - rules = [] for installer_name, rule in installer_dict.items(): priority = ic.get_installer_priority(installer_name) @@ -73,7 +65,6 @@ def resolve(xylem_keys, prefix=None, os_override=None, all_keys=False): # TODO: use installer instead of installer_name here? rules.append((priority, installer_name, resolutions)) - if not rules: # This means we have rules, but non for registered # installers, ignore this key unless it is in the requested @@ -96,12 +87,10 @@ def resolve(xylem_keys, prefix=None, os_override=None, all_keys=False): else: rules.sort(reverse=True) result.append((key, rules)) - if not list_argument: assert(len(result) == 1) key, resolution = result[0] result = resolution else: result = sorted(result) - return result diff --git a/xylem/sources/database.py b/xylem/sources/database.py index 5580c85..5340b4e 100644 --- a/xylem/sources/database.py +++ b/xylem/sources/database.py @@ -26,6 +26,7 @@ from ..log_utils import warning from ..log_utils import info from ..log_utils import is_verbose +from ..log_utils import debug from .impl import get_default_source_descriptions from .impl import get_source_descriptions from ..text_utils import to_str @@ -191,6 +192,9 @@ def __init__(self, sources_context): self.raise_on_error = True def init_from_sources(self): + debug("initializing database with sources dir `{}` and cache dir `{}`". + format(self.sources_context.sources_dir, + self.sources_context.cache_dir)) self.sources = [] sources_dir = self.sources_context.sources_dir sources_gen = get_source_descriptions(sources_dir) diff --git a/xylem/sources/impl.py b/xylem/sources/impl.py index a57401b..c10c8ac 100644 --- a/xylem/sources/impl.py +++ b/xylem/sources/impl.py @@ -19,16 +19,22 @@ import pkg_resources import yaml -from .. import DEFAULT_PREFIX +from ..config import DEFAULT_SOURCES_DIR +from ..config import DEFAULT_CACHE_DIR +from ..config import get_config from ..log_utils import error from ..log_utils import is_verbose from ..log_utils import info -from ..log_utils import debug -from ..util import load_yaml +from ..yaml_utils import load_yaml from ..util import raise_from from ..text_utils import to_str from ..specs import verify_spec_name from ..specs import get_spec_plugin_list +from ..exception import XylemError + + +SOURCES_CACHE_PATH = "sources" +"""Path of sources cache inside the cache dir.""" # is supposed to work like a entry_point? Does package data work that @@ -174,27 +180,15 @@ def verify_source_description(descr): "spec name '{0}'".format(keys[0], e)) -def sources_dir_from_prefix(prefix): - return os.path.join(prefix, "etc", "xylem", "sources.d") - - -def cache_dir_from_prefix(prefix): - return os.path.join(prefix, "var", "cache", "xylem", "sources") - - -def sources_dir_from_xylem_dir(xylem_dir): - return os.path.join(xylem_dir, "sources.d") - - -def cache_dir_from_xylem_dir(xylem_dir): - return os.path.join(xylem_dir, "cache", "sources") +def sources_cache_dir(cache_dir): + return os.path.join(cache_dir, SOURCES_CACHE_PATH) # TODO: Introduce CachePermissionError to specifically indicate # permission problems with creating or manipulating the cache dir -class UnknownSpecError(Exception): +class UnknownSpecError(XylemError): pass @@ -207,49 +201,36 @@ class UnknownSpecError(Exception): # argument for the rosdistro spec plugin). Keeping plugins single- # purpose seems like a good idea. -# TODO: rethink 'prefix': don't use environment variable; provide better -# default for sources.d/cache in user home dir (no fhs structure); -# possibly provide option to set sources.d separately; Note: we have a -# suggestion towards this with the `xylem_dir` below, which can be set -# alternative to `prefix` (use prefix in FHS scenario and xylem_dir to -# manage sources/cache in user's home or in temporary directory) class SourcesContext(object): - def __init__(self, prefix=None, xylem_dir=None, spec_plugins=None): - self.setup_paths(prefix, xylem_dir) + def __init__(self, config=None, spec_plugins=None): + if config is None: + config = get_config() + self.setup_paths(config) self.spec_plugins = spec_plugins or get_spec_plugin_list() - def setup_paths(self, prefix=None, xylem_dir=None): - if prefix and xylem_dir: - debug("Specifed both prefix '{0}' and xylem dir '{1}'. " - "The prefix will be ignored.".format(prefix, xylem_dir)) - if xylem_dir: - self.xylem_dir = xylem_dir - self.prefix = None - self.sources_dir = sources_dir_from_xylem_dir(self.xylem_dir) - self.cache_dir = cache_dir_from_xylem_dir(self.xylem_dir) - else: - self.prefix = prefix or DEFAULT_PREFIX - self.xylem_dir = None - self.sources_dir = sources_dir_from_prefix(self.prefix) - self.cache_dir = cache_dir_from_prefix(self.prefix) + def setup_paths(self, config): + self.sources_dir = config.sources_dir or DEFAULT_SOURCES_DIR + cache_dir = config.cache_dir or DEFAULT_CACHE_DIR + self.cache_dir = sources_cache_dir(cache_dir) def ensure_cache_dir(self): if not self.cache_dir_exists(): os.makedirs(self.cache_dir) - # TODO: check if dir is directory (or link to dir) not file + if not self.cache_dir_exists(): + raise OSError("Could not create cache dir: `{}`". + format(self.cache_dir)) def cache_dir_exists(self): - return os.path.exists(self.cache_dir) - # TODO: check if dir is directory (or link to dir) not file + return os.path.isdir(self.cache_dir) def sources_dir_exists(self): - return os.path.exists(self.sources_dir) - # TODO: check if dir is directory (or link to dir) not file + return os.path.isdir(self.sources_dir) def is_default_dirs(self): - return self.prefix == DEFAULT_PREFIX + return (self.sources_dir == DEFAULT_SOURCES_DIR and + self.cache_dir == sources_cache_dir(DEFAULT_CACHE_DIR)) def get_spec(self, spec_name): for spec in self.spec_plugins: diff --git a/xylem/specs/plugins/rules.py b/xylem/specs/plugins/rules.py index 56df202..57a267a 100644 --- a/xylem/specs/plugins/rules.py +++ b/xylem/specs/plugins/rules.py @@ -254,12 +254,13 @@ from ..impl import Spec -from ...util import load_yaml -from ...util import dump_yaml +from ...yaml_utils import load_yaml +from ...yaml_utils import dump_yaml from ...text_utils import text_type from ...text_utils import to_str from ...load_url import load_url from ...log_utils import error +from ...exception import XylemError DESCRIPTION = """\ @@ -331,7 +332,7 @@ def keys(self, data, installer_context): # functions/variables/parameters be # TODO: what is the right abstraction here? -class SpecParsingError(ValueError): +class SpecParsingError(XylemError): """Raised when an invalid spec element is encountered while parsing.""" diff --git a/xylem/text_utils.py b/xylem/text_utils.py index 3952be9..8b34058 100644 --- a/xylem/text_utils.py +++ b/xylem/text_utils.py @@ -82,3 +82,8 @@ def to_bytes(obj, encoding='utf-8', errors='replace'): if isinstance(value, six.text_type): value = value.encode(encoding, errors) return value + + +def type_name(obj): + """Return name of the type of ``obj``.""" + return to_str(type(obj)) diff --git a/xylem/update.py b/xylem/update.py index 876d5e7..37d4b07 100644 --- a/xylem/update.py +++ b/xylem/update.py @@ -14,18 +14,15 @@ # TODO: update docstrings -"""Implements the update functionality. - -This includes the functions to collect and process source files. Part of -this process is to load and run the spec parser, which are given by name -in the source files. -""" +"""Implements the update functionality.""" from __future__ import unicode_literals from .sources import SourcesContext from .sources import RulesDatabase +from .config import get_config + # TODO: remove handle_spec_urls (move some logic into the accoring # method in Rules database) @@ -61,21 +58,26 @@ # return rules_dict_list -def update(prefix=None, dry_run=False): +def update(dry_run=False, config=None, sources_context=None): """Update the xylem cache. If the prefix is set then the source lists are searched for in the prefix. If the prefix is not set (None) or the source lists are not found in the prefix, then the default, builtin source list is used. - :param prefix: The config and cache prefix, usually '/' or someother - prefix - :type prefix: :py:obj:`str` or :py:obj:`None` - :param dry_run: If True, then no actual action is taken, only + :param bool dry_run: if `True`, then no actual action is taken, only pretend to - :type dry_run: bool + :param config: config dict to create source context with; if `None` + is passed, use global configuration + :type config: `dict` or `None` + :param sources_context: the sources context to be used to + instantiate the rules database; if `None` is passed, a sources + context from ``config`` is created + :type sources_context: `SourcesContext` or `None` """ - sources_context = SourcesContext(prefix=prefix) + if config is None: + config = get_config() + sources_context = sources_context or SourcesContext(config) sources_context.ensure_cache_dir() database = RulesDatabase(sources_context) database.print_info = True diff --git a/xylem/util.py b/xylem/util.py index 465191f..7a75a98 100644 --- a/xylem/util.py +++ b/xylem/util.py @@ -17,21 +17,16 @@ from __future__ import print_function from __future__ import unicode_literals -import argparse import os import shutil import sys import tempfile import six -import yaml import subprocess from six import StringIO -from . import DEFAULT_PREFIX from .text_utils import to_str -from .log_utils import enable_debug, enable_verbose -from .terminal_color import disable_ANSI_colors class change_directory(object): @@ -100,47 +95,22 @@ def raise_from(exc_type, exc_args, from_exc): raise exc -def add_global_arguments(parser): - from xylem import __version__ - group = parser.add_argument_group('global', description="""\ -The XYLEM_PREFIX environment variable sets the path under which xylem -operates on source configurations and caches, which can be overwritten -by the --prefix argument. If set, the XYLEM_DEBUG environment variable -enables debug messages.""") - add = group.add_argument - add('-d', '--debug', help='enable debug messages', - action='store_true', default=False) - add('--pdb', help=argparse.SUPPRESS, - action='store_true', default=False) - add('--version', action='version', version=__version__, - help="prints the xylem version") - add('-v', '--verbose', action='store_true', default=False, - help="verbose console output") - add('--no-color', action='store_true', default=False, - dest='no_color', help=argparse.SUPPRESS) - add('-p', '--prefix', metavar='XYLEM_PREFIX', - default=os.environ.get('XYLEM_PREFIX', DEFAULT_PREFIX), - help="Sets the prefix for finding configs and caches. " - "The default is either '/' or, if set, the XYLEM_PREFIX " - "environment variable.") - return parser +# TODO: document this soft dependency on pygments, and also add unit +# test for printing exceptions with and without pygments + _pdb = False -def handle_global_arguments(args): +def enable_pdb(pdb=True): global _pdb - enable_debug(args.debug or 'XYLEM_DEBUG' in os.environ) - _pdb = args.pdb - args.prefix = os.path.expanduser(args.prefix) - if args.verbose: - enable_verbose() - if args.no_color: - disable_ANSI_colors() + _pdb = pdb -# TODO: document this soft dependency on pygments, and also add unit -# test for printing exceptions with and without pygments +def pdb_enabled(): + global _pdb + return _pdb + def print_exc(formated_exc): exc_str = ''.join(formated_exc) @@ -157,11 +127,10 @@ def print_exc(formated_exc): def custom_exception_handler(type, value, tb): - global _pdb # Print traceback import traceback print_exc(traceback.format_exception(type, value, tb)) - if not _pdb or hasattr(sys, 'ps1') or not sys.stderr.isatty(): + if not pdb_enabled() or hasattr(sys, 'ps1') or not sys.stderr.isatty(): pass else: # ...then start the debugger in post-mortem mode. @@ -173,8 +142,7 @@ def custom_exception_handler(type, value, tb): def pdb_hook(): - global _pdb - if _pdb: + if pdb_enabled(): import pdb pdb.set_trace() @@ -185,58 +153,6 @@ def create_temporary_directory(prefix_dir=None): return mkdtemp(prefix='xylem_', dir=prefix_dir) -# use this utility function throughout to make sure the custom -# constructors for unicode handling are loaded -def load_yaml(data): - """Parse a unicode string containing yaml. - - This calls ``yaml.load(data)`` but makes sure unicode is handled correctly. - - See :func:`yaml.load`. - - :raises yaml.YAMLError: if parsing fails""" - - class MyLoader(yaml.SafeLoader): - def construct_yaml_str(self, node): - # Override the default string handling function - # to always return unicode objects - return self.construct_scalar(node) - - MyLoader.add_constructor( - 'tag:yaml.org,2002:str', MyLoader.construct_yaml_str) - - return yaml.load(data, Loader=MyLoader) - - -def dump_yaml(data, inline=False): - """Dump data to unicode string.""" - - class MyDumper(yaml.SafeDumper): - def ignore_aliases(self, _data): - return True - - def represent_sequence(self, tag, data, flow_style=False): - # represent lists inline - return yaml.SafeDumper.represent_sequence( - self, tag, data, flow_style=True) - - def represent_none(self, data): - return self.represent_scalar('tag:yaml.org,2002:null', '') - - MyDumper.add_representer(type(None), MyDumper.represent_none) - - result = yaml.dump(data, - Dumper=MyDumper, - # TODO: use this for inline==True ?? - # default_style=None, - default_flow_style=False, - allow_unicode=True, - indent=2, - width=10000000) - - return result - - def read_stdout(cmd): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() diff --git a/xylem/yaml_utils.py b/xylem/yaml_utils.py new file mode 100644 index 0000000..66a55f7 --- /dev/null +++ b/xylem/yaml_utils.py @@ -0,0 +1,82 @@ +# Copyright 2014 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Customize YAML loading and dumping to our needs. + +Use these utility function throughout to make sure unicde is handled +correctly and output YAML looks consistent. +""" + + +from __future__ import unicode_literals + + +import yaml + + +def load_yaml(data): + """Parse a unicode string containing yaml. + + This calls ``yaml.load(data)`` but makes sure unicode is handled correctly. + + See :func:`yaml.load`. + + :raises yaml.YAMLError: if parsing fails""" + + class MyLoader(yaml.SafeLoader): + def construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + return self.construct_scalar(node) + + MyLoader.add_constructor( + 'tag:yaml.org,2002:str', MyLoader.construct_yaml_str) + + return yaml.load(data, Loader=MyLoader) + + +def dump_yaml(data, inline=False): + """Dump data to unicode string.""" + + # TODO: handle `inline` argument properly to produce output on a + # single line + + # delay import to prevent circular import + from .config_utils import ConfigDict + + class MyDumper(yaml.SafeDumper): + def ignore_aliases(self, _data): + return True + + def represent_sequence(self, tag, data, flow_style=False): + # represent lists inline + return yaml.SafeDumper.represent_sequence( + self, tag, data, flow_style=True) + + def represent_none(self, data): + return self.represent_scalar('tag:yaml.org,2002:null', '') + + MyDumper.add_representer(type(None), MyDumper.represent_none) + MyDumper.add_representer(ConfigDict, MyDumper.represent_dict) + + result = yaml.dump(data, + Dumper=MyDumper, + # TODO: use this for inline==True ?? + # default_style=None, + default_flow_style=False, + allow_unicode=True, + indent=2, + width=10000000) + + return result