From ae09cc38d1c7fbe0bd32c9f0540cc66aeeebf458 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Mon, 4 Oct 2021 15:56:54 -0400 Subject: [PATCH 1/3] Allow user to supply a c++ header file --- cmdstanpy/compiler_opts.py | 60 ++++++++-- cmdstanpy/model.py | 29 ++++- docsrc/examples.rst | 1 + docsrc/examples/.gitignore | 3 +- docsrc/examples/Using External C++.ipynb | 145 +++++++++++++++++++++++ docsrc/examples/bernoulli_external.stan | 18 +++ docsrc/examples/make_odds.hpp | 13 ++ test/data/.gitignore | 3 +- test/data/external.stan | 7 ++ test/data/return_one.hpp | 9 ++ test/test_compiler_opts.py | 35 +++++- test/test_model.py | 18 ++- test/test_optimize.py | 6 +- test/test_sample.py | 6 +- test/test_variational.py | 12 +- 15 files changed, 337 insertions(+), 28 deletions(-) create mode 100644 docsrc/examples/Using External C++.ipynb create mode 100644 docsrc/examples/bernoulli_external.stan create mode 100644 docsrc/examples/make_odds.hpp create mode 100644 test/data/external.stan create mode 100644 test/data/return_one.hpp diff --git a/cmdstanpy/compiler_opts.py b/cmdstanpy/compiler_opts.py index adefa0a1..a51f48bc 100644 --- a/cmdstanpy/compiler_opts.py +++ b/cmdstanpy/compiler_opts.py @@ -48,6 +48,7 @@ class CompilerOptions: Attributes: stanc_options - stanc compiler flags, options cpp_options - makefile options (NAME=value) + user_header - path to a user .hpp file to include during compilation """ def __init__( @@ -55,11 +56,13 @@ def __init__( *, stanc_options: Optional[Dict[str, Any]] = None, cpp_options: Optional[Dict[str, Any]] = None, + user_header: Optional[str] = None, logger: Optional[logging.Logger] = None, ) -> None: """Initialize object.""" self._stanc_options = stanc_options if stanc_options is not None else {} self._cpp_options = cpp_options if cpp_options is not None else {} + self._user_header = user_header if user_header is not None else '' if logger is not None: get_logger().warning( "Parameter 'logger' is deprecated." @@ -67,8 +70,8 @@ def __init__( ) def __repr__(self) -> str: - return 'stanc_options={}, cpp_options={}'.format( - self._stanc_options, self._cpp_options + return 'stanc_options={}, cpp_options={}, user_header={}'.format( + self._stanc_options, self._cpp_options, self._user_header ) @property @@ -81,6 +84,11 @@ def cpp_options(self) -> Dict[str, Union[bool, int]]: """C++ compiler options.""" return self._cpp_options + @property + def user_header(self) -> str: + """The user header file if it exists, otherwise empty""" + return self._user_header + def validate(self) -> None: """ Check compiler args. @@ -88,6 +96,7 @@ def validate(self) -> None: """ self.validate_stanc_opts() self.validate_cpp_opts() + self.validate_user_header() def validate_stanc_opts(self) -> None: """ @@ -104,9 +113,7 @@ def validate_stanc_opts(self) -> None: get_logger().info('ignoring compiler option: %s', key) ignore.append(key) elif key not in STANC_OPTS: - raise ValueError( - 'unknown stanc compiler option: {}'.format(key) - ) + raise ValueError(f'unknown stanc compiler option: {key}') elif key == 'include_paths': paths = val if isinstance(val, str): @@ -114,7 +121,7 @@ def validate_stanc_opts(self) -> None: elif not isinstance(val, list): raise ValueError( 'Invalid include_paths, expecting list or ' - 'string, found type: {}.'.format(type(val)) + f'string, found type: {type(val)}.' ) elif key == 'use-opencl': if self._cpp_options is None: @@ -149,10 +156,37 @@ def validate_cpp_opts(self) -> None: val = self._cpp_options[key] if not isinstance(val, int) or val < 0: raise ValueError( - '{} must be a non-negative integer value,' - ' found {}.'.format(key, val) + f'{key} must be a non-negative integer value,' + f' found {val}.' ) + def validate_user_header(self) -> None: + """ + User header exists. + Raise ValueError if bad config is found. + """ + if self._user_header != "": + if not ( + os.path.exists(self._user_header) + and os.path.isfile(self._user_header) + ): + raise ValueError( + f"User header file {self._user_header} cannot be found" + ) + if self._user_header[-4:] != '.hpp': + raise ValueError( + f"Header file must end in .hpp, got {self._user_header}" + ) + if "allow_undefined" not in self._stanc_options: + self._stanc_options["allow_undefined"] = True + # set full path + self._user_header = os.path.abspath(self._user_header) + + if ' ' in self._user_header: + raise ValueError( + "User header must be in a folder with no spaces in path!" + ) + def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000 """Adds options to existing set of compiler options.""" if new_opts.stanc_options is not None: @@ -167,6 +201,8 @@ def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000 if new_opts.cpp_options is not None: for key, val in new_opts.cpp_options.items(): self._cpp_options[key] = val + if new_opts.user_header != '' and self._user_header == '': + self._user_header = new_opts.user_header def add_include_path(self, path: str) -> None: """Adds include path to existing set of compiler options.""" @@ -191,10 +227,12 @@ def compose(self) -> List[str]: ) ) elif key == 'name': - opts.append('STANCFLAGS+=--{}={}'.format(key, val)) + opts.append(f'STANCFLAGS+=--name={val}') else: - opts.append('STANCFLAGS+=--{}'.format(key)) + opts.append(f'STANCFLAGS+=--{key}') if self._cpp_options is not None and len(self._cpp_options) > 0: for key, val in self._cpp_options.items(): - opts.append('{}={}'.format(key, val)) + opts.append(f'{key}={val}') + if self._user_header: + opts.append(f'USER_HEADER={self._user_header}') return opts diff --git a/cmdstanpy/model.py b/cmdstanpy/model.py index 360a4ddc..20c8f3cd 100644 --- a/cmdstanpy/model.py +++ b/cmdstanpy/model.py @@ -68,6 +68,10 @@ class CmdStanModel: :param cpp_options: Options for C++ compiler, specified as a Python dictionary containing C++ compiler option name, value pairs. Optional. + + :param user_header: A path to a header file to include during C++ + compilation. + Optional. """ def __init__( @@ -78,6 +82,7 @@ def __init__( compile: bool = True, stanc_options: Optional[Dict[str, Any]] = None, cpp_options: Optional[Dict[str, Any]] = None, + user_header: Optional[str] = None, logger: Optional[logging.Logger] = None, ) -> None: """ @@ -89,12 +94,16 @@ def __init__( :param compile: Whether or not to compile the model. :param stanc_options: Options for stanc compiler. :param cpp_options: Options for C++ compiler. + :param user_header: A path to a header file to include during C++ + compilation. """ self._name = '' self._stan_file = None self._exe_file = None self._compiler_options = CompilerOptions( - stanc_options=stanc_options, cpp_options=cpp_options + stanc_options=stanc_options, + cpp_options=cpp_options, + user_header=user_header, ) if logger is not None: get_logger().warning( @@ -227,6 +236,11 @@ def cpp_options(self) -> Dict[str, Union[bool, int]]: """Options to C++ compilers.""" return self._compiler_options._cpp_options + @property + def user_header(self) -> str: + """The user header file if it exists, otherwise empty""" + return self._compiler_options._user_header + def code(self) -> Optional[str]: """Return Stan program as a string.""" if not self._stan_file: @@ -247,6 +261,7 @@ def compile( force: bool = False, stanc_options: Optional[Dict[str, Any]] = None, cpp_options: Optional[Dict[str, Any]] = None, + user_header: Optional[str] = None, override_options: bool = False, ) -> None: """ @@ -264,6 +279,8 @@ def compile( :param stanc_options: Options for stanc compiler. :param cpp_options: Options for C++ compiler. + :param user_header: A path to a header file to include during C++ + compilation. :param override_options: When ``True``, override existing option. When ``False``, add/replace existing options. Default is ``False``. @@ -272,9 +289,15 @@ def compile( raise RuntimeError('Please specify source file') compiler_options = None - if not (stanc_options is None and cpp_options is None): + if not ( + stanc_options is None + and cpp_options is None + and user_header is None + ): compiler_options = CompilerOptions( - stanc_options=stanc_options, cpp_options=cpp_options + stanc_options=stanc_options, + cpp_options=cpp_options, + user_header=user_header, ) compiler_options.validate() if self._compiler_options is None: diff --git a/docsrc/examples.rst b/docsrc/examples.rst index 0c0a44c2..844ac9ba 100644 --- a/docsrc/examples.rst +++ b/docsrc/examples.rst @@ -8,3 +8,4 @@ __________________ examples/Maximum Likelihood Estimation.ipynb examples/Variational Inference.ipynb examples/Run Generated Quantities.ipynb + examples/Using External C++.ipynb diff --git a/docsrc/examples/.gitignore b/docsrc/examples/.gitignore index ae000631..831b0bf3 100644 --- a/docsrc/examples/.gitignore +++ b/docsrc/examples/.gitignore @@ -7,4 +7,5 @@ *.hpp *.exe *.csv -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +!make_odds.hpp \ No newline at end of file diff --git a/docsrc/examples/Using External C++.ipynb b/docsrc/examples/Using External C++.ipynb new file mode 100644 index 00000000..5e2b201f --- /dev/null +++ b/docsrc/examples/Using External C++.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Advanced Topic: Using External C++ Functions\n", + "\n", + "This is based on the relevant portion of the CmdStan documentation [here](https://mc-stan.org/docs/cmdstan-guide/using-external-cpp-code.html)" + ], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "Consider the following Stan model, based on the bernoulli example." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import os\n", + "from cmdstanpy import cmdstan_path, CmdStanModel\n", + "model_external = CmdStanModel(stan_file='bernoulli_external.stan', compile=False)\n", + "print(model_external.code())" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "As you can see, it features a function declaration for `make_odds`, but no definition. If we try to compile this, we will get an error. " + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "model_external.compile()" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "Even enabling the `--allow_undefined` flag to stanc3 will not allow this model to be compiled quite yet." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "model_external.compile(stanc_options={'allow_undefined':True})" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "To resolve this, we need to both tell the Stan compiler an undefined function is okay **and** let C++ know what it should be. \n", + "\n", + "We can provide a definition in a C++ header file by using the `user_header` argument to either the CmdStanModel constructor or the `compile` method. \n", + "\n", + "This will enables the `allow_undefined` flag automatically." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "model_external.compile(user_header='make_odds.hpp')" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "We can then run this model and inspect the output" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "fit = model_external.sample(data={'N':10, 'y':[0,1,0,0,0,0,0,0,0,1]})\n", + "fit.stan_variable('odds')" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "The contents of this header file are a bit complicated unless you are familiar with the C++ internals of Stan, so they are presented without comment:\n", + "\n", + "```c++\n", + "#include \n", + "#include \n", + "\n", + "namespace bernoulli_model_namespace {\n", + " template inline typename\n", + " boost::math::tools::promote_args::type \n", + " make_odds(const T0__& theta, std::ostream* pstream__) {\n", + " return theta / (1 - theta); \n", + " }\n", + "}\n", + "```" + ], + "metadata": {} + } + ], + "metadata": { + "orig_nbformat": 4, + "language_info": { + "name": "python", + "version": "3.9.5", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3.9.5 64-bit ('stan': conda)" + }, + "interpreter": { + "hash": "d31ce8e45781476cfd394e192e0962028add96ff436d4fd4e560a347d206b9cb" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/docsrc/examples/bernoulli_external.stan b/docsrc/examples/bernoulli_external.stan new file mode 100644 index 00000000..724a6ee1 --- /dev/null +++ b/docsrc/examples/bernoulli_external.stan @@ -0,0 +1,18 @@ +functions { + real make_odds(real theta); +} +data { + int N; + array[N] int y; +} +parameters { + real theta; +} +model { + theta ~ beta(1, 1); // uniform prior on interval 0, 1 + y ~ bernoulli(theta); +} +generated quantities { + real odds; + odds = make_odds(theta); +} \ No newline at end of file diff --git a/docsrc/examples/make_odds.hpp b/docsrc/examples/make_odds.hpp new file mode 100644 index 00000000..16b223c1 --- /dev/null +++ b/docsrc/examples/make_odds.hpp @@ -0,0 +1,13 @@ +#include +#include + +namespace bernoulli_external_model_namespace +{ + template + inline typename boost::math::tools::promote_args::type make_odds(const T0__ & + theta, + std::ostream *pstream__) + { + return theta / (1 - theta); + } +} \ No newline at end of file diff --git a/test/data/.gitignore b/test/data/.gitignore index 4955b2fc..d12b6c6c 100644 --- a/test/data/.gitignore +++ b/test/data/.gitignore @@ -5,4 +5,5 @@ !*.* # and we re-ignore hpp and exe files *.hpp -*.exe \ No newline at end of file +*.exe +!return_one.hpp \ No newline at end of file diff --git a/test/data/external.stan b/test/data/external.stan new file mode 100644 index 00000000..7a399c8c --- /dev/null +++ b/test/data/external.stan @@ -0,0 +1,7 @@ +functions { + real return_one(); +} + +generated quantities { + real one = return_one(); +} \ No newline at end of file diff --git a/test/data/return_one.hpp b/test/data/return_one.hpp new file mode 100644 index 00000000..6caa1a3f --- /dev/null +++ b/test/data/return_one.hpp @@ -0,0 +1,9 @@ +#include +namespace external_model_namespace +{ + double return_one( + std::ostream *pstream__) + { + return 1.0; + } +} \ No newline at end of file diff --git a/test/test_compiler_opts.py b/test/test_compiler_opts.py index 0517d7bf..4d9f97d4 100644 --- a/test/test_compiler_opts.py +++ b/test/test_compiler_opts.py @@ -14,7 +14,9 @@ def test_opts_empty(self): opts = CompilerOptions() opts.validate() self.assertEqual(opts.compose(), []) - self.assertEqual(opts.__repr__(), 'stanc_options={}, cpp_options={}') + self.assertEqual( + opts.__repr__(), 'stanc_options={}, cpp_options={}, user_header=' + ) stanc_opts = {} opts = CompilerOptions(stanc_options=stanc_opts) @@ -29,7 +31,9 @@ def test_opts_empty(self): opts = CompilerOptions(stanc_options=stanc_opts, cpp_options=cpp_opts) opts.validate() self.assertEqual(opts.compose(), []) - self.assertEqual(opts.__repr__(), 'stanc_options={}, cpp_options={}') + self.assertEqual( + opts.__repr__(), 'stanc_options={}, cpp_options={}, user_header=' + ) def test_opts_stanc(self): stanc_opts = {} @@ -149,6 +153,27 @@ def test_opts_cpp_opencl(self): with self.assertRaises(ValueError): opts.validate() + def test_user_header(self): + header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp') + opts = CompilerOptions(user_header=header_file) + opts.validate() + self.assertTrue(opts.stanc_options['allow_undefined']) + + bad = os.path.join(DATAFILES_PATH, 'nonexistant.hpp') + opts = CompilerOptions(user_header=bad) + with self.assertRaisesRegex(ValueError, "cannot be found"): + opts.validate() + + bad_dir = os.path.join(DATAFILES_PATH, 'optimize') + opts = CompilerOptions(user_header=bad_dir) + with self.assertRaisesRegex(ValueError, "cannot be found"): + opts.validate() + + non_header = os.path.join(DATAFILES_PATH, 'bernoulli.stan') + opts = CompilerOptions(user_header=non_header) + with self.assertRaisesRegex(ValueError, "must end in .hpp"): + opts.validate() + def test_opts_add(self): stanc_opts = {'warn-uninitialized': True} cpp_opts = {'STAN_OPENCL': 'TRUE', 'OPENCL_DEVICE_ID': 1} @@ -184,6 +209,12 @@ def test_opts_add(self): opts_list = opts.compose() self.assertTrue(expect in opts_list) + header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp') + opts = CompilerOptions() + opts.add(CompilerOptions(user_header=header_file)) + opts_list = opts.compose() + self.assertTrue(len(opts_list) == 1) + if __name__ == '__main__': unittest.main() diff --git a/test/test_model.py b/test/test_model.py index 86c90309..93035492 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -44,11 +44,15 @@ def do_clean_up(self): for root, _, files in os.walk(DATAFILES_PATH): for filename in files: _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" + if ( + ext.lower() in ('.o', '.d', '.hpp', '.exe', '') + and filename != ".gitignore" + and filename != "return_one.hpp" ): filepath = os.path.join(root, filename) os.remove(filepath) + # we should really make this module-level + # and use something like git clean -Xf data/ def show_cmdstan_version(self): print('\n\nCmdStan version: {}\n\n'.format(cmdstan_path())) @@ -61,6 +65,12 @@ def test_model_good(self): self.assertTrue(model.exe_file.endswith(BERN_EXE.replace('\\', '/'))) self.assertEqual('bern', model.name) + # compile with external header + model = CmdStanModel( + stan_file=os.path.join(DATAFILES_PATH, "external.stan"), + user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp'), + ) + # default model name model = CmdStanModel(stan_file=BERN_STAN) self.assertEqual(BERN_BASENAME, model.name) @@ -109,6 +119,10 @@ def test_model_bad(self): CmdStanModel(model_name='', stan_file=BERN_STAN) with self.assertRaises(ValueError): CmdStanModel(model_name=' ', stan_file=BERN_STAN) + with self.assertRaises(ValueError): + CmdStanModel( + stan_file=os.path.join(DATAFILES_PATH, "external.stan") + ) def test_stanc_options(self): opts = { diff --git a/test/test_optimize.py b/test/test_optimize.py index 87dcc88b..56ee7d70 100644 --- a/test/test_optimize.py +++ b/test/test_optimize.py @@ -25,8 +25,10 @@ def do_clean_up(self): for root, _, files in os.walk(DATAFILES_PATH): for filename in files: _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" + if ( + ext.lower() in ('.o', '.d', '.hpp', '.exe', '') + and filename != ".gitignore" + and filename != "return_one.hpp" ): filepath = os.path.join(root, filename) os.remove(filepath) diff --git a/test/test_sample.py b/test/test_sample.py index c1e85bc4..a0a3f1c5 100644 --- a/test/test_sample.py +++ b/test/test_sample.py @@ -63,8 +63,10 @@ def do_clean_up(self): for root, _, files in os.walk(DATAFILES_PATH): for filename in files: _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" + if ( + ext.lower() in ('.o', '.d', '.hpp', '.exe', '') + and filename != ".gitignore" + and filename != "return_one.hpp" ): filepath = os.path.join(root, filename) os.remove(filepath) diff --git a/test/test_variational.py b/test/test_variational.py index d11c0ab2..d68ca880 100644 --- a/test/test_variational.py +++ b/test/test_variational.py @@ -25,8 +25,10 @@ def do_clean_up(self): ): for filename in files: _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" + if ( + ext.lower() in ('.o', '.d', '.hpp', '.exe', '') + and filename != ".gitignore" + and filename != "return_one.hpp" ): filepath = os.path.join(root, filename) os.remove(filepath) @@ -162,8 +164,10 @@ def do_clean_up(self): ): for filename in files: _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" + if ( + ext.lower() in ('.o', '.d', '.hpp', '.exe', '') + and filename != ".gitignore" + and filename != "return_one.hpp" ): filepath = os.path.join(root, filename) os.remove(filepath) From 3b7d48283c7aa76690431290c89daa2a079ac908 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Mon, 4 Oct 2021 16:13:41 -0400 Subject: [PATCH 2/3] Simplify test cleanup --- test/conftest.py | 19 +++++++++++++++++++ test/test_model.py | 19 +------------------ test/test_optimize.py | 16 ---------------- test/test_sample.py | 16 ---------------- test/test_variational.py | 35 ----------------------------------- 5 files changed, 20 insertions(+), 85 deletions(-) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..bc33cb5b --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,19 @@ +"""The global configuration for the test suite""" +import os +import subprocess + +import pytest + +HERE = os.path.dirname(os.path.abspath(__file__)) +DATAFILES_PATH = os.path.join(HERE, 'data') + + +# after we have run all tests, use git to delete the built files in data/ +@pytest.fixture(scope='session', autouse=True) +def cleanup_test_files(): + yield + subprocess.Popen( + ['git', 'clean', '-fX', DATAFILES_PATH], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) diff --git a/test/test_model.py b/test/test_model.py index 93035492..dc71587e 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -8,7 +8,6 @@ from unittest.mock import Mock import numpy as np -import pytest import tqdm from testfixtures import LogCapture @@ -37,23 +36,6 @@ class CmdStanModelTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk(DATAFILES_PATH): - for filename in files: - _, ext = os.path.splitext(filename) - if ( - ext.lower() in ('.o', '.d', '.hpp', '.exe', '') - and filename != ".gitignore" - and filename != "return_one.hpp" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - # we should really make this module-level - # and use something like git clean -Xf data/ - def show_cmdstan_version(self): print('\n\nCmdStan version: {}\n\n'.format(cmdstan_path())) self.assertTrue(True) @@ -94,6 +76,7 @@ def test_model_good(self): self.assertEqual(BERN_STAN, model.stan_file) self.assertEqual(None, model.exe_file) + # pylint: disable=no-self-use def test_model_pedantic(self): with LogCapture() as log: logging.getLogger() diff --git a/test/test_optimize.py b/test/test_optimize.py index 56ee7d70..1068bf8b 100644 --- a/test/test_optimize.py +++ b/test/test_optimize.py @@ -5,7 +5,6 @@ import unittest import numpy as np -import pytest from testfixtures import LogCapture from cmdstanpy.cmdstan_args import CmdStanArgs, OptimizeArgs @@ -18,21 +17,6 @@ class CmdStanMLETest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk(DATAFILES_PATH): - for filename in files: - _, ext = os.path.splitext(filename) - if ( - ext.lower() in ('.o', '.d', '.hpp', '.exe', '') - and filename != ".gitignore" - and filename != "return_one.hpp" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_instantiate(self): stan = os.path.join(DATAFILES_PATH, 'optimize', 'rosenbrock.stan') model = CmdStanModel(stan_file=stan) diff --git a/test/test_sample.py b/test/test_sample.py index a0a3f1c5..a6afd011 100644 --- a/test/test_sample.py +++ b/test/test_sample.py @@ -13,7 +13,6 @@ from time import time import numpy as np -import pytest from testfixtures import LogCapture try: @@ -56,21 +55,6 @@ def without_import(library, module): class SampleTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk(DATAFILES_PATH): - for filename in files: - _, ext = os.path.splitext(filename) - if ( - ext.lower() in ('.o', '.d', '.hpp', '.exe', '') - and filename != ".gitignore" - and filename != "return_one.hpp" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_bernoulli_good(self, stanfile='bernoulli.stan'): stan = os.path.join(DATAFILES_PATH, stanfile) bern_model = CmdStanModel(stan_file=stan) diff --git a/test/test_variational.py b/test/test_variational.py index d68ca880..4f6302a9 100644 --- a/test/test_variational.py +++ b/test/test_variational.py @@ -4,7 +4,6 @@ import unittest from math import fabs -import pytest from testfixtures import LogCapture from cmdstanpy.cmdstan_args import CmdStanArgs, VariationalArgs @@ -16,23 +15,6 @@ class CmdStanVBTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk( - os.path.join(DATAFILES_PATH, 'variational') - ): - for filename in files: - _, ext = os.path.splitext(filename) - if ( - ext.lower() in ('.o', '.d', '.hpp', '.exe', '') - and filename != ".gitignore" - and filename != "return_one.hpp" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_instantiate(self): stan = os.path.join( DATAFILES_PATH, 'variational', 'eta_should_be_big.stan' @@ -155,23 +137,6 @@ def test_variables_3d(self): class VariationalTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk( - os.path.join(DATAFILES_PATH, 'variational') - ): - for filename in files: - _, ext = os.path.splitext(filename) - if ( - ext.lower() in ('.o', '.d', '.hpp', '.exe', '') - and filename != ".gitignore" - and filename != "return_one.hpp" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_variational_good(self): stan = os.path.join( DATAFILES_PATH, 'variational', 'eta_should_be_big.stan' From d90af13e88caa6ecc1cdea7e7b5c9b9d198abb76 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Mon, 11 Oct 2021 10:05:17 -0400 Subject: [PATCH 3/3] Use cpp-opts internally for user header, check clashes --- cmdstanpy/compiler_opts.py | 28 ++++--- docsrc/env.yml | 1 + docsrc/examples/Using External C++.ipynb | 100 +++++++++++++---------- test/test_compiler_opts.py | 20 +++-- 4 files changed, 86 insertions(+), 63 deletions(-) diff --git a/cmdstanpy/compiler_opts.py b/cmdstanpy/compiler_opts.py index a51f48bc..1db853a4 100644 --- a/cmdstanpy/compiler_opts.py +++ b/cmdstanpy/compiler_opts.py @@ -70,8 +70,8 @@ def __init__( ) def __repr__(self) -> str: - return 'stanc_options={}, cpp_options={}, user_header={}'.format( - self._stanc_options, self._cpp_options, self._user_header + return 'stanc_options={}, cpp_options={}'.format( + self._stanc_options, self._cpp_options ) @property @@ -84,11 +84,6 @@ def cpp_options(self) -> Dict[str, Union[bool, int]]: """C++ compiler options.""" return self._cpp_options - @property - def user_header(self) -> str: - """The user header file if it exists, otherwise empty""" - return self._user_header - def validate(self) -> None: """ Check compiler args. @@ -184,9 +179,20 @@ def validate_user_header(self) -> None: if ' ' in self._user_header: raise ValueError( - "User header must be in a folder with no spaces in path!" + "User header must be in a location with no spaces in path!" ) + if ( + 'USER_HEADER' in self._cpp_options + and self._user_header != self._cpp_options['USER_HEADER'] + ): + raise ValueError( + "Disagreement in user_header C++ options found!\n" + f"{self._user_header}, {self._cpp_options['USER_HEADER']}" + ) + + self._cpp_options['USER_HEADER'] = self._user_header + def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000 """Adds options to existing set of compiler options.""" if new_opts.stanc_options is not None: @@ -201,8 +207,8 @@ def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000 if new_opts.cpp_options is not None: for key, val in new_opts.cpp_options.items(): self._cpp_options[key] = val - if new_opts.user_header != '' and self._user_header == '': - self._user_header = new_opts.user_header + if new_opts._user_header != '' and self._user_header == '': + self._user_header = new_opts._user_header def add_include_path(self, path: str) -> None: """Adds include path to existing set of compiler options.""" @@ -233,6 +239,4 @@ def compose(self) -> List[str]: if self._cpp_options is not None and len(self._cpp_options) > 0: for key, val in self._cpp_options.items(): opts.append(f'{key}={val}') - if self._user_header: - opts.append(f'USER_HEADER={self._user_header}') return opts diff --git a/docsrc/env.yml b/docsrc/env.yml index 444c9d61..774fc66c 100644 --- a/docsrc/env.yml +++ b/docsrc/env.yml @@ -6,6 +6,7 @@ dependencies: - python=3.7 - ipykernel - ipython + - ipywidgets - numpy>=1.15 - pandas - xarray diff --git a/docsrc/examples/Using External C++.ipynb b/docsrc/examples/Using External C++.ipynb index 5e2b201f..c83a1997 100644 --- a/docsrc/examples/Using External C++.ipynb +++ b/docsrc/examples/Using External C++.ipynb @@ -2,103 +2,116 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ "# Advanced Topic: Using External C++ Functions\n", "\n", "This is based on the relevant portion of the CmdStan documentation [here](https://mc-stan.org/docs/cmdstan-guide/using-external-cpp-code.html)" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Consider the following Stan model, based on the bernoulli example." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {"nbsphinx": "hidden"}, + "outputs": [], "source": [ "import os\n", - "from cmdstanpy import cmdstan_path, CmdStanModel\n", + "try:\n", + " os.remove('bernoulli_external')\n", + "except:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from cmdstanpy import CmdStanModel\n", "model_external = CmdStanModel(stan_file='bernoulli_external.stan', compile=False)\n", "print(model_external.code())" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "As you can see, it features a function declaration for `make_odds`, but no definition. If we try to compile this, we will get an error. " - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "model_external.compile()" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Even enabling the `--allow_undefined` flag to stanc3 will not allow this model to be compiled quite yet." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "model_external.compile(stanc_options={'allow_undefined':True})" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "To resolve this, we need to both tell the Stan compiler an undefined function is okay **and** let C++ know what it should be. \n", "\n", "We can provide a definition in a C++ header file by using the `user_header` argument to either the CmdStanModel constructor or the `compile` method. \n", "\n", "This will enables the `allow_undefined` flag automatically." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "model_external.compile(user_header='make_odds.hpp')" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "We can then run this model and inspect the output" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "fit = model_external.sample(data={'N':10, 'y':[0,1,0,0,0,0,0,0,0,1]})\n", "fit.stan_variable('odds')" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "The contents of this header file are a bit complicated unless you are familiar with the C++ internals of Stan, so they are presented without comment:\n", "\n", @@ -114,32 +127,31 @@ " }\n", "}\n", "```" - ], - "metadata": {} + ] } ], "metadata": { - "orig_nbformat": 4, + "interpreter": { + "hash": "8765ce46b013071999fc1966b52035a7309a0da7551e066cc0f0fa23e83d4f60" + }, + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('stan': conda)", + "name": "python3" + }, "language_info": { - "name": "python", - "version": "3.9.5", - "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, - "pygments_lexer": "ipython3", + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", "nbconvert_exporter": "python", - "file_extension": ".py" - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3.9.5 64-bit ('stan': conda)" + "pygments_lexer": "ipython3", + "version": "3.9.5" }, - "interpreter": { - "hash": "d31ce8e45781476cfd394e192e0962028add96ff436d4fd4e560a347d206b9cb" - } + "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/test/test_compiler_opts.py b/test/test_compiler_opts.py index 4d9f97d4..1f535e5c 100644 --- a/test/test_compiler_opts.py +++ b/test/test_compiler_opts.py @@ -14,9 +14,7 @@ def test_opts_empty(self): opts = CompilerOptions() opts.validate() self.assertEqual(opts.compose(), []) - self.assertEqual( - opts.__repr__(), 'stanc_options={}, cpp_options={}, user_header=' - ) + self.assertEqual(opts.__repr__(), 'stanc_options={}, cpp_options={}') stanc_opts = {} opts = CompilerOptions(stanc_options=stanc_opts) @@ -31,9 +29,7 @@ def test_opts_empty(self): opts = CompilerOptions(stanc_options=stanc_opts, cpp_options=cpp_opts) opts.validate() self.assertEqual(opts.compose(), []) - self.assertEqual( - opts.__repr__(), 'stanc_options={}, cpp_options={}, user_header=' - ) + self.assertEqual(opts.__repr__(), 'stanc_options={}, cpp_options={}') def test_opts_stanc(self): stanc_opts = {} @@ -174,6 +170,13 @@ def test_user_header(self): with self.assertRaisesRegex(ValueError, "must end in .hpp"): opts.validate() + header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp') + opts = CompilerOptions( + user_header=header_file, cpp_options={'USER_HEADER': 'foo'} + ) + with self.assertRaisesRegex(ValueError, "Disagreement"): + opts.validate() + def test_opts_add(self): stanc_opts = {'warn-uninitialized': True} cpp_opts = {'STAN_OPENCL': 'TRUE', 'OPENCL_DEVICE_ID': 1} @@ -213,7 +216,10 @@ def test_opts_add(self): opts = CompilerOptions() opts.add(CompilerOptions(user_header=header_file)) opts_list = opts.compose() - self.assertTrue(len(opts_list) == 1) + self.assertEqual(len(opts_list), 0) + opts.validate() + opts_list = opts.compose() + self.assertEqual(len(opts_list), 2) if __name__ == '__main__':