diff --git a/cmdstanpy/compiler_opts.py b/cmdstanpy/compiler_opts.py index adefa0a1..1db853a4 100644 --- a/cmdstanpy/compiler_opts.py +++ b/cmdstanpy/compiler_opts.py @@ -48,6 +48,7 @@ class CompilerOptions: Attributes: stanc_options - stanc compiler flags, options cpp_options - makefile options (NAME=value) + user_header - path to a user .hpp file to include during compilation """ def __init__( @@ -55,11 +56,13 @@ def __init__( *, stanc_options: Optional[Dict[str, Any]] = None, cpp_options: Optional[Dict[str, Any]] = None, + user_header: Optional[str] = None, logger: Optional[logging.Logger] = None, ) -> None: """Initialize object.""" self._stanc_options = stanc_options if stanc_options is not None else {} self._cpp_options = cpp_options if cpp_options is not None else {} + self._user_header = user_header if user_header is not None else '' if logger is not None: get_logger().warning( "Parameter 'logger' is deprecated." @@ -88,6 +91,7 @@ def validate(self) -> None: """ self.validate_stanc_opts() self.validate_cpp_opts() + self.validate_user_header() def validate_stanc_opts(self) -> None: """ @@ -104,9 +108,7 @@ def validate_stanc_opts(self) -> None: get_logger().info('ignoring compiler option: %s', key) ignore.append(key) elif key not in STANC_OPTS: - raise ValueError( - 'unknown stanc compiler option: {}'.format(key) - ) + raise ValueError(f'unknown stanc compiler option: {key}') elif key == 'include_paths': paths = val if isinstance(val, str): @@ -114,7 +116,7 @@ def validate_stanc_opts(self) -> None: elif not isinstance(val, list): raise ValueError( 'Invalid include_paths, expecting list or ' - 'string, found type: {}.'.format(type(val)) + f'string, found type: {type(val)}.' ) elif key == 'use-opencl': if self._cpp_options is None: @@ -149,10 +151,48 @@ def validate_cpp_opts(self) -> None: val = self._cpp_options[key] if not isinstance(val, int) or val < 0: raise ValueError( - '{} must be a non-negative integer value,' - ' found {}.'.format(key, val) + f'{key} must be a non-negative integer value,' + f' found {val}.' ) + def validate_user_header(self) -> None: + """ + User header exists. + Raise ValueError if bad config is found. + """ + if self._user_header != "": + if not ( + os.path.exists(self._user_header) + and os.path.isfile(self._user_header) + ): + raise ValueError( + f"User header file {self._user_header} cannot be found" + ) + if self._user_header[-4:] != '.hpp': + raise ValueError( + f"Header file must end in .hpp, got {self._user_header}" + ) + if "allow_undefined" not in self._stanc_options: + self._stanc_options["allow_undefined"] = True + # set full path + self._user_header = os.path.abspath(self._user_header) + + if ' ' in self._user_header: + raise ValueError( + "User header must be in a location with no spaces in path!" + ) + + if ( + 'USER_HEADER' in self._cpp_options + and self._user_header != self._cpp_options['USER_HEADER'] + ): + raise ValueError( + "Disagreement in user_header C++ options found!\n" + f"{self._user_header}, {self._cpp_options['USER_HEADER']}" + ) + + self._cpp_options['USER_HEADER'] = self._user_header + def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000 """Adds options to existing set of compiler options.""" if new_opts.stanc_options is not None: @@ -167,6 +207,8 @@ def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000 if new_opts.cpp_options is not None: for key, val in new_opts.cpp_options.items(): self._cpp_options[key] = val + if new_opts._user_header != '' and self._user_header == '': + self._user_header = new_opts._user_header def add_include_path(self, path: str) -> None: """Adds include path to existing set of compiler options.""" @@ -191,10 +233,10 @@ def compose(self) -> List[str]: ) ) elif key == 'name': - opts.append('STANCFLAGS+=--{}={}'.format(key, val)) + opts.append(f'STANCFLAGS+=--name={val}') else: - opts.append('STANCFLAGS+=--{}'.format(key)) + opts.append(f'STANCFLAGS+=--{key}') if self._cpp_options is not None and len(self._cpp_options) > 0: for key, val in self._cpp_options.items(): - opts.append('{}={}'.format(key, val)) + opts.append(f'{key}={val}') return opts diff --git a/cmdstanpy/model.py b/cmdstanpy/model.py index 1188b3af..649dd0f1 100644 --- a/cmdstanpy/model.py +++ b/cmdstanpy/model.py @@ -76,6 +76,10 @@ class CmdStanModel: :param cpp_options: Options for C++ compiler, specified as a Python dictionary containing C++ compiler option name, value pairs. Optional. + + :param user_header: A path to a header file to include during C++ + compilation. + Optional. """ def __init__( @@ -86,6 +90,7 @@ def __init__( compile: bool = True, stanc_options: Optional[Dict[str, Any]] = None, cpp_options: Optional[Dict[str, Any]] = None, + user_header: Optional[str] = None, logger: Optional[logging.Logger] = None, ) -> None: """ @@ -97,12 +102,16 @@ def __init__( :param compile: Whether or not to compile the model. :param stanc_options: Options for stanc compiler. :param cpp_options: Options for C++ compiler. + :param user_header: A path to a header file to include during C++ + compilation. """ self._name = '' self._stan_file = None self._exe_file = None self._compiler_options = CompilerOptions( - stanc_options=stanc_options, cpp_options=cpp_options + stanc_options=stanc_options, + cpp_options=cpp_options, + user_header=user_header, ) if logger is not None: get_logger().warning( @@ -235,6 +244,11 @@ def cpp_options(self) -> Dict[str, Union[bool, int]]: """Options to C++ compilers.""" return self._compiler_options._cpp_options + @property + def user_header(self) -> str: + """The user header file if it exists, otherwise empty""" + return self._compiler_options._user_header + def code(self) -> Optional[str]: """Return Stan program as a string.""" if not self._stan_file: @@ -255,6 +269,7 @@ def compile( force: bool = False, stanc_options: Optional[Dict[str, Any]] = None, cpp_options: Optional[Dict[str, Any]] = None, + user_header: Optional[str] = None, override_options: bool = False, ) -> None: """ @@ -272,6 +287,8 @@ def compile( :param stanc_options: Options for stanc compiler. :param cpp_options: Options for C++ compiler. + :param user_header: A path to a header file to include during C++ + compilation. :param override_options: When ``True``, override existing option. When ``False``, add/replace existing options. Default is ``False``. @@ -280,9 +297,15 @@ def compile( raise RuntimeError('Please specify source file') compiler_options = None - if not (stanc_options is None and cpp_options is None): + if not ( + stanc_options is None + and cpp_options is None + and user_header is None + ): compiler_options = CompilerOptions( - stanc_options=stanc_options, cpp_options=cpp_options + stanc_options=stanc_options, + cpp_options=cpp_options, + user_header=user_header, ) compiler_options.validate() if self._compiler_options is None: diff --git a/docsrc/env.yml b/docsrc/env.yml index 444c9d61..774fc66c 100644 --- a/docsrc/env.yml +++ b/docsrc/env.yml @@ -6,6 +6,7 @@ dependencies: - python=3.7 - ipykernel - ipython + - ipywidgets - numpy>=1.15 - pandas - xarray diff --git a/docsrc/examples.rst b/docsrc/examples.rst index 0c0a44c2..844ac9ba 100644 --- a/docsrc/examples.rst +++ b/docsrc/examples.rst @@ -8,3 +8,4 @@ __________________ examples/Maximum Likelihood Estimation.ipynb examples/Variational Inference.ipynb examples/Run Generated Quantities.ipynb + examples/Using External C++.ipynb diff --git a/docsrc/examples/.gitignore b/docsrc/examples/.gitignore index ae000631..831b0bf3 100644 --- a/docsrc/examples/.gitignore +++ b/docsrc/examples/.gitignore @@ -7,4 +7,5 @@ *.hpp *.exe *.csv -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +!make_odds.hpp \ No newline at end of file diff --git a/docsrc/examples/Using External C++.ipynb b/docsrc/examples/Using External C++.ipynb new file mode 100644 index 00000000..c83a1997 --- /dev/null +++ b/docsrc/examples/Using External C++.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced Topic: Using External C++ Functions\n", + "\n", + "This is based on the relevant portion of the CmdStan documentation [here](https://mc-stan.org/docs/cmdstan-guide/using-external-cpp-code.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Consider the following Stan model, based on the bernoulli example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"nbsphinx": "hidden"}, + "outputs": [], + "source": [ + "import os\n", + "try:\n", + " os.remove('bernoulli_external')\n", + "except:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from cmdstanpy import CmdStanModel\n", + "model_external = CmdStanModel(stan_file='bernoulli_external.stan', compile=False)\n", + "print(model_external.code())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, it features a function declaration for `make_odds`, but no definition. If we try to compile this, we will get an error. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_external.compile()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Even enabling the `--allow_undefined` flag to stanc3 will not allow this model to be compiled quite yet." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_external.compile(stanc_options={'allow_undefined':True})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To resolve this, we need to both tell the Stan compiler an undefined function is okay **and** let C++ know what it should be. \n", + "\n", + "We can provide a definition in a C++ header file by using the `user_header` argument to either the CmdStanModel constructor or the `compile` method. \n", + "\n", + "This will enables the `allow_undefined` flag automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_external.compile(user_header='make_odds.hpp')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then run this model and inspect the output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fit = model_external.sample(data={'N':10, 'y':[0,1,0,0,0,0,0,0,0,1]})\n", + "fit.stan_variable('odds')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The contents of this header file are a bit complicated unless you are familiar with the C++ internals of Stan, so they are presented without comment:\n", + "\n", + "```c++\n", + "#include \n", + "#include \n", + "\n", + "namespace bernoulli_model_namespace {\n", + " template inline typename\n", + " boost::math::tools::promote_args::type \n", + " make_odds(const T0__& theta, std::ostream* pstream__) {\n", + " return theta / (1 - theta); \n", + " }\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "8765ce46b013071999fc1966b52035a7309a0da7551e066cc0f0fa23e83d4f60" + }, + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('stan': conda)", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docsrc/examples/bernoulli_external.stan b/docsrc/examples/bernoulli_external.stan new file mode 100644 index 00000000..724a6ee1 --- /dev/null +++ b/docsrc/examples/bernoulli_external.stan @@ -0,0 +1,18 @@ +functions { + real make_odds(real theta); +} +data { + int N; + array[N] int y; +} +parameters { + real theta; +} +model { + theta ~ beta(1, 1); // uniform prior on interval 0, 1 + y ~ bernoulli(theta); +} +generated quantities { + real odds; + odds = make_odds(theta); +} \ No newline at end of file diff --git a/docsrc/examples/make_odds.hpp b/docsrc/examples/make_odds.hpp new file mode 100644 index 00000000..16b223c1 --- /dev/null +++ b/docsrc/examples/make_odds.hpp @@ -0,0 +1,13 @@ +#include +#include + +namespace bernoulli_external_model_namespace +{ + template + inline typename boost::math::tools::promote_args::type make_odds(const T0__ & + theta, + std::ostream *pstream__) + { + return theta / (1 - theta); + } +} \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..bc33cb5b --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,19 @@ +"""The global configuration for the test suite""" +import os +import subprocess + +import pytest + +HERE = os.path.dirname(os.path.abspath(__file__)) +DATAFILES_PATH = os.path.join(HERE, 'data') + + +# after we have run all tests, use git to delete the built files in data/ +@pytest.fixture(scope='session', autouse=True) +def cleanup_test_files(): + yield + subprocess.Popen( + ['git', 'clean', '-fX', DATAFILES_PATH], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) diff --git a/test/data/.gitignore b/test/data/.gitignore index 4955b2fc..d12b6c6c 100644 --- a/test/data/.gitignore +++ b/test/data/.gitignore @@ -5,4 +5,5 @@ !*.* # and we re-ignore hpp and exe files *.hpp -*.exe \ No newline at end of file +*.exe +!return_one.hpp \ No newline at end of file diff --git a/test/data/external.stan b/test/data/external.stan new file mode 100644 index 00000000..7a399c8c --- /dev/null +++ b/test/data/external.stan @@ -0,0 +1,7 @@ +functions { + real return_one(); +} + +generated quantities { + real one = return_one(); +} \ No newline at end of file diff --git a/test/data/return_one.hpp b/test/data/return_one.hpp new file mode 100644 index 00000000..6caa1a3f --- /dev/null +++ b/test/data/return_one.hpp @@ -0,0 +1,9 @@ +#include +namespace external_model_namespace +{ + double return_one( + std::ostream *pstream__) + { + return 1.0; + } +} \ No newline at end of file diff --git a/test/test_compiler_opts.py b/test/test_compiler_opts.py index 0517d7bf..1f535e5c 100644 --- a/test/test_compiler_opts.py +++ b/test/test_compiler_opts.py @@ -149,6 +149,34 @@ def test_opts_cpp_opencl(self): with self.assertRaises(ValueError): opts.validate() + def test_user_header(self): + header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp') + opts = CompilerOptions(user_header=header_file) + opts.validate() + self.assertTrue(opts.stanc_options['allow_undefined']) + + bad = os.path.join(DATAFILES_PATH, 'nonexistant.hpp') + opts = CompilerOptions(user_header=bad) + with self.assertRaisesRegex(ValueError, "cannot be found"): + opts.validate() + + bad_dir = os.path.join(DATAFILES_PATH, 'optimize') + opts = CompilerOptions(user_header=bad_dir) + with self.assertRaisesRegex(ValueError, "cannot be found"): + opts.validate() + + non_header = os.path.join(DATAFILES_PATH, 'bernoulli.stan') + opts = CompilerOptions(user_header=non_header) + with self.assertRaisesRegex(ValueError, "must end in .hpp"): + opts.validate() + + header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp') + opts = CompilerOptions( + user_header=header_file, cpp_options={'USER_HEADER': 'foo'} + ) + with self.assertRaisesRegex(ValueError, "Disagreement"): + opts.validate() + def test_opts_add(self): stanc_opts = {'warn-uninitialized': True} cpp_opts = {'STAN_OPENCL': 'TRUE', 'OPENCL_DEVICE_ID': 1} @@ -184,6 +212,15 @@ def test_opts_add(self): opts_list = opts.compose() self.assertTrue(expect in opts_list) + header_file = os.path.join(DATAFILES_PATH, 'return_one.hpp') + opts = CompilerOptions() + opts.add(CompilerOptions(user_header=header_file)) + opts_list = opts.compose() + self.assertEqual(len(opts_list), 0) + opts.validate() + opts_list = opts.compose() + self.assertEqual(len(opts_list), 2) + if __name__ == '__main__': unittest.main() diff --git a/test/test_model.py b/test/test_model.py index 0c2f6545..b428dbaa 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -6,7 +6,6 @@ import tempfile import unittest -import pytest from testfixtures import LogCapture, StringComparison from cmdstanpy.model import CmdStanModel @@ -35,19 +34,6 @@ class CmdStanModelTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk(DATAFILES_PATH): - for filename in files: - _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_model_good(self): # compile on instantiation, override model name model = CmdStanModel(model_name='bern', stan_file=BERN_STAN) @@ -55,6 +41,12 @@ def test_model_good(self): self.assertTrue(model.exe_file.endswith(BERN_EXE.replace('\\', '/'))) self.assertEqual('bern', model.name) + # compile with external header + model = CmdStanModel( + stan_file=os.path.join(DATAFILES_PATH, "external.stan"), + user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp'), + ) + # default model name model = CmdStanModel(stan_file=BERN_STAN) self.assertEqual(BERN_BASENAME, model.name) @@ -78,6 +70,7 @@ def test_model_good(self): self.assertEqual(BERN_STAN, model.stan_file) self.assertEqual(None, model.exe_file) + # pylint: disable=no-self-use def test_model_pedantic(self): with LogCapture(level=logging.WARNING) as log: logging.getLogger() @@ -105,6 +98,10 @@ def test_model_bad(self): CmdStanModel(model_name='', stan_file=BERN_STAN) with self.assertRaises(ValueError): CmdStanModel(model_name=' ', stan_file=BERN_STAN) + with self.assertRaises(ValueError): + CmdStanModel( + stan_file=os.path.join(DATAFILES_PATH, "external.stan") + ) def test_stanc_options(self): opts = { diff --git a/test/test_optimize.py b/test/test_optimize.py index e1201341..81ed9847 100644 --- a/test/test_optimize.py +++ b/test/test_optimize.py @@ -7,7 +7,6 @@ import unittest import numpy as np -import pytest from testfixtures import LogCapture from cmdstanpy.cmdstan_args import CmdStanArgs, OptimizeArgs @@ -20,19 +19,6 @@ class CmdStanMLETest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk(DATAFILES_PATH): - for filename in files: - _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_instantiate(self): stan = os.path.join(DATAFILES_PATH, 'optimize', 'rosenbrock.stan') model = CmdStanModel(stan_file=stan) diff --git a/test/test_sample.py b/test/test_sample.py index 065d75e7..9eb35dce 100644 --- a/test/test_sample.py +++ b/test/test_sample.py @@ -14,7 +14,6 @@ from time import time import numpy as np -import pytest from testfixtures import LogCapture try: @@ -57,19 +56,6 @@ def without_import(library, module): class SampleTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk(DATAFILES_PATH): - for filename in files: - _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_bernoulli_good(self, stanfile='bernoulli.stan'): stan = os.path.join(DATAFILES_PATH, stanfile) bern_model = CmdStanModel(stan_file=stan) diff --git a/test/test_variational.py b/test/test_variational.py index 4bfabd49..679b9ac0 100644 --- a/test/test_variational.py +++ b/test/test_variational.py @@ -6,7 +6,6 @@ import unittest from math import fabs -import pytest from testfixtures import LogCapture from cmdstanpy.cmdstan_args import CmdStanArgs, VariationalArgs @@ -18,21 +17,6 @@ class CmdStanVBTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk( - os.path.join(DATAFILES_PATH, 'variational') - ): - for filename in files: - _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_instantiate(self): stan = os.path.join( DATAFILES_PATH, 'variational', 'eta_should_be_big.stan' @@ -155,21 +139,6 @@ def test_variables_3d(self): class VariationalTest(unittest.TestCase): - - # pylint: disable=no-self-use - @pytest.fixture(scope='class', autouse=True) - def do_clean_up(self): - for root, _, files in os.walk( - os.path.join(DATAFILES_PATH, 'variational') - ): - for filename in files: - _, ext = os.path.splitext(filename) - if ext.lower() in ('.o', '.d', '.hpp', '.exe', '') and ( - filename != ".gitignore" - ): - filepath = os.path.join(root, filename) - os.remove(filepath) - def test_variational_good(self): stan = os.path.join( DATAFILES_PATH, 'variational', 'eta_should_be_big.stan'