Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow user to supply a c++ header file #464

Merged
merged 5 commits into from
Oct 11, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 51 additions & 9 deletions cmdstanpy/compiler_opts.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,18 +48,21 @@ class CompilerOptions:
Attributes:
stanc_options - stanc compiler flags, options
cpp_options - makefile options (NAME=value)
user_header - path to a user .hpp file to include during compilation
"""

def __init__(
self,
*,
stanc_options: Optional[Dict[str, Any]] = None,
cpp_options: Optional[Dict[str, Any]] = None,
user_header: Optional[str] = None,
logger: Optional[logging.Logger] = None,
) -> None:
"""Initialize object."""
self._stanc_options = stanc_options if stanc_options is not None else {}
self._cpp_options = cpp_options if cpp_options is not None else {}
self._user_header = user_header if user_header is not None else ''
if logger is not None:
get_logger().warning(
"Parameter 'logger' is deprecated."
Expand Down Expand Up @@ -88,6 +91,7 @@ def validate(self) -> None:
"""
self.validate_stanc_opts()
self.validate_cpp_opts()
self.validate_user_header()

def validate_stanc_opts(self) -> None:
"""
Expand All @@ -104,17 +108,15 @@ def validate_stanc_opts(self) -> None:
get_logger().info('ignoring compiler option: %s', key)
ignore.append(key)
elif key not in STANC_OPTS:
raise ValueError(
'unknown stanc compiler option: {}'.format(key)
)
raise ValueError(f'unknown stanc compiler option: {key}')
elif key == 'include_paths':
paths = val
if isinstance(val, str):
paths = val.split(',')
elif not isinstance(val, list):
raise ValueError(
'Invalid include_paths, expecting list or '
'string, found type: {}.'.format(type(val))
f'string, found type: {type(val)}.'
)
elif key == 'use-opencl':
if self._cpp_options is None:
Expand Down Expand Up @@ -149,10 +151,48 @@ def validate_cpp_opts(self) -> None:
val = self._cpp_options[key]
if not isinstance(val, int) or val < 0:
raise ValueError(
'{} must be a non-negative integer value,'
' found {}.'.format(key, val)
f'{key} must be a non-negative integer value,'
f' found {val}.'
)

def validate_user_header(self) -> None:
"""
User header exists.
Raise ValueError if bad config is found.
"""
if self._user_header != "":
if not (
os.path.exists(self._user_header)
and os.path.isfile(self._user_header)
):
raise ValueError(
f"User header file {self._user_header} cannot be found"
)
if self._user_header[-4:] != '.hpp':
raise ValueError(
f"Header file must end in .hpp, got {self._user_header}"
)
if "allow_undefined" not in self._stanc_options:
self._stanc_options["allow_undefined"] = True
# set full path
self._user_header = os.path.abspath(self._user_header)

if ' ' in self._user_header:
mitzimorris marked this conversation as resolved.
Show resolved Hide resolved
raise ValueError(
"User header must be in a location with no spaces in path!"
)

if (
'USER_HEADER' in self._cpp_options
and self._user_header != self._cpp_options['USER_HEADER']
):
raise ValueError(
"Disagreement in user_header C++ options found!\n"
f"{self._user_header}, {self._cpp_options['USER_HEADER']}"
)

self._cpp_options['USER_HEADER'] = self._user_header

def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000
"""Adds options to existing set of compiler options."""
if new_opts.stanc_options is not None:
Expand All @@ -167,6 +207,8 @@ def add(self, new_opts: "CompilerOptions") -> None: # noqa: disable=Q000
if new_opts.cpp_options is not None:
for key, val in new_opts.cpp_options.items():
self._cpp_options[key] = val
if new_opts._user_header != '' and self._user_header == '':
self._user_header = new_opts._user_header

def add_include_path(self, path: str) -> None:
"""Adds include path to existing set of compiler options."""
Expand All @@ -191,10 +233,10 @@ def compose(self) -> List[str]:
)
)
elif key == 'name':
opts.append('STANCFLAGS+=--{}={}'.format(key, val))
opts.append(f'STANCFLAGS+=--name={val}')
else:
opts.append('STANCFLAGS+=--{}'.format(key))
opts.append(f'STANCFLAGS+=--{key}')
if self._cpp_options is not None and len(self._cpp_options) > 0:
for key, val in self._cpp_options.items():
opts.append('{}={}'.format(key, val))
opts.append(f'{key}={val}')
return opts
29 changes: 26 additions & 3 deletions cmdstanpy/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ class CmdStanModel:
:param cpp_options: Options for C++ compiler, specified as a Python
dictionary containing C++ compiler option name, value pairs.
Optional.

:param user_header: A path to a header file to include during C++
compilation.
Optional.
"""

def __init__(
Expand All @@ -86,6 +90,7 @@ def __init__(
compile: bool = True,
stanc_options: Optional[Dict[str, Any]] = None,
cpp_options: Optional[Dict[str, Any]] = None,
user_header: Optional[str] = None,
logger: Optional[logging.Logger] = None,
) -> None:
"""
Expand All @@ -97,12 +102,16 @@ def __init__(
:param compile: Whether or not to compile the model.
:param stanc_options: Options for stanc compiler.
:param cpp_options: Options for C++ compiler.
:param user_header: A path to a header file to include during C++
compilation.
"""
self._name = ''
self._stan_file = None
self._exe_file = None
self._compiler_options = CompilerOptions(
stanc_options=stanc_options, cpp_options=cpp_options
stanc_options=stanc_options,
cpp_options=cpp_options,
user_header=user_header,
)
if logger is not None:
get_logger().warning(
Expand Down Expand Up @@ -235,6 +244,11 @@ def cpp_options(self) -> Dict[str, Union[bool, int]]:
"""Options to C++ compilers."""
return self._compiler_options._cpp_options

@property
def user_header(self) -> str:
"""The user header file if it exists, otherwise empty"""
return self._compiler_options._user_header

def code(self) -> Optional[str]:
"""Return Stan program as a string."""
if not self._stan_file:
Expand All @@ -255,6 +269,7 @@ def compile(
force: bool = False,
stanc_options: Optional[Dict[str, Any]] = None,
cpp_options: Optional[Dict[str, Any]] = None,
user_header: Optional[str] = None,
override_options: bool = False,
) -> None:
"""
Expand All @@ -272,6 +287,8 @@ def compile(

:param stanc_options: Options for stanc compiler.
:param cpp_options: Options for C++ compiler.
:param user_header: A path to a header file to include during C++
compilation.

:param override_options: When ``True``, override existing option.
When ``False``, add/replace existing options. Default is ``False``.
Expand All @@ -280,9 +297,15 @@ def compile(
raise RuntimeError('Please specify source file')

compiler_options = None
if not (stanc_options is None and cpp_options is None):
if not (
stanc_options is None
and cpp_options is None
and user_header is None
):
compiler_options = CompilerOptions(
stanc_options=stanc_options, cpp_options=cpp_options
stanc_options=stanc_options,
cpp_options=cpp_options,
user_header=user_header,
)
compiler_options.validate()
if self._compiler_options is None:
Expand Down
1 change: 1 addition & 0 deletions docsrc/env.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ dependencies:
- python=3.7
- ipykernel
- ipython
- ipywidgets
- numpy>=1.15
- pandas
- xarray
Expand Down
1 change: 1 addition & 0 deletions docsrc/examples.rst
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ __________________
examples/Maximum Likelihood Estimation.ipynb
examples/Variational Inference.ipynb
examples/Run Generated Quantities.ipynb
examples/Using External C++.ipynb
3 changes: 2 additions & 1 deletion docsrc/examples/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@
*.hpp
*.exe
*.csv
.ipynb_checkpoints
.ipynb_checkpoints
!make_odds.hpp
157 changes: 157 additions & 0 deletions docsrc/examples/Using External C++.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Advanced Topic: Using External C++ Functions\n",
"\n",
"This is based on the relevant portion of the CmdStan documentation [here](https://mc-stan.org/docs/cmdstan-guide/using-external-cpp-code.html)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Consider the following Stan model, based on the bernoulli example."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {"nbsphinx": "hidden"},
"outputs": [],
"source": [
"import os\n",
"try:\n",
" os.remove('bernoulli_external')\n",
"except:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from cmdstanpy import CmdStanModel\n",
"model_external = CmdStanModel(stan_file='bernoulli_external.stan', compile=False)\n",
"print(model_external.code())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"As you can see, it features a function declaration for `make_odds`, but no definition. If we try to compile this, we will get an error. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model_external.compile()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Even enabling the `--allow_undefined` flag to stanc3 will not allow this model to be compiled quite yet."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model_external.compile(stanc_options={'allow_undefined':True})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To resolve this, we need to both tell the Stan compiler an undefined function is okay **and** let C++ know what it should be. \n",
"\n",
"We can provide a definition in a C++ header file by using the `user_header` argument to either the CmdStanModel constructor or the `compile` method. \n",
"\n",
"This will enables the `allow_undefined` flag automatically."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model_external.compile(user_header='make_odds.hpp')"
Comment on lines +81 to +92
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

update this to cpp_opts

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure what you mean. The user interface is still to use model_header, this just gets turned into part of the CPP options during validation

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you're right - user error on my part - testing in wrong environment.

]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can then run this model and inspect the output"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fit = model_external.sample(data={'N':10, 'y':[0,1,0,0,0,0,0,0,0,1]})\n",
"fit.stan_variable('odds')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The contents of this header file are a bit complicated unless you are familiar with the C++ internals of Stan, so they are presented without comment:\n",
"\n",
"```c++\n",
"#include <boost/math/tools/promotion.hpp>\n",
"#include <ostream>\n",
"\n",
"namespace bernoulli_model_namespace {\n",
" template <typename T0__> inline typename\n",
" boost::math::tools::promote_args<T0__>::type \n",
" make_odds(const T0__& theta, std::ostream* pstream__) {\n",
" return theta / (1 - theta); \n",
" }\n",
"}\n",
"```"
]
}
],
"metadata": {
"interpreter": {
"hash": "8765ce46b013071999fc1966b52035a7309a0da7551e066cc0f0fa23e83d4f60"
},
"kernelspec": {
"display_name": "Python 3.9.5 64-bit ('stan': conda)",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Loading