From 1521fd7c38e98d22d5e94673358f6c48588405e7 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 29 Apr 2022 11:44:53 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20RELEASE:=20v0.14.0=20(#406)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 156 +++++++++++++++++++++- docs/authoring/basics.md | 16 +++ docs/authoring/jupyter-notebooks.md | 18 +++ docs/authoring/text-notebooks.md | 14 +- docs/computation/execute.md | 36 +++-- docs/conf.py | 1 + docs/quickstart.md | 14 ++ docs/reference/changelog.md | 1 + myst_nb/cli.py | 42 ++++++ myst_nb/core/read.py | 6 +- pyproject.toml | 1 + tests/{test_quickstart.py => test_cli.py} | 27 +++- 12 files changed, 317 insertions(+), 15 deletions(-) rename tests/{test_quickstart.py => test_cli.py} (52%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 620464b2..3de33036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,167 @@ # Change Log +## v0.14.0 - 2022-04-27 + +[Full changelog](https://github.com/executablebooks/MyST-NB/compare/v0.13.2...v0.14.0) + +This release encompasses a **major** rewrite of the entire library and its documentation, primarily in [#380](https://github.com/executablebooks/MyST-NB/pull/380) and [#405](https://github.com/executablebooks/MyST-NB/pull/405). + +### Breaking Changes ‼️ + +#### Configuration + +A number of configuration option names have been changed, such that they now share the `nb_` prefix. +Most of the deprecated names will be auto-converted at the start of the build, emitting a warning such as: + +``` +WARNING: 'jupyter_execute_notebooks' is deprecated for 'nb_execution_mode' [mystnb.config] +``` + +`nb_render_priority` has been removed and replaced by `nb_mime_priority_overrides`, which has a different format and is more flexible. See [Outputs MIME priority](docs/render/format_code_cells.md) for more information. + +As per the changes in [`myst_parser`](myst:develop/_changelog), the `dollarmath` syntax extension is no longer included by default. +To re-add this extension, ensure that it is specified in your `conf.py`: `myst_enable_extensions = ["dollarmath"]`. + +For cell-level configuration the top-level key `render` has now been deprecated for `mystnb`. +For example, replace: + +````markdown +```{code-cell} +--- +render: + image: + width: 200px +--- +... +``` +```` + +with: + +````markdown +```{code-cell} +--- +mystnb: + image: + width: 200px +--- +... +``` +```` + +`render` will currently still be read, if present, and will issue a `[mystnb.cell_metadata_key]` warning. + +The `jupyter_sphinx_require_url` and `jupyter_sphinx_embed_url` configuration options are no longer used by this package, and are replaced by `nb_ipywidgets_js`. + +See the [configuration section](docs/configuration.md) for more details. + +#### Dependencies + +The [ipywidgets](https://ipywidgets.readthedocs.io) package has been removed from the requirements. +If required, please install it specifically. + +#### AST structure and rendering plugins + +The structure of the docutils AST and nodes produced by MyST-NB has been fully changed, for compatibility with the new [docutils only functionality](docs/docutils.md). +See [the API documentation](docs/reference/api.rst) for more details. + +The renderer plugin system (used by the `myst_nb.renderers` entry point) has also been completely rewritten, +so any current existing renderers will no longer work. +There is also now a new `myst_nb.mime_renderers` entry point, to allow for targeted rendering of specific code-cell output MIME types. +See [Customise the render process](docs/render/format_code_cells.md) for more information. + +#### Glue functionality + +By default, `glue` roles and directives now only work for keys within the same document. +To reference glued content in a different document, the `glue:any` directive allows for a `doc` option and `glue:any`/`glue:text` roles allow the (relative) doc path to be added, for example: + +````markdown +```{glue:any} var_text +:doc: other.ipynb +``` + +{glue:text}`other.ipynb::var_float:.2E` +```` + +This cross-document functionality is currently restricted to only `text/plain` and `text/html` output MIME types, not images. + +See [Embedding outputs as variables](docs/render/glue.md) for more details. + +### Dependency changes ⬆️ + +- Removed: + - `ipywidgets` + - `jupyter_sphinx` + - `nbconvert` +- Updated: + - `Python`: `3.6+ -> 3.7+` + - `myst_parser`: [`0.15 -> 0.17`](myst:develop/_changelog) + - `jupyter-cache`: [`0.4 -> 0.5`](https://github.com/executablebooks/jupyter-cache/blob/master/CHANGELOG.md) + - `sphinx-togglebutton`: [`0.1 -> 0.3`](https://sphinx-togglebutton.readthedocs.io/en/latest/changelog.html) + +### New and improved ✨ + +The following is a non-exhaustive list of new features and improvements, see the rest of the documentation for all the changes. + +- Multi-level configuration (global (`conf.py`) < notebook level metadata < cell level metadata) + - Plus new config options including: `nb_number_source_lines`, `nb_remove_code_source`, `nb_remove_code_outputs`, `nb_render_error_lexer`, `nb_execution_raise_on_error`, `nb_kernel_rgx_aliases` + - See the [configuration section](docs/configuration.md) for more details. + +- Added `mystnb-quickstart` and `mystnb-to-jupyter` CLI commands. + +- MyST text-based notebooks can now be specified by just: + + ```yaml + --- + file_format: mystnb + kernelspec: + name: python3 + --- + ``` + + as opposed to the alternative jupytext top-matter. + See [Text-based Notebooks](docs/authoring/text-notebooks.md) for more details. + +- docutils API/CLI with command line tools, e.g. `mystnb-docutils-html` + - Includes `glue` roles and directives + - See [single page builds](docs/docutils.md) for more details. + +- Parallel friendly (e.g. `sphinx-build -j 4` can execute four notebooks in parallel) + +- Page specific loading of ipywidgets JavaScript, i.e. only when ipywidgets are present in the notebook. + +- Added raw cell rendering, with the `raw-cell` directive. + See [Raw cells authoring](docs/authoring/jupyter-notebooks.md) for more details. + +- Added MIME render plugins. See [Customise the render process](docs/render/format_code_cells.md) for more details. + +- Better log info/warnings, with `type.subtype` specifiers for warning suppression. + See [Warning suppression](docs/configuration.md) for more details. + +- Reworked jupyter-cache integration to be easier to use (including parallel execution) + +- Added image options to `glue:figure` + +- New `glue:md` role/directive includes nested parsing of MyST Markdown. + See [Embedding outputs as variables](docs/render/glue.md) for more details. + +- Improved `nb-exec-table` directive (includes links to documents, etc) + +### Additional Pull Requests + +- 👌 IMPROVE: Update ANSI CSS colors by @saulshanabrook in [#384](https://github.com/executablebooks/MyST-NB/pull/384) +- ✨ NEW: Add `nb_execution_raise_on_error` config by @chrisjsewell in [#404](https://github.com/executablebooks/MyST-NB/pull/404) +- 👌 IMPROVE: Add image options to `glue:figure` by @chrisjsewell in [#403](https://github.com/executablebooks/MyST-NB/pull/403) + ## v0.13.2 - 2022-02-10 This release improves for cell outputs and brings UI improvements for toggling cell inputs and outputs. It also includes several bugfixes. -- Add CSS support for 8-bit ANSI colours [#379](https://github.com/executablebooks/MyST-NB/pull/379) ([@thiippal](https://github.com/thiippal)) +- Add CSS support for 8-bit ANSI colours [#379](https://github.com/executablebooks/MyST-NB/pull/379) ([@thiippal](https://github.com/thiippal)) - Use configured `nb_render_plugin` for glue nodes [#337](https://github.com/executablebooks/MyST-NB/pull/337) ([@bryanwweber](https://github.com/bryanwweber)) - UPGRADE: sphinx-togglebutton v0.3.0 [#390](https://github.com/executablebooks/MyST-NB/pull/390) ([@choldgraf](https://github.com/choldgraf)) - ## 0.13.1 - 2021-10-04 ✨ NEW: `nb_merge_streams` configuration [[PR #364](https://github.com/executablebooks/MyST-NB/pull/364)] diff --git a/docs/authoring/basics.md b/docs/authoring/basics.md index 15893288..b10f323a 100644 --- a/docs/authoring/basics.md +++ b/docs/authoring/basics.md @@ -93,6 +93,22 @@ kernelspec: The `kernelspec.name` should relate to a [Jupyter kernel](https://github.com/jupyter/jupyter/wiki/Jupyter-kernels) installed in your environment. +MyST-NB will also recognise [jupytext](https://jupytext.readthedocs.io) top-matter, such as: + +```yaml +--- +kernelspec: + name: python3 + display_name: python3 +jupytext: + text_representation: + extension: .md + format_name: myst + format_version: '0.13' + jupytext_version: 1.13.8 +--- +``` + Code cells are then designated by the `code-cell` directive: ````markdown diff --git a/docs/authoring/jupyter-notebooks.md b/docs/authoring/jupyter-notebooks.md index 5407ec1d..e07f0d0d 100644 --- a/docs/authoring/jupyter-notebooks.md +++ b/docs/authoring/jupyter-notebooks.md @@ -161,3 +161,21 @@ be embedded. fig, ax = plt.subplots() ax.scatter(*data, c=data[2]) ``` + +### Raw cells + +The [raw cell type](https://nbformat.readthedocs.io/en/latest/format_description.html#raw-nbconvert-cells) can be used to specifically render the content as a specific [MIME media type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types). + +````markdown +```{raw-cell} +:format: text/html + +

My cat is very grumpy.

+``` +```` + +```{raw-cell} +:format: text/html + +

My cat is very grumpy.

+``` diff --git a/docs/authoring/text-notebooks.md b/docs/authoring/text-notebooks.md index ededa9c8..8029df06 100644 --- a/docs/authoring/text-notebooks.md +++ b/docs/authoring/text-notebooks.md @@ -145,13 +145,23 @@ raise ValueError("oopsie!") (converting-ipynb)= ## Convert between MyST notebooks and `.ipynb` -MyST notebooks can be converted back-and-forth from `ipynb` files using [jupytext](https://jupytext.readthedocs.io), +MyST notebooks can be converted to Jupyter notebooks using the `mystnb-to-jupyter` CLI command. + +```console +$ mystnb-to-jupyter path/to/text-notebook.md +Wrote notebook to: path/to/text-notebook.ipynb +``` + +MyST notebooks can also be converted back-and-forth from `ipynb` files using [jupytext](https://jupytext.readthedocs.io), a Python library for two-way conversion of `ipynb` files with many text-based formats. -To let jupytext know the format of the notebook, add the notebook top-matter metadata: +To let jupytext know the format of the notebook, add the notebook top-matter similar to: ```yaml --- +kernelspec: + name: python3 + display_name: python3 jupytext: text_representation: extension: .md diff --git a/docs/computation/execute.md b/docs/computation/execute.md index 8a25d159..771a8eb4 100644 --- a/docs/computation/execute.md +++ b/docs/computation/execute.md @@ -17,13 +17,15 @@ See the sections below for a description of each configuration option and their ## Triggering notebook execution -To trigger the execution of notebook pages, use the following configuration in `conf.py`: +To trigger the execution of notebook pages, use the global `nb_execution_mode` configuration key, or per-file `execution_mode` key: + +By default this is set to: ```python nb_execution_mode = "auto" ``` -By default, this will only execute notebooks that are missing at least one output. +This will only execute notebooks that are missing at least one output. If a notebook has *all* of its outputs populated, then it will not be executed. **To force the execution of all notebooks, regardless of their outputs**, change the above configuration value to: @@ -57,26 +59,36 @@ Any file that matches one of the items in `nb_execution_excludepatterns` will no (execute/cache)= ## Cache execution outputs -As mentioned above, you can **cache the results of executing a notebook page** by setting: +As mentioned above, you can cache the results of executing a notebook page by setting: ```python nb_execution_mode = "cache" ``` -in your conf.py file. - In this case, when a page is executed, its outputs will be stored in a local database. This allows you to be sure that the outputs in your documentation are up-to-date, while saving time avoiding unnecessary re-execution. It also allows you to store your `.ipynb` files (or their `.md` equivalent) in your `git` repository *without their outputs*, but still leverage a cache to save time when building your site. +:::{tip} +You should only use this option when notebooks have deterministic execution outputs: + +- You use the same environment to run them (e.g. the same installed packages) +- They run no non-deterministic code (e.g. random numbers) +- They do not depend on external resources (e.g. files or network connections) that change over time +::: + When you re-build your site, the following will happen: -* Notebooks that have not seen changes to their **code cells** or **metadata** since the last build will not be re-executed. +- Notebooks that have not seen changes to their **code cells** or **metadata** since the last build will not be re-executed. Instead, their outputs will be pulled from the cache and inserted into your site. -* Notebooks that **have any change to their code cells** will be re-executed, and the cache will be updated with the new outputs. +- Notebooks that **have any change to their code cells** will be re-executed, and the cache will be updated with the new outputs. By default, the cache will be placed in the parent of your build folder. -Generally, this is in `_build/.jupyter_cache`. +Generally, this is in `_build/.jupyter_cache`, and it will also be specified in the build log, e.g. + +``` +Using jupyter-cache at: ./docs/_build/.jupyter_cache +``` You may also specify a path to the location of a jupyter cache you'd like to use: @@ -86,6 +98,14 @@ nb_execution_cache_path = "path/to/mycache" The path should point to an **empty folder**, or a folder where a **jupyter cache already exists**. +Once you have run the documentation build, you can inspect the contents of the cache with the following command: + +```console +$ jcache notebook -p docs/_build/.jupyter_cache list +``` + +See [jupyter-cache] for more information. + [jupyter-cache]: https://github.com/executablebooks/jupyter-cache "the Jupyter Cache Project" ## Execute with a different kernel name diff --git a/docs/conf.py b/docs/conf.py index d4fdefcf..7a673508 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,6 +61,7 @@ nitpick_ignore = [ ("py:class", klass) for klass in [ + "Path", "docutils.nodes.document", "docutils.nodes.Node", "docutils.nodes.Element", diff --git a/docs/quickstart.md b/docs/quickstart.md index 8c07ffb0..3d5f04ba 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -31,3 +31,17 @@ By default, MyST-NB will now parse both markdown (`.md`) and notebooks (`.ipynb` Begin authoring your content {material-regular}`navigate_next;2em` ``` + +Once you have finished authoring your content, you can now use the [sphinx-build CLI](https://www.sphinx-doc.org/en/master/man/sphinx-build.html) to build your documentation, e.g. + +```bash +sphinx-build -nW --keep-going -b html docs/ docs/_build/html +``` + +:::{tip} +MyST-NB is parallel-friendly, so you can also distribute the build (and execution of notebooks) over *N* processes with: `sphinx-build -j 4` +::: + +:::{seealso} +Check out [Read the Docs](https://docs.readthedocs.io) for hosting and *continuous deployment* of documentation +::: diff --git a/docs/reference/changelog.md b/docs/reference/changelog.md index 3139cd42..ceb0deef 100644 --- a/docs/reference/changelog.md +++ b/docs/reference/changelog.md @@ -1,2 +1,3 @@ ```{include} ../../CHANGELOG.md +:relative-docs: docs/ ``` diff --git a/myst_nb/cli.py b/myst_nb/cli.py index aa897cb6..58637d03 100644 --- a/myst_nb/cli.py +++ b/myst_nb/cli.py @@ -8,6 +8,7 @@ import nbformat from .core.config import NbParserConfig +from .core.read import read_myst_markdown_notebook def quickstart(args: list[str] | None = None): @@ -139,3 +140,44 @@ def generate_text_notebook() -> str: + "\n" ) return content + + +def md_to_nb(args: list[str] | None = None): + namespace = create_md_to_nb_cli().parse_args(args) + path = Path(namespace.inpath).resolve() + if namespace.outpath: + outpath = Path(namespace.outpath).resolve() + else: + outpath = path.with_suffix(".ipynb") + verbose: bool = namespace.verbose + overwrite: bool = namespace.overwrite + if outpath.exists(): + if not overwrite: + raise FileExistsError(f"{outpath} already exists.") + if verbose: + print(f"Overwriting existing file: {outpath}") + nb = read_myst_markdown_notebook(path.read_text("utf8"), path=path) + with outpath.open("w", encoding="utf8") as handle: + nbformat.write(nb, handle) + print(f"Wrote notebook to: {outpath}") + + +def create_md_to_nb_cli(): + cli = argparse.ArgumentParser( + description="Convert a text-based notebook to a Jupyter notebook." + ) + cli.add_argument( + "inpath", metavar="PATH_IN", type=str, help="Path to Markdown file." + ) + cli.add_argument( + "outpath", + metavar="PATH_OUT", + nargs="?", + type=str, + help="Path to output to.", + ) + cli.add_argument( + "-o", "--overwrite", action="store_true", help="Overwrite existing files." + ) + cli.add_argument("-v", "--verbose", action="store_true", help="Increase verbosity.") + return cli diff --git a/myst_nb/core/read.py b/myst_nb/core/read.py index ca2acc19..39d12cde 100644 --- a/myst_nb/core/read.py +++ b/myst_nb/core/read.py @@ -174,7 +174,7 @@ def read_myst_markdown_notebook( code_directive="{code-cell}", raw_directive="{raw-cell}", add_source_map=False, - path: str | None = None, + path: str | Path | None = None, ) -> nbf.NotebookNode: """Convert text written in the myst format to a notebook. @@ -351,7 +351,9 @@ def _read_cell_metadata(token, cell_index): return metadata -def _load_code_from_file(nb_path, file_name, token, body_lines): +def _load_code_from_file( + nb_path: None | str | Path, file_name: str, token, body_lines: list[str] +): """load source code from a file.""" if nb_path is None: raise _LoadFileParsingError("path to notebook not supplied for :load:") diff --git a/pyproject.toml b/pyproject.toml index 7b7f2705..2f941223 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,7 @@ testing = [ [project.scripts] mystnb-quickstart = "myst_nb.cli:quickstart" +mystnb-to-jupyter = "myst_nb.cli:md_to_nb" mystnb-docutils-html = "myst_nb.docutils_:cli_html" mystnb-docutils-html5 = "myst_nb.docutils_:cli_html5" mystnb-docutils-latex = "myst_nb.docutils_:cli_latex" diff --git a/tests/test_quickstart.py b/tests/test_cli.py similarity index 52% rename from tests/test_quickstart.py rename to tests/test_cli.py index ee225f52..c3ced9f6 100644 --- a/tests/test_quickstart.py +++ b/tests/test_cli.py @@ -1,9 +1,10 @@ """Test the quickstart CLI""" from pathlib import Path +import nbformat from sphinx.testing.path import path as sphinx_path -from myst_nb.cli import quickstart +from myst_nb.cli import md_to_nb, quickstart def test_quickstart(tmp_path: Path, make_app): @@ -21,3 +22,27 @@ def test_quickstart(tmp_path: Path, make_app): app.build() assert app._warning.getvalue().strip() == "" assert (path / "_build/html/index.html").exists() + + +def test_md_to_nb(tmp_path: Path): + """Test the md_to_nb CLI.""" + path = tmp_path / "notebook.md" + outpath = path.with_suffix(".ipynb") + path.write_text( + """\ +--- +kernelspec: + name: python3 +--- +# Title ++++ +next cell +""", + "utf8", + ) + md_to_nb([str(path)]) + assert path.exists() + with outpath.open("r") as handle: + nb = nbformat.read(handle, as_version=4) + assert nb.metadata == {"kernelspec": {"display_name": "python3", "name": "python3"}} + assert len(nb.cells) == 2