⬆️ UPDATE: Sphinx v5, myst-parser v0.18 (#424)

Also removes support for sphinx v3.

myst-parser v0.18 now supports file-level config for markdown parsing configuration, e.g. in text-based notebook:

```yaml
---
file_format: mystnb
kernelspec:
  name: python3
myst:
    enable_extensions:
    - "deflist"
mystnb:
    execution_mode: "off"
---
```

.github/workflows/tests.yml

@@ -27,14 +27,17 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10"]
-        sphinx: [">=4,<5"]
+        sphinx: [">=5,<6"]
         include:
+          - os: ubuntu-latest
+            python-version: 3.7
+            sphinx: ">=4,<5"
           - os: windows-latest
             python-version: 3.8
-            sphinx: ">=4,<5"
+            sphinx: ">=5,<6"
           - os: macos-latest
             python-version: 3.8
-            sphinx: ">=4,<5"
+            sphinx: ">=5,<6"
 
     runs-on: ${{ matrix.os }}
 

.pre-commit-config.yaml

@@ -39,7 +39,7 @@ repos:
     rev: 4.0.1
     hooks:
     - id: flake8
-      additional_dependencies: [flake8-bugbear==21.3.1]
+      additional_dependencies: [flake8-bugbear]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.960
@@ -48,8 +48,8 @@ repos:
       args: [--config-file=pyproject.toml]
       additional_dependencies:
       - importlib_metadata
-      - myst-parser~=0.17.2
-      - "sphinx~=4.3.2"
+      - myst-parser~=0.18.0
+      - "sphinx~=5.0"
       - nbclient
       - types-PyYAML
       files: >

docs/conf.py

@@ -70,7 +70,6 @@
         "docutils.nodes.system_message",
         "DocutilsNbRenderer",
         "SphinxNbRenderer",
-        "myst_parser.main.MdParserConfig",
         "nbformat.notebooknode.NotebookNode",
         "nbf.NotebookNode",
         "NotebookNode",
@@ -134,7 +133,7 @@ def setup(app):
 
     from docutils import nodes
     from docutils.parsers.rst import directives
-    from myst_parser.main import MdParserConfig
+    from myst_parser.config.main import MdParserConfig
     from sphinx.application import Sphinx
     from sphinx.util.docutils import SphinxDirective
 

myst_nb/core/config.py

@@ -3,7 +3,7 @@
 from enum import Enum
 from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple
 
-from myst_parser.dc_validators import (
+from myst_parser.config.dc_validators import (
     ValidatorType,
     deep_iterable,
     deep_mapping,
@@ -78,16 +78,17 @@ def has_items(*validators) -> ValidatorType:
     :param validators: Validator to apply per item
     """
 
-    def _validator(inst, attr, value):
+    def _validator(inst, field: dc.Field, value, suffix=""):
         if not isinstance(value, Sequence):
-            raise TypeError(f"{attr.name} must be a sequence: {value}")
+            raise TypeError(f"{suffix}{field.name} must be a sequence: {value}")
         if len(value) != len(validators):
             raise TypeError(
-                f"{attr.name!r} must be a sequence of length {len(validators)}: {value}"
+                f"{suffix}{field.name!r} must be a sequence of length "
+                f"{len(validators)}: {value}"
             )
 
-        for validator, member in zip(validators, value):
-            validator(inst, attr, member)
+        for idx, (validator, member) in enumerate(zip(validators, value)):
+            validator(inst, field, member, suffix=f"{suffix}[{idx}]")
 
     return _validator
 

myst_nb/core/read.py

@@ -9,7 +9,8 @@
 
 from docutils.parsers.rst import Directive
 from markdown_it.renderer import RendererHTML
-from myst_parser.main import MdParserConfig, create_md_parser
+from myst_parser.config.main import MdParserConfig
+from myst_parser.parsers.mdit import create_md_parser
 import nbformat as nbf
 import yaml
 
@@ -312,7 +313,10 @@ class _MockDirective:
 
 
 def _read_fenced_cell(token, cell_index, cell_type):
-    from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text
+    from myst_parser.parsers.directives import (
+        DirectiveParsingError,
+        parse_directive_text,
+    )
 
     try:
         _, options, body_lines, _ = parse_directive_text(

myst_nb/core/render.py

@@ -20,8 +20,9 @@
 from docutils import nodes
 from docutils.parsers.rst import directives as options_spec
 from importlib_metadata import entry_points
-from myst_parser.docutils_renderer import token_line
-from myst_parser.main import MdParserConfig, create_md_parser
+from myst_parser.config.main import MdParserConfig
+from myst_parser.mdit_to_docutils.base import token_line
+from myst_parser.parsers.mdit import create_md_parser
 from nbformat import NotebookNode
 from typing_extensions import Protocol
 

myst_nb/docutils_.py

@@ -13,11 +13,18 @@
 from docutils.parsers.rst.roles import _roles
 from markdown_it.token import Token
 from markdown_it.tree import SyntaxTreeNode
-from myst_parser.docutils_ import DOCUTILS_EXCLUDED_ARGS as DOCUTILS_EXCLUDED_ARGS_MYST
-from myst_parser.docutils_ import Parser as MystParser
-from myst_parser.docutils_ import create_myst_config, create_myst_settings_spec
-from myst_parser.docutils_renderer import DocutilsRenderer, token_line
-from myst_parser.main import MdParserConfig, create_md_parser
+from myst_parser.config.main import MdParserConfig, merge_file_level
+from myst_parser.mdit_to_docutils.base import (
+    DocutilsRenderer,
+    create_warning,
+    token_line,
+)
+from myst_parser.parsers.docutils_ import (
+    DOCUTILS_EXCLUDED_ARGS as DOCUTILS_EXCLUDED_ARGS_MYST,
+)
+from myst_parser.parsers.docutils_ import Parser as MystParser
+from myst_parser.parsers.docutils_ import create_myst_config, create_myst_settings_spec
+from myst_parser.parsers.mdit import create_md_parser
 import nbformat
 from nbformat import NotebookNode
 from pygments.formatters import get_formatter_by_name
@@ -141,6 +148,14 @@ def _parse(self, inputstring: str, document: nodes.document) -> None:
             nb_reader = NbReader(standard_nb_read, md_config)
         notebook = nb_reader.read(inputstring)
 
+        # update the global markdown config with the file-level config
+        warning = lambda wtype, msg: create_warning(  # noqa: E731
+            document, msg, line=1, append_to=document, subtype=wtype
+        )
+        nb_reader.md_config = merge_file_level(
+            nb_reader.md_config, notebook.metadata, warning
+        )
+
         # Update mystnb configuration with notebook level metadata
         if nb_config.metadata_key in notebook.metadata:
             overrides = nb_node_to_dict(notebook.metadata[nb_config.metadata_key])

myst_nb/sphinx_.py

@@ -10,10 +10,11 @@
 from docutils import nodes
 from markdown_it.token import Token
 from markdown_it.tree import SyntaxTreeNode
-from myst_parser.docutils_renderer import token_line
-from myst_parser.main import MdParserConfig, create_md_parser
-from myst_parser.sphinx_parser import MystParser
-from myst_parser.sphinx_renderer import SphinxRenderer
+from myst_parser.config.main import MdParserConfig, merge_file_level
+from myst_parser.mdit_to_docutils.base import token_line
+from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer, create_warning
+from myst_parser.parsers.mdit import create_md_parser
+from myst_parser.parsers.sphinx_ import MystParser
 import nbformat
 from sphinx.application import Sphinx
 from sphinx.environment import BuildEnvironment
@@ -72,6 +73,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
 
         # get markdown parsing configuration
         md_config: MdParserConfig = self.env.myst_config
+
         # get notebook rendering configuration
         nb_config: NbParserConfig = self.env.mystnb_config
 
@@ -82,6 +84,14 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
             return super().parse(inputstring, document)
         notebook = nb_reader.read(inputstring)
 
+        # update the global markdown config with the file-level config
+        warning = lambda wtype, msg: create_warning(  # noqa: E731
+            document, msg, line=1, append_to=document, subtype=wtype
+        )
+        nb_reader.md_config = merge_file_level(
+            nb_reader.md_config, notebook.metadata, warning
+        )
+
         # potentially replace kernel name with alias
         kernel_name = notebook.metadata.get("kernelspec", {}).get("name", None)
         if kernel_name is not None and nb_config.kernel_rgx_aliases:

myst_nb/sphinx_ext.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import Any
 
-from myst_parser import setup_sphinx as setup_myst_parser
+from myst_parser.sphinx_ext.main import setup_sphinx as setup_myst_parser
 from sphinx.application import Sphinx
 from sphinx.util import logging as sphinx_logging
 from sphinx.util.fileutil import copy_asset_file

pyproject.toml

@@ -35,15 +35,14 @@ keywords = [
 ]
 requires-python = ">=3.7"
 dependencies = [
-    "docutils>=0.15,<0.18",
     "importlib_metadata",
     "ipython",
     "jupyter-cache~=0.5.0",
     "nbclient",  # nbclient version pinned by jupyter-client
-    "myst-parser~=0.17.2",
+    "myst-parser~=0.18.0",
     "nbformat~=5.0",
     "pyyaml",
-    "sphinx>=3.5,<5",
+    "sphinx>=4,<6",
     "sphinx-togglebutton~=0.3.0",
     "typing-extensions",
     # ipykernel is not a requirement of the library,
@@ -95,12 +94,10 @@ testing = [
     "ipython!=8.1.0", # see https://github.com/ipython/ipython/issues/13554
     "ipywidgets",
     "jupytext~=1.11.2",
-    # TODO: 3.4.0 has some warnings that need to be fixed in the tests.
-    "matplotlib~=3.3.0",
+    "matplotlib",
     "nbdime",
     "numpy",
-    # TODO: 1.4.0 has some warnings that need to be fixed in the tests.
-    "pandas<1.4",
+    "pandas",
     "pytest~=7.1",
     "pytest-cov~=3.0",
     "pytest-regressions",

tests/nb_fixtures/basic.txt

@@ -171,18 +171,18 @@ cells:
 .
 <document nb_kernelspec="{'name': 'python3', 'display_name': 'Python 3', 'language': ''}" source="<string>">
     <paragraph>
-        <footnote_reference auto="1" ids="id1" refid="a">
+        <footnote_reference auto="1" ids="footnote-reference-1" refid="a">
             1
 
-        <footnote_reference auto="1" ids="id2" refid="b">
+        <footnote_reference auto="1" ids="footnote-reference-2" refid="b">
             2
     <transition classes="footnotes">
-    <footnote auto="1" backrefs="id1" ids="a" names="a">
+    <footnote auto="1" backrefs="footnote-reference-1" ids="a" names="a">
         <label>
             1
         <paragraph>
             before
-    <footnote auto="1" backrefs="id2" ids="b" names="b">
+    <footnote auto="1" backrefs="footnote-reference-2" ids="b" names="b">
         <label>
             2
         <paragraph>

tests/notebooks/complex_outputs_unrun.ipynb

@@ -13,7 +13,6 @@
    "source": [
     "import matplotlib.pyplot as plt\n",
     "import pandas as pd\n",
-    "pd.set_option('display.latex.repr', True)\n",
     "import sympy as sym\n",
     "sym.init_printing(use_latex=True)\n",
     "import numpy as np\n",

tests/notebooks/file_level_config.ipynb

@@ -0,0 +1,32 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "be9f2c4c",
+   "metadata": {},
+   "source": [
+    "# Title\n",
+    "\n",
+    "name\n",
+    ": definition"
+   ]
+  }
+ ],
+ "metadata": {
+  "file_format": "mystnb",
+  "kernelspec": {
+   "display_name": "python3",
+   "name": "python3"
+  },
+  "myst": {
+   "enable_extensions": [
+    "deflist"
+   ]
+  },
+  "mystnb": {
+   "execution_mode": "off"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

tests/notebooks/file_level_config.md

@@ -0,0 +1,15 @@
+---
+file_format: mystnb
+kernelspec:
+  name: python3
+myst:
+    enable_extensions:
+    - "deflist"
+mystnb:
+    execution_mode: "off"
+---
+
+# Title
+
+name
+: definition