feat: Move from pygments to a treesitter based parsing.

* [WIP] Work on writing a parser using the textmate grammar backend. (#253) * Actually not only enums, quick and dirty start to using textmate parser * a little more boilerplate * property validator parsing * working function parsing without docstrings yet * start enum work * some enum parsing * working parsing for enumeration comments * add handling for block comments to enums * backport enum docstring parsing to properties * remove vestigial file * minor fixes + black * Hack for object hierarchy * Initial hack to get enumerations to work * better classdef parsing including changes to MATLAB-language-grammar prs #86, #88, and #90 * parse function docstring * extract function parser * initial work * finish integrating mat_textmate_parser with mat_types * [skip-ci] some minor changes * initial work on a tree sitter based parser * nearly finished with tree-sitter implementation * everything but events working * working events * exit early if query returns for block with no elements * integrating tree-sitter parser into mat_types * fixing default value parsing * some test fixes and requires tree-sitter * rm textmate parser on this branch * also install tree-sitter * bump required tree-sitter * tree-sitter version bump * dealing with tree-sitter version diffs to maintain py 3.8 compatibility * ML_LANG versions * a better attributes query * Ci: Test on Sphinx 8 / Dev. (#259) * CI: Testing for latest Sphinx (8.0) * CI: Fix helper class version checking. * Fixing nearly all tests in test_parse_mfile * fix a _lot_ of autodoc * temporarily point to tree-sitter-matlab branch on apozharski fork * fix old property syntax + update tests * fixing test_autodoc and test_matlabify, only comment and line continuation issues remain * fixing the last of the tests * remove dead code * address PR comments made by @joeced * fix * lint * fix typo in __all__ * remove Pygments dependencies * py-tree-sitter v0.23.0 has a breaking change for Query.match() * remove duplicat entry in yaml * fix spec printing * minor fixes for arg block parsing and a test * fix bug regarding output block argument parsing and add test * remove print and fix test_matlabify * remove textmate from dev-reqs * fix typo --------- Co-authored-by: Jørgen Cederberg <[email protected]>
sphinx-contrib · Sep 18, 2024 · 4cd5c51 · 4cd5c51
1 parent f9cbf4d
commit 4cd5c51
Show file tree

Hide file tree

Showing 11 changed files with 1,385 additions and 1,136 deletions.
diff --git a/setup.py b/setup.py
@@ -5,7 +5,11 @@
 with open("README.rst", "r") as f_readme:
     long_desc = f_readme.read()
 
-requires = ["Sphinx>=4.0.0", "Pygments>=2.0.1"]
+requires = [
+    "Sphinx>=4.0.0",
+    "tree-sitter-matlab>=1.0.2",
+    "tree-sitter>=0.21.3,<0.23.0",
+]
 
 setup(
     name="sphinxcontrib-matlabdomain",

diff --git a/sphinxcontrib/mat_documenters.py b/sphinxcontrib/mat_documenters.py
@@ -14,6 +14,7 @@
     MatFunction,
     MatClass,
     MatProperty,
+    MatEnumeration,
     MatMethod,
     MatScript,
     MatException,
@@ -555,6 +556,9 @@ def member_is_friend_of(member, friends):
             else:
                 return False
 
+        def member_is_enum(member):
+            return isinstance(member, MatEnumeration)
+
         ret = []
 
         # search for members in source code too
@@ -637,7 +641,7 @@ def member_is_friend_of(member, friends):
                 isattr = True
             else:
                 # ignore undocumented members if :undoc-members: is not given
-                keep = has_doc or self.options.undoc_members
+                keep = has_doc or self.options.undoc_members or member_is_enum(member)
 
             # give the user a chance to decide whether this member
             # should be skipped
@@ -656,7 +660,6 @@ def member_is_friend_of(member, friends):
 
             if keep:
                 ret.append((membername, member, isattr))
-
         return ret
 
     def document_members(self, all_members=False):
@@ -1229,11 +1232,19 @@ def document_members(self, all_members=False):
             for (membername, member) in filtered_members
             if isinstance(member, MatMethod) and member.name != member.cls.name
         ]
+        # create list of enums
+        enum_names = [
+            membername
+            for (membername, member) in filtered_members
+            if isinstance(member, MatEnumeration)
+        ]
         # create list of other members
         other_names = [
             membername
             for (membername, member) in filtered_members
-            if not isinstance(member, MatMethod) and not isinstance(member, MatProperty)
+            if not isinstance(member, MatMethod)
+            and not isinstance(member, MatProperty)
+            and not isinstance(member, MatEnumeration)
             # exclude parent modules with names matching members (as in Myclass.Myclass)
             and not (hasattr(member, "module") and member.name == member.module)
         ]
@@ -1255,6 +1266,12 @@ def document_members(self, all_members=False):
             for (membername, member) in members
             if not isinstance(member, MatMethod) or member.name == member.cls.name
         ]
+        # create list of members that are not properties
+        non_enums = [
+            membername
+            for (membername, member) in members
+            if not isinstance(member, MatEnumeration)
+        ]
         # create list of members that are not non-constructor methods
         non_other = [
             membername
@@ -1281,6 +1298,10 @@ def document_members(self, all_members=False):
                 "Property Summary", non_properties, all_members
             )
 
+        # enumss
+        if enum_names:
+            self.document_member_section("Enumeration Values", non_enums, all_members)
+
         # methods
         if meth_names:
             self.document_member_section("Method Summary", non_methods, all_members)
@@ -1359,10 +1380,11 @@ def format_args(self):
         is_ctor = self.object.cls.name == self.object.name
 
         if self.object.args:
-            if self.object.args[0] in ("obj", "self") and not is_ctor:
-                return "(" + ", ".join(self.object.args[1:]) + ")"
+            arglist = list(self.object.args.keys())
+            if arglist[0] in ("obj", "self") and not is_ctor:
+                return "(" + ", ".join(arglist[1:]) + ")"
             else:
-                return "(" + ", ".join(self.object.args) + ")"
+                return "(" + ", ".join(arglist) + ")"
 
     def document_members(self, all_members=False):
         pass
@@ -1453,7 +1475,16 @@ def add_directive_header(self, sig):
                     obj_default = " = " + obj_default
 
                 if self.env.config.matlab_show_property_specs:
-                    obj_default = self.object.specs + obj_default
+                    prop_spec = ""
+                    if self.object.size is not None:
+                        prop_spec = prop_spec + "(" + ",".join(self.object.size) + ")"
+                    if self.object.type is not None:
+                        prop_spec = prop_spec + " " + self.object.type
+                    if self.object.validators is not None:
+                        prop_spec = (
+                            prop_spec + " {" + ",".join(self.object.validators) + "}"
+                        )
+                    obj_default = prop_spec + obj_default
 
                 self.add_line("   :annotation: " + obj_default, "<autodoc>")
         elif self.options.annotation is SUPPRESS:

diff --git a/sphinxcontrib/mat_parser.py b/sphinxcontrib/mat_parser.py