Get Mathics3 Modules to loading properly ...

This needs the doc-code-rebased-rebased branch of Mathics core.

.github/workflows/osx.yaml

@@ -30,7 +30,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone https://github.com/Mathics3/mathics-core
+        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django

.github/workflows/ubuntu.yml

@@ -29,7 +29,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone https://github.com/Mathics3/mathics-core
+        git clone --branch clone doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
         (cd mathics-core && pip3 install -e .[full])
         (cd mathics-core && bash ./admin-tools/make-op-tables.sh)
     - name: Install Mathics3 Django

.github/workflows/windows.yml

@@ -32,7 +32,7 @@ jobs:
         # For testing 3.11 we need to do something like the below until the next Mathics3 is released
         # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
         # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        git clone https://github.com/Mathics3/mathics-core
+        git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core
         bash -c '(cd mathics-core && pip3 install -e .[full])'
         bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)'
     - name: Install Mathics3 Django

mathics_django/doc/django_doc.py

@@ -18,7 +18,7 @@
     DocTests,
     DocText,
     Documentation,
-    XMLDoc,
+    DocumentationEntry,
     gather_tests,
     get_results_by_test,
     sorted_chapters,
@@ -80,7 +80,7 @@ def __init__(self):
         self.section_class = DjangoDocSection
         self.subsection_class = DjangoDocSubsection
 
-        self.gather_doctest_data()
+        self.load_documentation_sources()
         self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL
         self.pymathics_doc_loaded = False
         self.doc_data_file = settings.get_doctest_latex_data_path(
@@ -148,7 +148,7 @@ def search_sections(section, result):
         return sorted_results
 
 
-class DjangoDoc(XMLDoc):
+class DjangoDoc(DocumentationEntry):
     def __init__(self, doc, title, section, key_prefix=None):
         self.title = title
         if key_prefix is None:

mathics_django/docpipeline.py

@@ -17,16 +17,10 @@
 import mathics
 import mathics.docpipeline as md
 from mathics.core.definitions import Definitions
-from mathics.core.load_builtin import (
-    builtins_by_module,
-    builtins_dict,
-    import_and_load_builtins,
-)
+from mathics.core.load_builtin import import_and_load_builtins
+from mathics.doc.utils import open_ensure_dir
 from mathics.docpipeline import (
     MAX_TESTS,
-    create_output,
-    open_ensure_dir,
-    print_and_log,
     test_all,
     test_chapters,
     test_sections,
@@ -38,13 +32,6 @@
 from mathics_django.settings import get_doctest_html_data_path
 
 
-def load_doc_data():
-    doc_html_data_path = get_doctest_html_data_path(should_be_readable=True)
-    print(f"Loading internal document data from {doc_html_data_path}")
-    with open_ensure_dir(doc_html_data_path, "rb") as doc_data_file:
-        return pickle.load(doc_data_file)
-
-
 def save_doctest_data(output_data):
     """
     Save doctest tests and test results to a Python PCL file.
@@ -207,7 +194,7 @@ def main():
             else:
                 print(f"Mathics3 Module {module_name} loaded")
 
-    md.DOCUMENTATION.gather_doctest_data()
+    # md.DOCUMENTATION.load_documentation_sources()
 
     start_time = None
     total = 0
@@ -244,7 +231,6 @@ def main():
                 generate_output=args.output,
                 stop_on_failure=args.stop_on_failure,
                 start_at=start_at,
-                count=args.count,
                 doc_even_if_error=args.keep_going,
                 excludes=excludes,
             )

mathics_django/web/controllers/doc.py

@@ -8,7 +8,8 @@
 from django.core.handlers.wsgi import WSGIRequest
 from django.http import Http404, HttpResponse
 from django.shortcuts import render
-from mathics.doc.common_doc import get_module_doc, mathics3_module_part
+from mathics.doc.common_doc import MATHICS3_MODULES_TITLE
+from mathics.doc.utils import slugify
 from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
 
 from mathics_django.doc import documentation
@@ -23,37 +24,38 @@
 
 seen_pymathics_modules = copy(pymathics_modules)
 
+MATHICS3_MODULES_SLUG = slugify(MATHICS3_MODULES_TITLE)
 
-def check_for_pymathics_load():
+
+def check_for_new_load_modules():
+    """
+    See if we have laoded any new Mathics3 modules since the last time
+    we checked. If so get an add the documenation for that.
+    """
     global seen_pymathics_modules
     if seen_pymathics_modules != pymathics_modules:
-        print("XXX refresh pymathics doc", pymathics_modules)
-        new_modules = pymathics_modules - seen_pymathics_modules
-        for new_module in new_modules:
-            title, _ = get_module_doc(new_module)
-            chapter = mathics3_module_part.doc.gather_chapter_doc_fn(
-                mathics3_module_part,
-                title,
-                mathics3_module_part.doc,
-            )
-            from trepan.api import debug
-
-            debug()
-            submodule_names_seen = set()
-            chapter.doc.doc_chapter(
-                new_module,
-                mathics3_module_part,
-                pymathics_builtins_by_module,
-                seen_pymathics_modules,
-                submodule_names_seen,
-            )
-            chapter.get_tests()
+        mathics3_module_part = documentation.parts_by_slug.get(
+            MATHICS3_MODULES_SLUG, None
+        )
+        if mathics3_module_part is None:
+            print("Something is wrong: mathics3_module variable should not be None")
+            return
+        else:
+            # The "Mathics3 modules" part already exists; add the new chapters.
+            new_modules = pymathics_modules - seen_pymathics_modules
+            for new_module in new_modules:
+                chapter = documentation.doc_chapter(
+                    new_module, mathics3_module_part, pymathics_builtins_by_module
+                )
+                mathics3_module_part.chapters.append(chapter)
+                pass
+            pass
         seen_pymathics_modules = copy(pymathics_modules)
-        pass
+    return
 
 
 def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse:
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     return render_doc(
         request,
         "overview.html",
@@ -73,7 +75,7 @@ def doc_chapter(request: WSGIRequest, part, chapter, ajax: bool = False) -> DocR
     * Introduction (in part Manual)
     * Procedural Programming (in part Reference of Built-in Symbols)
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     chapter = documentation.get_chapter(part, chapter)
     if not chapter:
         raise Http404
@@ -96,7 +98,7 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse:
     * Manual
     * Reference of Built-in Symbols
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     part = documentation.get_part(part)
     if not part:
         raise Http404
@@ -113,7 +115,7 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse:
 
 
 def doc_search(request: WSGIRequest) -> DocResponse:
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     query = request.GET.get("query", "")
     result = documentation.search(query)
     if len([item for exact, item in result if exact]) <= 1:
@@ -170,7 +172,7 @@ def doc_section(
     * A list of builtin-functions under a Guide Section. For example: Color Directives.
       The guide section here would be Colors.
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     section_obj = documentation.get_section(part, chapter, section)
     if not section_obj:
         raise Http404
@@ -204,7 +206,7 @@ def doc_subsection(
     organized in a guide section are tagged as a section rather than a
     subsection.)
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     subsection_obj = documentation.get_subsection(part, chapter, section, subsection)
     if not subsection_obj:
         raise Http404
@@ -240,7 +242,7 @@ def render_doc(
     If ``ajax`` is True the should the ajax URI prefix, e.g. " it we pass the result
 
     """
-    check_for_pymathics_load()
+    check_for_new_load_modules()
     object = context.get("object")
     context.update(
         {