From 451f4d7afde1fa7f212cc835af9245310a75289e Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 19 Jan 2024 12:46:21 -0500 Subject: [PATCH 1/4] Bump copyright --- mathics_django/web/templates/about.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mathics_django/web/templates/about.html b/mathics_django/web/templates/about.html index 45a12a6c3..e390df2fc 100644 --- a/mathics_django/web/templates/about.html +++ b/mathics_django/web/templates/about.html @@ -165,7 +165,7 @@

Connection Information

Mathics3 is a general-purpose computer algebra system.

-

Copyright (C) 2021-2023 The Mathics3 Team


+

Copyright (C) 2021-2024 The Mathics3 Team


This program comes with ABSOLUTELY NO WARRANTY; This is free software, and you are welcome to redistribute it From a2be2e022577d398b6174c9e863707639630f51a Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 2 Feb 2024 16:44:08 -0500 Subject: [PATCH 2/4] Changes to go track changes in core PR #984 --- mathics_django/doc/__init__.py | 4 +- mathics_django/doc/django_doc.py | 162 +++------ mathics_django/docpipeline.py | 481 +++----------------------- mathics_django/settings.py | 2 +- mathics_django/web/controllers/doc.py | 40 ++- 5 files changed, 127 insertions(+), 562 deletions(-) diff --git a/mathics_django/doc/__init__.py b/mathics_django/doc/__init__.py index 7e1bb0b8b..0ec4b2531 100644 --- a/mathics_django/doc/__init__.py +++ b/mathics_django/doc/__init__.py @@ -8,6 +8,6 @@ import_and_load_builtins() # FIXME: should we really do this here? -from mathics_django.doc.django_doc import MathicsDjangoDocumentation +from mathics_django.doc.django_doc import DjangoDocumentation -documentation = MathicsDjangoDocumentation() +documentation = DjangoDocumentation() diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py index ed676d06c..392013669 100644 --- a/mathics_django/doc/django_doc.py +++ b/mathics_django/doc/django_doc.py @@ -11,12 +11,13 @@ from mathics import settings from mathics.doc.common_doc import ( DocChapter, - DocGuideSection, + DocPart, + DocSection, + DocSubsection, DocTest, DocTests, DocText, Documentation, - Tests, XMLDoc, gather_tests, get_results_by_test, @@ -28,8 +29,8 @@ from mathics_django.settings import get_doctest_html_data_path # FIXME: remove globalness +doctest_html_data_path = get_doctest_html_data_path(should_be_readable=True) try: - doctest_html_data_path = get_doctest_html_data_path(should_be_readable=True) with open(doctest_html_data_path, "rb") as doctest_html_data_file: doc_data = pickle.load(doctest_html_data_file) except IOError: @@ -37,12 +38,16 @@ doc_data = {} -class DjangoDocElement(object): +class DjangoDocElement: + """ + Adds some HTML functions onto existing Django Document Elements. + """ + def href(self, ajax=False): if ajax: - return "javascript:loadDoc('%s')" % self.get_uri() + return f"javascript:loadDoc('{self.get_uri()}')" else: - return "/doc%s" % self.get_uri() + return f"/doc{self.get_uri()}" def get_prev(self): return self.get_prev_next()[0] @@ -65,58 +70,27 @@ def get_title_html(self): class DjangoDocumentation(Documentation, DjangoDocElement): + def __init__(self): + super(DjangoDocumentation, self).__init__() + self.doc_class = DjangoDoc + self.doc_dir = settings.DOC_DIR + self.chapter_class = DjangoDocChapter + self.guide_section_class = DjangoDocGuideSection + self.part_class = DjangoDocPart + self.section_class = DjangoDocSection + self.subsection_class = DjangoDocSubsection + + self.gather_doctest_data() + self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL + self.pymathics_doc_loaded = False + self.doc_data_file = settings.get_doctest_latex_data_path( + should_be_readable=True + ) + self.title = "Overview" + def __str__(self): return "\n\n\n".join(str(part) for part in self.parts) - def get_tests(self): - for part in self.parts: - for chapter in sorted_chapters(part.chapters): - tests = chapter.doc.get_tests() - if tests: - yield Tests(part.title, chapter.title, "", tests) - for section in chapter.sections: - if section.installed: - if isinstance(section, DocGuideSection): - for docsection in section.subsections: - for docsubsection in docsection.subsections: - # FIXME: Something is weird here - # where tests for subsection items - # appear not as a collection but - # individually and need to be - # iterated below. Probably some - # other code is faulty and when - # fixed the below loop and - # collection into doctest_list[] - # will be removed. - doctest_list = [] - index = 1 - for doctests in docsubsection.items: - doctest_list += list(doctests.get_tests()) - for test in doctest_list: - test.index = index - index += 1 - - if doctest_list: - yield Tests( - section.chapter.part.title, - section.chapter.title, - docsubsection.title, - doctest_list, - ) - else: - tests = section.doc.get_tests() - if tests: - yield Tests( - part.title, chapter.title, section.title, tests - ) - pass - pass - pass - pass - pass - pass - return - def get_uri(self) -> str: return "/" @@ -174,33 +148,17 @@ def search_sections(section, result): return sorted_results -class MathicsDjangoDocumentation(DjangoDocumentation): - def __init__(self, want_sorting=True): - - self.doc_chapter_fn = DjangoDocChapter - self.doc_dir = settings.DOC_DIR - self.doc_fn = DjangoDoc - self.doc_guide_section_fn = DjangoDocGuideSection - self.doc_part_fn = DjangoDocPart - self.doc_section_fn = DjangoDocSection - self.doc_subsection_fn = DjangoDocSubsection - self.parts = [] - self.parts_by_slug = {} - self.title = "Overview" - - self.gather_doctest_data() - - class DjangoDoc(XMLDoc): - def __init__(self, doc, title, section): + def __init__(self, doc, title, section, key_prefix=None): self.title = title - if section: - chapter = section.chapter - part = chapter.part - # Note: we elide section.title - key_prefix = (part.title, chapter.title, title) - else: - key_prefix = None + if key_prefix is None: + if section is not None: + chapter = section.chapter + part = chapter.part + # Note: we elide section.title + key_prefix = (part.title, chapter.title, title) + else: + key_prefix = None self.rawdoc = doc self.items = gather_tests( @@ -218,7 +176,6 @@ def get_tests(self): return tests def html(self): - counters = {} items = [item for item in self.items if not item.is_private()] title_line = self.title + "\n" if len(items) and items[0].text.startswith(title_line): @@ -227,7 +184,7 @@ def html(self): # Or that is the intent. This code is a bit hacky. items[0].text = items[0].text[len(title_line) :] - text = "\n".join(item.html(counters) for item in items if not item.is_private()) + text = "\n".join(item.html() for item in items if not item.is_private()) if text == "": # HACK ALERT if text is "" we may have missed some test markup. return mark_safe(escape_html(self.rawdoc)) @@ -247,28 +204,16 @@ def get_uri(self) -> str: return f"/{self.part.slug}/{self.slug}/" -class DjangoDocPart(DjangoDocElement): +class DjangoDocPart(DocPart, DjangoDocElement): def __init__(self, doc, title, is_reference=False): - self.doc = doc - self.title = title - self.slug = slugify(title) - self.chapters = [] - self.chapters_by_slug = {} - self.is_reference = is_reference - self.is_appendix = False - doc.parts_by_slug[self.slug] = self - - def __str__(self): - return "%s\n\n%s" % ( - self.title, - "\n".join(str(chapter) for chapter in sorted_chapters(self.chapters)), - ) + super(DjangoDocPart, self).__init__(doc, title, is_reference) + self.chapter_class = DjangoDocChapter def get_collection(self): """Return a list of parts in this doc""" return self.doc.parts - def html(self, counters=None): + def html(self): if len(self.tests) == 0: return "\n" return '

' % ( @@ -281,7 +226,7 @@ def get_uri(self) -> str: return f"/{self.slug}/" -class DjangoDocSection(DjangoDocElement): +class DjangoDocSection(DocSection, DjangoDocElement): """An object for a Django Documented Section. A Section is part of a Chapter. It can contain subsections. """ @@ -308,8 +253,7 @@ def __init__( if text.count("
") != text.count("
"): raise ValueError( - "Missing opening or closing
tag in " - "{} documentation".format(title) + f"Missing opening or closing
tag in {title} documentation" ) # Needs to come after self.chapter is initialized since @@ -342,7 +286,7 @@ def get_uri(self) -> str: return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}/" -class DjangoDocGuideSection(DjangoDocSection): +class DjangoDocGuideSection(DjangoDocSection, DjangoDocElement): """An object for a Django Documented Guide Section. A Guide Section is part of a Chapter. "Colors" or "Special Functions" are examples of Guide Sections, and each contains a number of Sections. @@ -370,8 +314,7 @@ def __init__( if text.count("
") != text.count("
"): raise ValueError( - "Missing opening or closing
tag in " - "{} documentation".format(title) + f"Missing opening or closing
tag in {title} documentation" ) # print("YYY Adding section", title) chapter.sections_by_slug[self.slug] = self @@ -381,7 +324,7 @@ def get_uri(self) -> str: return f"/{self.chapter.part.slug}/{self.chapter.slug}/guide/" -class DjangoDocSubsection(DjangoDocElement): +class DjangoDocSubsection(DocSubsection, DjangoDocElement): """An object for a Django Documented Subsection. A Subsection is part of a Section. """ @@ -451,8 +394,7 @@ def __init__( if text.count("
") != text.count("
"): raise ValueError( - "Missing opening or closing
tag in " - "{} documentation".format(title) + f"Missing opening or closing
tag in {title} documentation" ) self.section.subsections_by_slug[self.slug] = self @@ -521,7 +463,7 @@ def html(self) -> str: class DjangoDocTests(DocTests): - def html(self, counters=None): + def html(self): if len(self.tests) == 0: return "\n" return '
    %s
' % ( @@ -532,6 +474,6 @@ def html(self, counters=None): class DjangoDocText(DocText): - def html(self, counters=None) -> str: - result = escape_html(self.text, counters=counters) + def html(self) -> str: + result = escape_html(self.text) return result diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py index 04a1854f6..7d0b96418 100644 --- a/mathics_django/docpipeline.py +++ b/mathics_django/docpipeline.py @@ -1,412 +1,42 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# FIXME: combine with same thing in Mathics Django +# FIXME: combine with same thing in Mathics core """ Does 2 things which can either be done independently or as a pipeline: -1. Extracts tests and runs them from static mdoc files and docstrings from Mathics - built-in functions +1. Extracts tests and runs them from static mdoc files and docstrings from + Mathics built-in functions 2. Creates/updates internal documentation data """ -import os -import os.path as osp import pickle -import re -import sys from argparse import ArgumentParser from datetime import datetime import mathics -from mathics import version_string +import mathics.docpipeline as md from mathics.core.definitions import Definitions -from mathics.core.evaluation import Evaluation, Output from mathics.core.load_builtin import ( builtins_by_module, builtins_dict, import_and_load_builtins, ) -from mathics.core.parser import MathicsSingleLineFeeder +from mathics.docpipeline import ( + MAX_TESTS, + create_output, + open_ensure_dir, + print_and_log, + test_all, + test_chapters, + test_sections, + write_doctest_data, +) from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule -from mathics_django.doc import MathicsDjangoDocumentation +from mathics_django.doc import DjangoDocumentation from mathics_django.settings import get_doctest_html_data_path -builtins = builtins_dict(builtins_by_module) - - -class TestOutput(Output): - def max_stored_size(self): - return None - - -sep = "-" * 70 + "\n" - -# Global variables -definitions = None -documentation = None -check_partial_enlapsed_time = False -logfile = None - - -MAX_TESTS = 100000 # Number than the total number of tests - - -def print_and_log(*args): - a = [a.decode("utf-8") if isinstance(a, bytes) else a for a in args] - string = "".join(a) - print(string) - if logfile: - logfile.write(string) - - -def compare(result, wanted) -> bool: - if result == wanted: - return True - - if result is None or wanted is None: - return False - result = result.splitlines() - wanted = wanted.splitlines() - if result == [] and wanted == ["#<--#"]: - return True - if len(result) != len(wanted): - return False - for r, w in zip(result, wanted): - wanted_re = re.escape(w.strip()) - wanted_re = wanted_re.replace("\\.\\.\\.", ".*?") - wanted_re = "^%s$" % wanted_re - if not re.match(wanted_re, r.strip()): - return False - return True - - -stars = "*" * 10 - - -def test_case(test, tests, index=0, subindex=0, quiet=False, section=None) -> bool: - global check_partial_enlapsed_time - test, wanted_out, wanted = test.test, test.outs, test.result - - def fail(why): - part, chapter, section = tests.part, tests.chapter, tests.section - print_and_log( - f"""{sep}Test failed: {section} in {part} / {chapter} -{part} -{why} -""".encode( - "utf-8" - ) - ) - return False - - if not quiet: - if section: - print(f"{stars} {tests.chapter} / {section} {stars}".encode("utf-8")) - print(f"{index:4d} ({subindex:2d}): TEST {test}".encode("utf-8")) - - feeder = MathicsSingleLineFeeder(test, "") - evaluation = Evaluation(definitions, catch_interrupt=False, output=TestOutput()) - try: - time_parsing = datetime.now() - query = evaluation.parse_feeder(feeder) - if check_partial_enlapsed_time: - print(" parsing took", datetime.now() - time_parsing) - if query is None: - # parsed expression is None - result = None - out = evaluation.out - else: - result = evaluation.evaluate(query) - if check_partial_enlapsed_time: - print(" evaluation took", datetime.now() - time_parsing) - out = result.out - result = result.result - except Exception as exc: - fail("Exception %s" % exc) - info = sys.exc_info() - sys.excepthook(*info) - return False - - time_comparing = datetime.now() - comparison_result = compare(result, wanted) - - if check_partial_enlapsed_time: - print(" comparison took ", datetime.now() - time_comparing) - if not comparison_result: - print("result =!=wanted") - fail_msg = "Result: %s\nWanted: %s" % (result, wanted) - if out: - fail_msg += "\nAdditional output:\n" - fail_msg += "\n".join(str(o) for o in out) - return fail(fail_msg) - output_ok = True - time_comparing = datetime.now() - if len(wanted_out) == 1 and wanted_out[0].text == "...": - # If we have ... don't check - pass - elif len(out) != len(wanted_out): - # Mismatched number of output lines and we don't have "..." - output_ok = False - else: - # Need to check all output line by line - for got, wanted in zip(out, wanted_out): - if not got == wanted and wanted.text != "...": - output_ok = False - break - if check_partial_enlapsed_time: - print(" comparing messages took ", datetime.now() - time_comparing) - if not output_ok: - return fail( - "Output:\n%s\nWanted:\n%s" - % ("\n".join(str(o) for o in out), "\n".join(str(o) for o in wanted_out)) - ) - return True - - -def test_tests( - tests, - index, - quiet=False, - stop_on_failure=False, - start_at=0, - max_tests=MAX_TESTS, - excludes=[], -): - definitions.reset_user_definitions() - total = failed = skipped = 0 - failed_symbols = set() - section = tests.section - if section in excludes: - return total, failed, len(tests.tests), failed_symbols, index - count = 0 - for subindex, test in enumerate(tests.tests): - index += 1 - if test.ignore: - continue - if index < start_at: - skipped += 1 - continue - elif count >= max_tests: - break - - total += 1 - count += 1 - if not test_case(test, tests, index, subindex + 1, quiet, section): - failed += 1 - failed_symbols.add((tests.part, tests.chapter, tests.section)) - if stop_on_failure: - break - - section = None - return total, failed, skipped, failed_symbols, index - - -# FIXME: move this to common routine -def create_output(tests, doc_data, format="xml"): - definitions.reset_user_definitions() - for test in tests.tests: - if test.private: - continue - key = test.key - evaluation = Evaluation( - definitions, format=format, catch_interrupt=True, output=TestOutput() - ) - try: - result = evaluation.parse_evaluate(test.test) - except: # noqa - result = None - if result is None: - result = [] - else: - result = [result.get_data()] - doc_data[key] = { - "query": test.test, - "results": result, - } - - -def test_chapters( - chapters: set, - quiet=False, - stop_on_failure=False, - generate_output=False, - reload=False, - want_sorting=False, -): - if documentation is None: - print_and_log("documentation is not loaded.") - return - failed = 0 - index = 0 - chapter_names = ", ".join(chapters) - print(f"Testing chapter(s): {chapter_names}") - output_data = load_doc_data() if reload else {} - prev_key = [] - for tests in documentation.get_tests(): - if tests.chapter in sorted(chapters): - for test in tests.tests: - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - print(f'Testing section: {" / ".join(key)}') - index = 0 - if test.ignore: - continue - index += 1 - if not test_case(test, tests, index, quiet=quiet): - failed += 1 - if stop_on_failure: - break - if generate_output and failed == 0: - create_output(tests, output_data) - - print() - if index == 0: - print_and_log(f"No chapters found named {chapter_names}.") - elif failed > 0: - if not (keep_going and format == "xml"): - print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else "")) - else: - print_and_log("All tests passed.") - - -def test_sections( - sections: set, - quiet=False, - stop_on_failure=False, - generate_output=False, - reload=False, - want_sorting=False, -): - if documentation is None: - print_and_log("documentation is not loaded.") - return - failed = 0 - index = 0 - section_names = ", ".join(sections) - print(f"Testing section(s): {section_names}") - sections |= {"$" + s for s in sections} - output_data = load_doc_data() if reload else {} - prev_key = [] - for tests in documentation.get_tests(): - if tests.section in sections: - for test in tests.tests: - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - print(f'Testing section: {" / ".join(key)}') - index = 0 - if test.ignore: - continue - index += 1 - if not test_case(test, tests, index, quiet=quiet): - failed += 1 - if stop_on_failure: - break - if generate_output and failed == 0: - create_output(tests, output_data) - - print() - if index == 0: - print_and_log(f"No sections found named {section_names}.") - elif failed > 0: - if not (keep_going and format == "xml"): - print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else "")) - else: - print_and_log("All tests passed.") - if generate_output and (failed == 0 or keep_going): - save_doctest_data(output_data) - - -def open_ensure_dir(f, *args, **kwargs): - try: - return open(f, *args, **kwargs) - except (IOError, OSError): - d = osp.dirname(f) - if d and not osp.exists(d): - os.makedirs(d) - return open(f, *args, **kwargs) - - -def test_all( - quiet=False, - generate_output=False, - stop_on_failure=False, - start_at=0, - count=MAX_TESTS, - texdatafolder=None, - doc_even_if_error=False, - excludes=[], - want_sorting=False, -): - if not quiet: - print(f"Testing {version_string}") - - if documentation is None: - print_and_log("documentation is not loaded.") - return - - try: - index = 0 - total = failed = skipped = 0 - failed_symbols = set() - output_data = {} - for tests in documentation.get_tests(): - sub_total, sub_failed, sub_skipped, symbols, index = test_tests( - tests, - index, - quiet=quiet, - stop_on_failure=stop_on_failure, - start_at=start_at, - max_tests=count, - excludes=excludes, - ) - if generate_output: - create_output(tests, output_data) - total += sub_total - failed += sub_failed - skipped += sub_skipped - failed_symbols.update(symbols) - if sub_failed and stop_on_failure: - break - if total >= count: - break - builtin_total = len(builtins) - except KeyboardInterrupt: - print("\nAborted.\n") - return - - if failed > 0: - print(sep) - if count == MAX_TESTS: - print_and_log( - "%d Tests for %d built-in symbols, %d passed, %d failed, %d skipped." - % (total, builtin_total, total - failed - skipped, failed, skipped) - ) - else: - print_and_log( - "%d Tests, %d passed, %d failed, %d skipped." - % (total, total - failed, failed, skipped) - ) - if failed_symbols: - if stop_on_failure: - print_and_log("(not all tests are accounted for due to --stop-on-failure)") - print_and_log("Failed:") - for part, chapter, section in sorted(failed_symbols): - print_and_log(" - %s in %s / %s" % (section, part, chapter)) - - if generate_output and (failed == 0 or doc_even_if_error): - save_doctest_data(output_data) - return True - - if failed == 0: - print("\nOK") - else: - print("\nFAILED") - return sys.exit(1) # Travis-CI knows the tests have failed - def load_doc_data(): doc_html_data_path = get_doctest_html_data_path(should_be_readable=True) @@ -437,36 +67,9 @@ def save_doctest_data(output_data): pickle.dump(output_data, output_file, 4) -def write_doctest_data(quiet=False, reload=False): - """ - Write internal (pickled) doc files and example data in docstrings. - """ - if documentation is None: - print_and_log("documentation is not loaded.") - return - - if not quiet: - print(f"Extracting internal doc data for {version_string}") - print("This may take a while...") - - try: - output_data = load_doc_data() if reload else {} - for tests in documentation.get_tests(): - create_output(tests, output_data) - except KeyboardInterrupt: - print("\nAborted.\n") - return - - print("done.\n") - save_doctest_data(output_data) - - def main(): - global check_partial_enlapsed_time - global definitions - import_and_load_builtins() - definitions = Definitions(add_builtin=True) + md.DEFINITIONS = Definitions(add_builtin=True) parser = ArgumentParser(description="Mathics test suite.", add_help=False) parser.add_argument( @@ -497,7 +100,7 @@ def main(): default="", dest="exclude", metavar="SECTION", - help="excude SECTION(s). " + help="exclude SECTION(s). " "You can list multiple sections by adding a comma (and no space) in between section names.", ) parser.add_argument( @@ -519,7 +122,7 @@ def main(): parser.add_argument( "--time-each", "-d", - dest="enlapsed_times", + dest="elapsed_times", action="store_true", help="check the time that take each test to parse, evaluate and compare.", ) @@ -573,39 +176,29 @@ def main(): default=MAX_TESTS, help="run only N tests", ) - # FIXME: there is some weird interacting going on with - # mathics when tests in sorted order. Some of the Plot - # show a noticeable 2 minute delay in processing. - # I think the problem is in Mathics itself rather than - # sorting, but until we figure that out, use - # sort as an option only. For normal testing we don't - # want it for speed. But for document building which is - # rarely done, we do want sorting of the sections and chapters. parser.add_argument( - "--want-sorting", - dest="want_sorting", + "--show-statistics", action="store_true", - help="Sort chapters and sections", + help="print cache statistics", ) - global logfile + global LOGFILE args = parser.parse_args() - if args.enlapsed_times: - check_partial_enlapsed_time = True + if args.elapsed_times: + md.CHECK_PARTIAL_ELAPSED_TIME = True # If a test for a specific section is called # just test it if args.logfilename: - logfile = open(args.logfilename, "wt") + md.LOGFILE = open(args.logfilename, "wt") - global documentation - documentation = MathicsDjangoDocumentation() + md.DOCUMENTATION = DjangoDocumentation() # LoadModule Mathics3 modules if args.pymathics: for module_name in args.pymathics.split(","): try: - eval_LoadModule(module_name, definitions) + eval_LoadModule(module_name, md.DEFINITIONS) except PyMathicsLoadException: print(f"Python module {module_name} is not a Mathics3 module.") @@ -614,11 +207,15 @@ def main(): else: print(f"Mathics3 Module {module_name} loaded") - documentation.gather_doctest_data() + md.DOCUMENTATION.gather_doctest_data() + + start_time = None + total = 0 if args.sections: sections = set(args.sections.split(",")) + start_time = datetime.now() test_sections( sections, stop_on_failure=args.stop_on_failure, @@ -626,9 +223,10 @@ def main(): reload=args.reload, ) elif args.chapters: + start_time = datetime.now() chapters = set(args.chapters.split(",")) - test_chapters( + total = test_chapters( chapters, stop_on_failure=args.stop_on_failure, reload=args.reload ) else: @@ -636,13 +234,12 @@ def main(): write_doctest_data( quiet=args.quiet, reload=args.reload, - want_sorting=args.want_sorting, ) else: excludes = set(args.exclude.split(",")) start_at = args.skip + 1 start_time = datetime.now() - test_all( + total = test_all( quiet=args.quiet, generate_output=args.output, stop_on_failure=args.stop_on_failure, @@ -650,12 +247,16 @@ def main(): count=args.count, doc_even_if_error=args.keep_going, excludes=excludes, - want_sorting=args.want_sorting, ) end_time = datetime.now() print("Tests took ", end_time - start_time) - if logfile: - logfile.close() + + if total > 0 and start_time is not None: + end_time = datetime.now() + print("Test evalation took ", end_time - start_time) + + if md.LOGFILE: + md.LOGFILE.close() if __name__ == "__main__": diff --git a/mathics_django/settings.py b/mathics_django/settings.py index 96fa92fd8..d618d8013 100644 --- a/mathics_django/settings.py +++ b/mathics_django/settings.py @@ -74,7 +74,7 @@ def get_bool_from_environment(env_var: str, default_value: str): ) # We need another version as a fallback, and that is distributed with the -# package. It is note user writable and not in the user space. +# package. It is not user writable and not in the user space. DOC_SYSTEM_HTML_DATA_PATH = os.environ.get( "DOC_SYSTEM_HTML_DATA_PATH", osp.join(ROOT_DIR, "doc", "doc_html_data.pcl") ) diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py index 191be91f6..5e7bbcb58 100644 --- a/mathics_django/web/controllers/doc.py +++ b/mathics_django/web/controllers/doc.py @@ -8,25 +8,48 @@ from django.core.handlers.wsgi import WSGIRequest from django.http import Http404, HttpResponse from django.shortcuts import render -from mathics.eval.pymathics import pymathics_modules +from mathics.doc.common_doc import get_module_doc, mathics3_module_part +from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules from mathics_django.doc import documentation from mathics_django.doc.django_doc import ( DjangoDocChapter, DjangoDocPart, DjangoDocSection, - MathicsDjangoDocumentation, ) from mathics_django.web.views import JsonResponse DocResponse = Union[HttpResponse, JsonResponse] +seen_pymathics_modules = copy(pymathics_modules) + def check_for_pymathics_load(): + global seen_pymathics_modules if seen_pymathics_modules != pymathics_modules: - # print("XXX refresh pymathics doc") - global documentation - documentation = MathicsDjangoDocumentation() + print("XXX refresh pymathics doc", pymathics_modules) + new_modules = pymathics_modules - seen_pymathics_modules + for new_module in new_modules: + title, _ = get_module_doc(new_module) + chapter = mathics3_module_part.doc.gather_chapter_doc_fn( + mathics3_module_part, + title, + mathics3_module_part.doc, + ) + from trepan.api import debug + + debug() + submodule_names_seen = set() + chapter.doc.doc_chapter( + new_module, + mathics3_module_part, + pymathics_builtins_by_module, + seen_pymathics_modules, + submodule_names_seen, + ) + chapter.get_tests() + seen_pymathics_modules = copy(pymathics_modules) + pass def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse: @@ -44,7 +67,9 @@ def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse: def doc_chapter(request: WSGIRequest, part, chapter, ajax: bool = False) -> DocResponse: """ - Produces HTML via jinja templating for a chapter. Some examples of Chapters: + Produces HTML via jinja templating for a chapter. Some examples of + Chapters: + * Introduction (in part Manual) * Procedural Programming (in part Reference of Built-in Symbols) """ @@ -87,9 +112,6 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse: ) -seen_pymathics_modules = copy(pymathics_modules) - - def doc_search(request: WSGIRequest) -> DocResponse: check_for_pymathics_load() query = request.GET.get("query", "") From 9182be19e93ec1d492162b3f9efdc424cc21dc9b Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 9 Feb 2024 03:12:17 -0500 Subject: [PATCH 3/4] Get Mathics3 Modules to loading properly ... This needs the doc-code-rebased-rebased branch of Mathics core. --- .github/workflows/osx.yaml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- mathics_django/doc/django_doc.py | 6 +-- mathics_django/docpipeline.py | 20 ++------- mathics_django/web/controllers/doc.py | 64 ++++++++++++++------------- 6 files changed, 42 insertions(+), 54 deletions(-) diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml index e8c7af1ff..6257100bb 100644 --- a/.github/workflows/osx.yaml +++ b/.github/workflows/osx.yaml @@ -30,7 +30,7 @@ jobs: # For testing 3.11 we need to do something like the below until the next Mathics3 is released # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full] - git clone https://github.com/Mathics3/mathics-core + git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core (cd mathics-core && pip3 install -e .[full]) (cd mathics-core && bash ./admin-tools/make-op-tables.sh) - name: Install Mathics3 Django diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 11a13af08..a38b17ab6 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -29,7 +29,7 @@ jobs: # For testing 3.11 we need to do something like the below until the next Mathics3 is released # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full] - git clone https://github.com/Mathics3/mathics-core + git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core (cd mathics-core && pip3 install -e .[full]) (cd mathics-core && bash ./admin-tools/make-op-tables.sh) - name: Install Mathics3 Django diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 1987c8612..1aa05578c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -32,7 +32,7 @@ jobs: # For testing 3.11 we need to do something like the below until the next Mathics3 is released # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full] - git clone https://github.com/Mathics3/mathics-core + git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core bash -c '(cd mathics-core && pip3 install -e .[full])' bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)' - name: Install Mathics3 Django diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py index 392013669..fac8d0e28 100644 --- a/mathics_django/doc/django_doc.py +++ b/mathics_django/doc/django_doc.py @@ -18,7 +18,7 @@ DocTests, DocText, Documentation, - XMLDoc, + DocumentationEntry, gather_tests, get_results_by_test, sorted_chapters, @@ -80,7 +80,7 @@ def __init__(self): self.section_class = DjangoDocSection self.subsection_class = DjangoDocSubsection - self.gather_doctest_data() + self.load_documentation_sources() self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL self.pymathics_doc_loaded = False self.doc_data_file = settings.get_doctest_latex_data_path( @@ -148,7 +148,7 @@ def search_sections(section, result): return sorted_results -class DjangoDoc(XMLDoc): +class DjangoDoc(DocumentationEntry): def __init__(self, doc, title, section, key_prefix=None): self.title = title if key_prefix is None: diff --git a/mathics_django/docpipeline.py b/mathics_django/docpipeline.py index 7d0b96418..cfc0d5d23 100644 --- a/mathics_django/docpipeline.py +++ b/mathics_django/docpipeline.py @@ -17,16 +17,10 @@ import mathics import mathics.docpipeline as md from mathics.core.definitions import Definitions -from mathics.core.load_builtin import ( - builtins_by_module, - builtins_dict, - import_and_load_builtins, -) +from mathics.core.load_builtin import import_and_load_builtins +from mathics.doc.utils import open_ensure_dir from mathics.docpipeline import ( MAX_TESTS, - create_output, - open_ensure_dir, - print_and_log, test_all, test_chapters, test_sections, @@ -38,13 +32,6 @@ from mathics_django.settings import get_doctest_html_data_path -def load_doc_data(): - doc_html_data_path = get_doctest_html_data_path(should_be_readable=True) - print(f"Loading internal document data from {doc_html_data_path}") - with open_ensure_dir(doc_html_data_path, "rb") as doc_data_file: - return pickle.load(doc_data_file) - - def save_doctest_data(output_data): """ Save doctest tests and test results to a Python PCL file. @@ -207,7 +194,7 @@ def main(): else: print(f"Mathics3 Module {module_name} loaded") - md.DOCUMENTATION.gather_doctest_data() + # md.DOCUMENTATION.load_documentation_sources() start_time = None total = 0 @@ -244,7 +231,6 @@ def main(): generate_output=args.output, stop_on_failure=args.stop_on_failure, start_at=start_at, - count=args.count, doc_even_if_error=args.keep_going, excludes=excludes, ) diff --git a/mathics_django/web/controllers/doc.py b/mathics_django/web/controllers/doc.py index 5e7bbcb58..9c10e4d8e 100644 --- a/mathics_django/web/controllers/doc.py +++ b/mathics_django/web/controllers/doc.py @@ -8,7 +8,8 @@ from django.core.handlers.wsgi import WSGIRequest from django.http import Http404, HttpResponse from django.shortcuts import render -from mathics.doc.common_doc import get_module_doc, mathics3_module_part +from mathics.doc.common_doc import MATHICS3_MODULES_TITLE +from mathics.doc.utils import slugify from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules from mathics_django.doc import documentation @@ -23,37 +24,38 @@ seen_pymathics_modules = copy(pymathics_modules) +MATHICS3_MODULES_SLUG = slugify(MATHICS3_MODULES_TITLE) -def check_for_pymathics_load(): + +def check_for_new_load_modules(): + """ + See if we have laoded any new Mathics3 modules since the last time + we checked. If so get an add the documenation for that. + """ global seen_pymathics_modules if seen_pymathics_modules != pymathics_modules: - print("XXX refresh pymathics doc", pymathics_modules) - new_modules = pymathics_modules - seen_pymathics_modules - for new_module in new_modules: - title, _ = get_module_doc(new_module) - chapter = mathics3_module_part.doc.gather_chapter_doc_fn( - mathics3_module_part, - title, - mathics3_module_part.doc, - ) - from trepan.api import debug - - debug() - submodule_names_seen = set() - chapter.doc.doc_chapter( - new_module, - mathics3_module_part, - pymathics_builtins_by_module, - seen_pymathics_modules, - submodule_names_seen, - ) - chapter.get_tests() + mathics3_module_part = documentation.parts_by_slug.get( + MATHICS3_MODULES_SLUG, None + ) + if mathics3_module_part is None: + print("Something is wrong: mathics3_module variable should not be None") + return + else: + # The "Mathics3 modules" part already exists; add the new chapters. + new_modules = pymathics_modules - seen_pymathics_modules + for new_module in new_modules: + chapter = documentation.doc_chapter( + new_module, mathics3_module_part, pymathics_builtins_by_module + ) + mathics3_module_part.chapters.append(chapter) + pass + pass seen_pymathics_modules = copy(pymathics_modules) - pass + return def doc(request: WSGIRequest, ajax: bool = False) -> DocResponse: - check_for_pymathics_load() + check_for_new_load_modules() return render_doc( request, "overview.html", @@ -73,7 +75,7 @@ def doc_chapter(request: WSGIRequest, part, chapter, ajax: bool = False) -> DocR * Introduction (in part Manual) * Procedural Programming (in part Reference of Built-in Symbols) """ - check_for_pymathics_load() + check_for_new_load_modules() chapter = documentation.get_chapter(part, chapter) if not chapter: raise Http404 @@ -96,7 +98,7 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse: * Manual * Reference of Built-in Symbols """ - check_for_pymathics_load() + check_for_new_load_modules() part = documentation.get_part(part) if not part: raise Http404 @@ -113,7 +115,7 @@ def doc_part(request: WSGIRequest, part, ajax: bool = False) -> DocResponse: def doc_search(request: WSGIRequest) -> DocResponse: - check_for_pymathics_load() + check_for_new_load_modules() query = request.GET.get("query", "") result = documentation.search(query) if len([item for exact, item in result if exact]) <= 1: @@ -170,7 +172,7 @@ def doc_section( * A list of builtin-functions under a Guide Section. For example: Color Directives. The guide section here would be Colors. """ - check_for_pymathics_load() + check_for_new_load_modules() section_obj = documentation.get_section(part, chapter, section) if not section_obj: raise Http404 @@ -204,7 +206,7 @@ def doc_subsection( organized in a guide section are tagged as a section rather than a subsection.) """ - check_for_pymathics_load() + check_for_new_load_modules() subsection_obj = documentation.get_subsection(part, chapter, section, subsection) if not subsection_obj: raise Http404 @@ -240,7 +242,7 @@ def render_doc( If ``ajax`` is True the should the ajax URI prefix, e.g. " it we pass the result """ - check_for_pymathics_load() + check_for_new_load_modules() object = context.get("object") context.update( { From 8c1096ea2038c1073fdd2897d23c92db1104c102 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 10 Feb 2024 01:30:28 -0500 Subject: [PATCH 4/4] Revise to current Mathics core master branch --- .github/workflows/osx.yaml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- mathics_django/doc/django_doc.py | 10 +++++++++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/osx.yaml b/.github/workflows/osx.yaml index 6257100bb..e8c7af1ff 100644 --- a/.github/workflows/osx.yaml +++ b/.github/workflows/osx.yaml @@ -30,7 +30,7 @@ jobs: # For testing 3.11 we need to do something like the below until the next Mathics3 is released # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full] - git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core + git clone https://github.com/Mathics3/mathics-core (cd mathics-core && pip3 install -e .[full]) (cd mathics-core && bash ./admin-tools/make-op-tables.sh) - name: Install Mathics3 Django diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a38b17ab6..11a13af08 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -29,7 +29,7 @@ jobs: # For testing 3.11 we need to do something like the below until the next Mathics3 is released # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full] - git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core + git clone https://github.com/Mathics3/mathics-core (cd mathics-core && pip3 install -e .[full]) (cd mathics-core && bash ./admin-tools/make-op-tables.sh) - name: Install Mathics3 Django diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 1aa05578c..1987c8612 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -32,7 +32,7 @@ jobs: # For testing 3.11 we need to do something like the below until the next Mathics3 is released # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full] # python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full] - git clone --branch doc-code-rebased-rebased https://github.com/Mathics3/mathics-core + git clone https://github.com/Mathics3/mathics-core bash -c '(cd mathics-core && pip3 install -e .[full])' bash -c '(cd mathics-core && bash ./admin-tools/make-op-tables.sh)' - name: Install Mathics3 Django diff --git a/mathics_django/doc/django_doc.py b/mathics_django/doc/django_doc.py index fac8d0e28..6dd6ed9da 100644 --- a/mathics_django/doc/django_doc.py +++ b/mathics_django/doc/django_doc.py @@ -205,9 +205,17 @@ def get_uri(self) -> str: class DjangoDocPart(DocPart, DjangoDocElement): + """ + Represents one of the main parts of the document customized for Django. Parts + can be loaded from a mdoc file, generated automatically from + the docstrings of Builtin objects under `mathics.builtin`, or loaded + as a Mathics3 module. + """ + + chapter_class = DjangoDocChapter + def __init__(self, doc, title, is_reference=False): super(DjangoDocPart, self).__init__(doc, title, is_reference) - self.chapter_class = DjangoDocChapter def get_collection(self): """Return a list of parts in this doc"""