diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6f6cfd..0aa17e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: stages: [commit] - id: end-of-file-fixer stages: [commit] + exclude: ChangeLog-spell-corrected.diff - id: trailing-whitespace - id: check-json exclude: mathics_scanner/data/characters.json diff --git a/CHANGES.rst b/CHANGES.rst index 74a4e4e..014eb47 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,16 @@ CHANGES ======= -Note: after the 1.3.0 release support for running on Python 3.6 will be dropped. Python -3.7 support will be dropped soon as well. +1.3.1 +------ + +Python 3.8 is now the minimum Python supported. Python 3.12 suported. +Various dependecies elsehwere force 3.8 or newer. + +* Packaging was redone to be able to support Python 3.12. +* Files now follow current Python black formatting and isort import ordering +* Some Python code linting 1.3.0 ------ diff --git a/ChangeLog-spell-corrected.diff b/ChangeLog-spell-corrected.diff new file mode 100644 index 0000000..4816c9a --- /dev/null +++ b/ChangeLog-spell-corrected.diff @@ -0,0 +1,131 @@ +--- ChangeLog-spell-corrected 2024-08-08 19:19:04.496248094 -0400 ++++ ChangeLog 2024-08-08 19:21:45.572417148 -0400 +@@ -97,7 +97,7 @@ + 2023-12-17 rocky + + * admin-tools/pyenv-versions, +- mathics_scanner/data/named-characters.yml, test/test_tokeniser.py: ++ mathics_scanner/data/named-characters.yml, test/test_tokeniser.py: + Administrivia: typos and bump versions + + 2023-08-23 rocky +@@ -425,18 +425,18 @@ + 2021-06-29 rocky + + * mathics_scanner/data/named-characters.yml, +- test/test_general_yaml_sanity.py, test/test_table_consistency.py: ++ test/test_general_yaml_sanity.py, test/test_table_consistency.py: + Appease current checker... The checker may change in the future. + + 2021-06-29 rocky + +- * mathics_scanner/data/named-characters.yml: We now have one more source ++ * mathics_scanner/data/named-characters.yml: Noe one more source + + 2021-06-29 rocky + + * mathics_scanner/data/named-characters.yml, + mathics_scanner/generate/build_tables.py: Add ApplyTo and reqularize +- unicode-equivalent Also adjust Tilde ++ unicode-equivalent Also adust Tilde + + 2021-06-28 rocky + +@@ -457,7 +457,7 @@ + + 2021-06-27 rocky + +- * CHANGES.rst, MANIFEST.in, mathics_scanner/version.py, setup.py: ++ * CHANGES.rst, MANIFEST.in, mathics_scanner/version.py, setup.py: + Get ready for release 1.2.3 + + 2021-06-26 rocky +@@ -547,7 +547,7 @@ + mathics_scanner/tokeniser.py, mathics_scanner/version.py, setup.py, + test/test_has_unicode_inverse_sanity.py, + test/test_letterlikes_sanity.py, test/test_table_consistency.py, +- test/test_translation_regressions.py, test/test_wl_to_ascii.py: ++ test/test_translation_regressions.py, test/test_wl_to_ascii.py: + Blacken files + + 2021-06-21 rocky +@@ -588,7 +588,7 @@ + + * .github/workflows/osx.yaml, .github/workflows/ubuntu.yaml, + .github/workflows/windows.yml, mathics_scanner/characters.py, +- mathics_scanner/version.py, requirements-dev.txt, setup.py: ++ mathics_scanner/version.py, requirements-dev.txt, setup.py: + Administrivia & make ujson optional * It's not clear that pyston-2.2 support ujson so make it optional * In github workflows use requirements-dev.txt * revise requirement-dev to remove click (in setup.py) and add ujson * setup.py make ujson optional * version.py: bump to dev + + 2021-05-14 rocky +@@ -788,7 +788,7 @@ + + * .github/workflows/osx.yaml, Makefile, + mathics_scanner/data/named-characters.yml, setup.py, +- test/test_general_yaml_sanity.py, test/test_table_consistency.py: ++ test/test_general_yaml_sanity.py, test/test_table_consistency.py: + Start tagging operators Note: use WL name when there is no conflict for name of symbol. A number of other small corrections: - Mark some symbols which are invertable when they are. However this should be redone and invertability should be detected automatically. - small correction in macos CI - AUTHORS.txt reduced to those who worked in this, rather than + Mathics in general - in tests prefer top-level tests when that is possible (i.e. test + is not parameterized) - use relative import as "test" can be ambiguous and lead to +@@ -889,7 +889,7 @@ + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * README.rst, implementation.rst, mathics_scanner/characters.py: ++ * README.rst, implementation.rst, mathics_scanner/characters.py: + Added an awfull lot of documentation + + 2021-01-31 Pablo Emilio Escobar Gaviria +@@ -922,7 +922,7 @@ + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * mathics_scanner/data/named-characters.yml, test/test_roundtrip.py: ++ * mathics_scanner/data/named-characters.yml, test/test_roundtrip.py: + Fixed another errors in the YAML table + + 2021-02-02 Pablo Emilio Escobar Gaviria +@@ -945,7 +945,7 @@ + + 2021-02-01 Pablo Emilio Escobar Gaviria + +- * mathics_scanner/errors.py, mathics_scanner/tokeniser.py: ++ * mathics_scanner/errors.py, mathics_scanner/tokeniser.py: + Documented stuff used by Mathics + + 2021-01-31 Pablo Emilio Escobar Gaviria +@@ -965,7 +965,7 @@ + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * test/test_general_yaml_sanity.py, test/test_letterlikes_sanity.py: ++ * test/test_general_yaml_sanity.py, test/test_letterlikes_sanity.py: + Improved the general tests for YAML + + 2021-01-31 Pablo Emilio Escobar Gaviria +@@ -994,22 +994,22 @@ + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: ++ * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: + Fixed typos + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: ++ * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: + Improved the sanity checks for has-unicode-inverse + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: ++ * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: + Fixed and improved the roundtrip tests + + 2021-01-31 Pablo Emilio Escobar Gaviria + +- * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: ++ * test/test_has_unicode_inverse_sanity.py, test/test_roundtrip.py: + Added sanity check for has-unicode-inverse and fixed the reound trip + tests diff --git a/Makefile b/Makefile index 294d9d6..15a6c71 100644 --- a/Makefile +++ b/Makefile @@ -82,3 +82,4 @@ rmChangeLog: #: Create a ChangeLog from git via git log and git2cl ChangeLog: rmChangeLog git log --pretty --numstat --summary | $(GIT2CL) >$@ + patch -R ChangeLog < ChangeLog-spell-corrected.diff diff --git a/admin-tools/pyenv-versions b/admin-tools/pyenv-versions index 405a4d5..b4703f5 100644 --- a/admin-tools/pyenv-versions +++ b/admin-tools/pyenv-versions @@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then echo "This script should be *sourced* rather than run directly through bash" exit 1 fi -export PYVERSIONS='3.8.18 3.9.18 3.10.13 pyston-2.3.5 3.11.8 3.12.2' +export PYVERSIONS='3.8 3.9 3.10 3.11 3.12' diff --git a/docs/source/conf.py b/docs/source/conf.py index 0db240a..4dca137 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,8 +17,8 @@ # -- Project information ----------------------------------------------------- project = "mathics-scanner" -copyright = "2021, The Mathics Team" -author = "The Mathics Team" +copyright = "2021, 2024 The Mathics Team" +author = "The Mathics3 Team" # The full version, including alpha/beta/rc tags release = "1.0.1" diff --git a/mathics_scanner/__init__.py b/mathics_scanner/__init__.py index 165e4b8..de83a8d 100644 --- a/mathics_scanner/__init__.py +++ b/mathics_scanner/__init__.py @@ -6,45 +6,44 @@ named characters, their Unicode/ASCII equivalents and code-points. """ -from mathics_scanner.version import __version__ - from mathics_scanner.characters import ( aliased_characters, named_characters, replace_unicode_with_wl, replace_wl_with_plain_text, ) - -# TODO: Move is_symbol_name to the characters module -from mathics_scanner.tokeniser import is_symbol_name, Tokeniser, Token from mathics_scanner.errors import ( - InvalidSyntaxError, IncompleteSyntaxError, + InvalidSyntaxError, ScanError, TranslateError, ) from mathics_scanner.feed import ( - LineFeeder, - SingleLineFeeder, FileLineFeeder, + LineFeeder, MultiLineFeeder, + SingleLineFeeder, ) +# TODO: Move is_symbol_name to the characters module +from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name +from mathics_scanner.version import __version__ + __all__ = [ - FileLineFeeder, - IncompleteSyntaxError, - InvalidSyntaxError, - LineFeeder, - MultiLineFeeder, - ScanError, - SingleLineFeeder, - Token, - Tokeniser, - TranslateError, - __version__, - aliased_characters, - is_symbol_name, - named_characters, - replace_unicode_with_wl, - replace_wl_with_plain_text, + "FileLineFeeder", + "IncompleteSyntaxError", + "InvalidSyntaxError", + "LineFeeder", + "MultiLineFeeder", + "ScanError", + "SingleLineFeeder", + "Token", + "Tokeniser", + "TranslateError", + "__version__", + "aliased_characters", + "is_symbol_name", + "named_characters", + "replace_unicode_with_wl", + "replace_wl_with_plain_text", ] diff --git a/mathics_scanner/feed.py b/mathics_scanner/feed.py index 4136ef4..f91c93b 100644 --- a/mathics_scanner/feed.py +++ b/mathics_scanner/feed.py @@ -5,6 +5,7 @@ """ from abc import ABCMeta, abstractmethod +from typing import List class LineFeeder(metaclass=ABCMeta): @@ -19,7 +20,7 @@ def __init__(self, filename: str): :param filename: A string that describes the source of the feeder, i.e. the filename that is being feed. """ - self.messages = [] + self.messages: List[str] = [] self.lineno = 0 self.filename = filename diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py index a35351f..0761c3f 100644 --- a/mathics_scanner/tokeniser.py +++ b/mathics_scanner/tokeniser.py @@ -19,8 +19,8 @@ (\*\^(\+|-)?\d+)? (?# Exponent) """ base_symbol_pattern = r"((?![0-9])([0-9${0}{1}])+)".format(_letters, _letterlikes) -full_symbol_pattern = r"(`?{0}(`{0})*)".format(base_symbol_pattern) -pattern_pattern = r"{0}?_(\.|(__?)?{0}?)?".format(full_symbol_pattern) +full_symbol_pattern_str = r"(`?{0}(`{0})*)".format(base_symbol_pattern) +pattern_pattern = r"{0}?_(\.|(__?)?{0}?)?".format(full_symbol_pattern_str) slot_pattern = r"\#(\d+|{0})?".format(base_symbol_pattern) FILENAME_PATTERN = r""" (?P\"?) (?# Opening quotation mark) @@ -39,7 +39,7 @@ ("Number", NUMBER_PATTERN), ("String", r'"'), ("Pattern", pattern_pattern), - ("Symbol", full_symbol_pattern), + ("Symbol", full_symbol_pattern_str), ("SlotSequence", r"\#\#\d*"), ("Slot", slot_pattern), ("Out", r"\%(\%+|\d+)?"), @@ -312,7 +312,7 @@ def compile_tokens(token_list): token_indices = find_indices(literal_tokens) tokens = compile_tokens(tokens) filename_tokens = compile_tokens(filename_tokens) -full_symbol_pattern = compile_pattern(full_symbol_pattern) +full_symbol_pattern_re: re.Pattern = compile_pattern(full_symbol_pattern_str) def is_symbol_name(text: str) -> bool: @@ -321,7 +321,7 @@ def is_symbol_name(text: str) -> bool: ``False``. """ # Can't we just call match here? - return full_symbol_pattern.sub("", text) == "" + return full_symbol_pattern_re.sub("", text) == "" class Token: diff --git a/mathics_scanner/version.py b/mathics_scanner/version.py index 2072e22..a950693 100644 --- a/mathics_scanner/version.py +++ b/mathics_scanner/version.py @@ -4,4 +4,4 @@ # well as importing into Python. That's why there is no # space around "=" below. # fmt: off -__version__="1.3.0" # noqa +__version__="1.3.1" # noqa diff --git a/pyproject.toml b/pyproject.toml index f743689..200bef3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Scientific/Engineering", diff --git a/setup.py b/setup.py index 5e8a4c9..d9a0892 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ def get_srcdir(): + """Return the directory of the location if this code""" filename = osp.normcase(osp.dirname(osp.abspath(__file__))) return osp.realpath(filename) @@ -43,7 +44,7 @@ def get_srcdir(): for kind in ("dev", "full"): extras_require = [] requirements_file = f"requirements-{kind}.txt" - for line in open(requirements_file).read().split("\n"): + for line in open(requirements_file, encoding="utf-8").read().split("\n"): if line and not line.startswith("#"): requires = re.sub(r"([^#]+)(\s*#.*$)?", r"\1", line) extras_require.append(requires) @@ -51,7 +52,7 @@ def get_srcdir(): class table_building_egg_info(egg_info): - # This runs as part of building an sdist + """This runs as part of building an sdist""" def finalize_options(self): """Run program to create JSON tables""" @@ -59,7 +60,7 @@ def finalize_options(self): get_srcdir(), "mathics_scanner", "generate", "build_tables.py" ) print(f"Building JSON tables via {build_tables_program}") - result = subprocess.run([sys.executable, build_tables_program]) + result = subprocess.run([sys.executable, build_tables_program], check=False) if result.returncode: raise RuntimeError( f"Running {build_tables_program} exited with code {result.returncode}"