Issue 89 & 90 (#91)

Fixes aimed at allowing broader platform support, including building under a Conda environment.

Importantly, the setuptools build step now (re)generates the ANTLR4 language files using the installed version of `antlr4-python3-runtime`. This makes the version dependency explicit and automatic; which should allow for the associated relaxation on the runtime dependency to produce working modules on more systems.

LICENSE

@@ -0,0 +1,11 @@
+Copyright 2024 Gregory Tucker, European Spallation Source ERIC
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pyproject.toml

@@ -1,20 +1,25 @@
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+requires = [
+    "setuptools>=45",
+    "setuptools_scm[toml]>=6.2",
+    "antlr4-tools>=0.2.1",
+    'antlr4-python3-runtime>=4.13.2'
+]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "mccode-antlr"
 dependencies = [
-    'antlr4-python3-runtime==4.13.2',  # should match the version of antlr4 used
+    'antlr4-python3-runtime>=4.13.2',
     'numpy>=2',
     'pooch>=1.7.0',
     'confuse>=2.0.1',
     'loguru>=0.7.2',
     'gitpython>=3.1.43',
-    "importlib_metadata; python_version<'3.8'",
 ]
 description = "ANTLR4 grammars for McStas and McXtrace"
 readme = "README.md"
+license = {text = "BSD-3-Clause"}
 requires-python = ">=3.9"
 authors = [
     { name = "Gregory Tucker", email = "[email protected]" },
@@ -34,7 +39,6 @@ dynamic = ["version"]
 [project.optional-dependencies]
 test = ["gputil==1.4.0", 'pytest']
 hdf5 = ["h5py>=3.11.0"]
-antlr = ['antlr4-tools==0.2.1']
 
 [project.urls]
 "Homepage" = "https://github.com/McStasMcXtrace/mccode-antlr"

setup.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from setuptools import Command, setup
+import setuptools.command.build
+
+from pathlib import Path
+from enum import Enum
+
+class Target(Enum):
+    python = 0
+    cpp = 1
+    def __str__(self):
+        if self == Target.python:
+            return 'Python3'
+        elif self == Target.cpp:
+            return 'Cpp'
+        raise ValueError(f'Unknown target {self}')
+
+
+class Feature(Enum):
+    listener = 0
+    visitor = 1
+    lexer = 2
+    parser = 3
+    def __str__(self):
+        if self == Feature.listener:
+            return 'Listener'
+        if self == Feature.visitor:
+            return 'Visitor'
+        if self == Feature.lexer:
+            return 'Lexer'
+        if self == Feature.parser:
+            return 'Parser'
+
+
+def BuildANTLRCommand(source: Path, destination: str, grammars):
+
+    def antlr4_runtime_version():
+        """Retrieve the ANTLR4 version used by the available antlr4-python3-runtime"""
+        from importlib import metadata
+        try:
+            return metadata.metadata('antlr4-python3-runtime').get('version')
+        except metadata.PackageNotFoundError:
+            raise RuntimeError("antlr4-python3-runtime is a build dependency!")
+
+    def build_language(grammar_file: Path,
+                       target: Target,
+                       features: list[Feature],
+                       output=None,
+                       ):
+        from subprocess import run
+        from antlr4_tool_runner import initialize_paths, install_jre_and_antlr
+        args = [
+            f'-Dlanguage={target}',
+            '-visitor' if Feature.visitor in features else '-no-visitor',
+            '-listener' if Feature.listener in features else '-no-listener',
+            '-o', output.resolve(),
+            grammar_file.name
+        ]
+        print(f"Generating ANTLR {target} {' '.join(str(f) for f in features)} in {output} for {grammar_file}")
+        # The following copies the implementation of antlr4_tool_runner.tool,
+        # which pulls `args` from the system argv list
+        initialize_paths()
+        jar, java = install_jre_and_antlr(antlr4_runtime_version())
+        run([java, '-cp', jar, 'org.antlr.v4.Tool'] + args, cwd=grammar_file.parent)
+
+    class BuildANTLR(Command):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self.build_lib = None
+            self.editable_mode = False
+
+        def initialize_options(self):
+            """Initialize command state to defaults"""
+            ...
+
+        def finalize_options(self):
+            """
+            Populate the command state. This is where I traverse the directory
+            tree to search for the *.ksc files to compile them later.
+            The self.set_undefined_options is used to inherit the `build_lib`
+            attribute from the `build_py` command.
+            """
+            self.set_undefined_options("build_py", ("build_lib", "build_lib"))
+
+        def run(self):
+            """
+            Perform actions with side-effects, such as invoking a ksc to python compiler.
+            The directory to which outputs are written depends on `editable_mode` attribute.
+            When editable_mode == False, the outputs are written to directory pointed by build_lib.
+            When editable_mode == True, the outputs are written in-place,
+            i.e. into the directory containing the sources.
+            The `run` method is not executed during sdist builds.
+            """
+            dest = Path(self.build_lib) / destination
+            for grammar, options in grammars.items():
+                build_language(source/f'{grammar}.g4',
+                               target=options['target'],
+                               features=options['features'],
+                               output=dest)
+
+        def get_output_mapping(self):
+            """
+            Return dict mapping output file paths to input file paths
+            Example:
+                dict = { "build/lib/output.py": "src/grammar/grammar.g4" }
+            """
+            files = {}
+            dest = Path(self.build_lib) / destination
+            for grammar, options in grammars.items():
+                for feature in [Feature.lexer, Feature.parser] + options['features']:
+                    files[dest / f"{grammar}{feature}.py"] = source / f"{grammar}.g4"
+                    if deps := options.get("deps"):
+                        for dep in deps:
+                            files[dest / f"{dep}{feature}.py"] = source / f"{dep}.g4"
+            return {str(k): str(v) for k, v in files.items()}
+
+        def get_outputs(self):
+            """Return list containing paths to output files (generated *.py files)"""
+            files = []
+            dest = Path(self.build_lib) / destination
+            for grammar, options in grammars.items():
+                for feature in [Feature.lexer, Feature.parser] + options['features']:
+                    files.append(dest / f"{grammar}{feature}.py")
+                    if deps := options.get("deps"):
+                        files.extend(dest / f"{dep}{feature}.py" for dep in deps)
+            return [str(file) for file in files]
+
+        def get_source_files(self):
+            """Returns list containing paths to input files (*.g4 ANTLR grammars)"""
+            files = []
+            for grammar, options in grammars.items():
+                files.append(source / f"{grammar}.g4")
+                if deps := options.get("deps"):
+                    files.extend(source / f"{dep}.g4" for dep in deps)
+            return [str(file) for file in files]
+
+    return BuildANTLR
+
+
+setuptools.command.build.build.sub_commands.append(("build_antlr", None))
+
+setup(cmdclass={
+    "build_antlr": BuildANTLRCommand(
+        source=Path() / "src" / "grammar",  # grammar file loc relative to this file
+        destination="mccode_antlr/grammar", # generated file loc in build dir
+        grammars={
+            'McComp': {
+                'target': Target.python,
+                'features': [Feature.visitor],
+                'deps': ('McCommon', 'c99'),
+            },
+            'McInstr': {
+                'target': Target.python,
+                'features': [Feature.visitor],
+                'deps': ('McCommon', 'c99'),
+            },
+            'C': {
+                'target': Target.python,
+                'features': [Feature.visitor, Feature.listener],
+            },
+        },
+    )
+})

src/grammar/README.md

@@ -4,15 +4,18 @@ contents of this folder are generated by `ANTLR`
 from the grammar files, `C.g4`, `cpp.g4`, `McCommon.g4`,
 `McComp.g4` and `McInstr.g4`.
 
+As such, the generated files should not be edited by hand.
+Instead, modify the grammar files, and possibly any
+functions which make use of `Mc*Visitor` or `CListener`.
+
+
+## Automatic generation
+
 The (re)generation of the various language files can be performed by
 calling `builder.py`, e.g.,
 ```console
 python builder.py --verbose
 ```
-Or, if working with the installed package,
-```console
-mccode-antlr-builder --verbose
-```
 
 Building the parser, visitor, and listener implementations requires
 `antlr4-tools` to be installed. This can be done via
@@ -22,6 +25,19 @@ pip install "antlr4-tools=0.2.1"
 Other versions of `antlr4-tools` may work, but unannounced changes in its API
 have been known to break the build process.
 
-As such, the generated files should not be edited by hand.
-Instead, modify the grammar files, and possibly any
-functions which make use of `Mc*Visitor` or `CListener`.
+### ANTLR version
+The version of ANTLR used in generating the required files can be
+specified as a command line argument, `-v` or `--version`.
+This version **must** match the version of `antlr4-python3-runtime`
+used by Python, and which `mccode-antlr` depends on.
+
+If no version is specified, the version of the installed `antlr4-python3-runtime`
+is taken, such that the produced files will work with the used Python environment.
+
+If no version is specified and `antlr4-python3-runtime` is not installed,
+the value of the `${ANTLR4_TOOLS_ANTLR_VERSION}` environment variable, 
+the result of querying the latest available `antlr` version on 
+[maven](https://mvnrepository.com/artifact/org.antlr/antlr4),
+or a version already present on the running system will be used,
+in that order.
+