diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 0000000..1ede09c --- /dev/null +++ b/.cruft.json @@ -0,0 +1,20 @@ +{ + "template": "https://github.com/adjavon/pycookie/", + "commit": "d7bd6b8521627b130001b270383b72bfed608ab3", + "checkout": null, + "context": { + "cookiecutter": { + "full_name": "Jan Funke", + "email": "funkej@janelia.hhmi.org", + "github_username": "funkelab", + "project_name": "witty", + "project_slug": "witty", + "project_short_description": "Well-in-Time Compiler for Cython Modules", + "_copy_without_render": [ + ".github/workflows/*" + ], + "_template": "https://github.com/adjavon/pycookie/" + } + }, + "directory": null +} diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000..60b6467 --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,17 @@ +name: Python Black + +on: [push, pull_request] + +jobs: + lint: + name: Python Lint + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v4 + - name: Setup checkout + uses: actions/checkout@master + - name: Lint with Black + run: | + pip install black + black --diff --check src/witty tests diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml new file mode 100644 index 0000000..495582f --- /dev/null +++ b/.github/workflows/mypy.yaml @@ -0,0 +1,18 @@ +name: Python mypy + +on: [push, pull_request] + +jobs: + static-analysis: + name: Python mypy + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v4 + - name: Setup checkout + uses: actions/checkout@v2 + - name: mypy + run: | + pip install . + pip install --upgrade mypy + mypy src/witty diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..384aaea --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,25 @@ +name: Test + +on: + push: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install ".[dev]" + - name: Test with pytest + run: | + pytest tests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2b2f55c --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.sw[pmno] +*.pyc +*.egg-info +*.dat +*.lock +*.so +*_wrapper.cpp +*_wrapper.c +.coverage +build +dist +.vscode +docs/_build/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a2bf1d6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +ci: + autoupdate_schedule: monthly + autofix_commit_msg: "style(pre-commit.ci): auto fixes [...]" + autoupdate_commit_msg: "ci(pre-commit.ci): autoupdate" + +default_install_hook_types: [pre-commit, commit-msg] + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.0.1 + hooks: + - id: mypy diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ace50b --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# witty + +[![tests](https://github.com/funkelab/witty/actions/workflows/tests.yaml/badge.svg)](https://github.com/funkelab/witty/actions/workflows/tests.yaml) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0dcb32f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel"] + +[project] +name = "witty" +description = "Well-in-Time Compiler for Cython Modules" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", +] +keywords = [] +license = { text = "BSD 3-Clause License" } +authors = [ + { email = "funkej@janelia.hhmi.org", name = "Jan Funke" }, +] +dynamic = ["version"] +dependencies = ["cython"] + +[project.optional-dependencies] +dev = [ + 'pytest', + 'black', + 'mypy', + 'pdoc', + 'pre-commit' +] + +[project.urls] +homepage = "https://github.com/funkelab/witty" +repository = "https://github.com/funkelab/witty" diff --git a/src/witty/__init__.py b/src/witty/__init__.py new file mode 100644 index 0000000..de2c623 --- /dev/null +++ b/src/witty/__init__.py @@ -0,0 +1 @@ +from .compile_module import compile_module diff --git a/src/witty/compile_module.py b/src/witty/compile_module.py new file mode 100644 index 0000000..092a0ed --- /dev/null +++ b/src/witty/compile_module.py @@ -0,0 +1,99 @@ +import Cython +import fcntl +import hashlib +import importlib.util +import sys +from Cython.Build import cythonize +from Cython.Build.Inline import to_unicode, _get_build_extension +from Cython.Utils import get_cython_cache_dir +from distutils.core import Extension +from pathlib import Path + + +def load_dynamic(module_name, module_lib): + spec = importlib.util.spec_from_file_location(module_name, module_lib) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return sys.modules[module_name] + + +def compile_module( + source_pyx, + source_files=None, + include_dirs=None, + library_dirs=None, + language="c", + extra_compile_args=None, + extra_link_args=None, + name=None, + force_rebuild=False, +): + if source_files is None: + source_files = [] + if include_dirs is None: + include_dirs = ["."] + if library_dirs is None: + library_dirs = [] + if name is None: + name = "_wit_module" + + source_pyx = to_unicode(source_pyx) + sources = [source_pyx] + + for source_file in source_files: + sources.append(open(source_file, "r").read()) + + source_hashes = [ + hashlib.md5(source.encode("utf-8")).hexdigest() for source in sources + ] + source_key = (source_hashes, sys.version_info, sys.executable, Cython.__version__) + module_hash = hashlib.md5(str(source_key).encode("utf-8")).hexdigest() + module_name = name + "_" + module_hash + + # already loaded? + if module_name in sys.modules and not force_rebuild: + return sys.modules[module_name] + + build_extension = _get_build_extension() + module_ext = build_extension.get_ext_filename("") + module_dir = Path(get_cython_cache_dir()) / "witty" + module_pyx = (module_dir / module_name).with_suffix(".pyx") + module_lib = (module_dir / module_name).with_suffix(module_ext) + module_lock = (module_dir / module_name).with_suffix(".lock") + + print(f"Compiling {module_name} into {module_lib}...") + + module_dir.mkdir(parents=True, exist_ok=True) + + # make sure the same module is not build concurrently + with open(module_lock, "w") as lock_file: + fcntl.lockf(lock_file, fcntl.LOCK_EX) + + # already compiled? + if module_lib.is_file() and not force_rebuild: + print(f"Reusing already compiled module from {module_lib}") + return load_dynamic(module_name, module_lib) + + # create pyx file + with open(module_pyx, "w") as f: + f.write(source_pyx) + + extension = Extension( + module_name, + sources=[str(module_pyx)], + include_dirs=include_dirs, + library_dirs=library_dirs, + language=language, + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + ) + + build_extension.extensions = cythonize( + [extension], compiler_directives={"language_level": "3"} + ) + build_extension.build_temp = str(module_dir) + build_extension.build_lib = str(module_dir) + build_extension.run() + + return load_dynamic(module_name, module_lib) diff --git a/tests/test_compile.py b/tests/test_compile.py new file mode 100644 index 0000000..af79bdc --- /dev/null +++ b/tests/test_compile.py @@ -0,0 +1,36 @@ +import witty + + +def test_compile(): + source_pxy_template = """ +cdef extern from '' namespace 'std': + + cdef cppclass vector[T]: + vector() + void push_back(T& item) + size_t size() + +def add({type} x, {type} y): + return x + y + +def to_vector({type} x): + v = vector[{type}]() + v.push_back(x) + return v +""" + + module_int = witty.compile_module( + source_pxy_template.format(type="int"), language="c++", force_rebuild=True + ) + result = module_int.add(3, 4) + + assert result == 7 + assert type(result) == int + + module_float = witty.compile_module( + source_pxy_template.format(type="float"), language="c++" + ) + result = module_float.add(3, 4) + + assert result == 7 + assert type(result) == float