From 995c1ba55eeedff6de4bab1b25e05ad995358ad1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 16 Nov 2017 13:49:26 -0500 Subject: [PATCH 01/11] Turn callers module into package --- src/pluggy/{callers.py => callers/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/pluggy/{callers.py => callers/__init__.py} (100%) diff --git a/src/pluggy/callers.py b/src/pluggy/callers/__init__.py similarity index 100% rename from src/pluggy/callers.py rename to src/pluggy/callers/__init__.py From e586fb57af56a362b22122d7265ae2aa874a83e4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 20 Nov 2017 15:50:52 -0500 Subject: [PATCH 02/11] Cythonize the call loop Cythonize `pluggy.callers._multicall` which gives around a 2x speedup according to the benchmark suite. Transform the `pluggy.callers` module in a new package and move utils and the legacy call loop into separate modules. --- src/pluggy/callers/__init__.py | 1 + src/pluggy/callers/cythonized.pyx | 60 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 src/pluggy/callers/cythonized.pyx diff --git a/src/pluggy/callers/__init__.py b/src/pluggy/callers/__init__.py index f66cc946..d66c358e 100644 --- a/src/pluggy/callers/__init__.py +++ b/src/pluggy/callers/__init__.py @@ -4,6 +4,7 @@ import sys from ._result import HookCallError, _Result, _raise_wrapfail +from .cythonized import _c_multicall def _multicall(hook_impls, caller_kwargs, firstresult=False): diff --git a/src/pluggy/callers/cythonized.pyx b/src/pluggy/callers/cythonized.pyx new file mode 100644 index 00000000..79621cdf --- /dev/null +++ b/src/pluggy/callers/cythonized.pyx @@ -0,0 +1,60 @@ +""" +Cynthonized hook call loop. + +NOTE: In order to build this source you must have cython installed. +""" +import sys +from . import _Result, _raise_wrapfail, HookCallError + + +cpdef _multicall(list hook_impls, dict caller_kwargs, bint firstresult=False): + """Execute a call into multiple python functions/methods and return the + result(s). + + ``caller_kwargs`` comes from _HookCaller.__call__(). + """ + __tracebackhide__ = True + results = [] + excinfo = None + try: # run impl and wrapper setup functions in a loop + teardowns = [] + try: + for hook_impl in reversed(hook_impls): + try: + args = [caller_kwargs[argname] for argname in hook_impl.argnames] + except KeyError: + for argname in hook_impl.argnames: + if argname not in caller_kwargs: + raise HookCallError( + "hook call must provide argument %r" % (argname,)) + + if hook_impl.hookwrapper: + try: + gen = hook_impl.function(*args) + next(gen) # first yield + teardowns.append(gen) + except StopIteration: + _raise_wrapfail(gen, "did not yield") + else: + res = hook_impl.function(*args) + if res is not None: + results.append(res) + if firstresult: # halt further impl calls + break + except BaseException: + excinfo = sys.exc_info() + finally: + if firstresult: # first result hooks return a single value + outcome = _Result(results[0] if results else None, excinfo) + else: + outcome = _Result(results, excinfo) + + # run all wrapper post-yield blocks + for gen in reversed(teardowns): + try: + gen.send(outcome) + _raise_wrapfail(gen, "has second yield") + except StopIteration: + pass + + return outcome.get_result() From a0ffaa6deaa4854980806a0c1aa4fdafb385bc9a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 20 Nov 2017 16:23:06 -0500 Subject: [PATCH 03/11] Build cython extension in packaging When cython is installed always rebuild the C sources, when not installed use whatever C sources were included in the sdist. --- MANIFEST.in | 2 ++ setup.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0cf8f3e0..e059db04 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,5 @@ include tox.ini include LICENSE graft testing recursive-exclude * *.pyc *.pyo +include pluggy/callers/cythonized.pyx +include pluggy/callers/cythonized.c diff --git a/setup.py b/setup.py index 48d016b2..99310114 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,7 @@ from setuptools import setup +from setuptools.command.sdist import sdist as _sdist +from setuptools.extension import Extension + classifiers = [ "Development Status :: 4 - Beta", @@ -29,6 +32,32 @@ } +cmdclass = {} + + +class sdist(_sdist): + """Custom sdist building using cython + """ + def run(self): + # Make sure the compiled Cython files in the distribution + # are up-to-date + from Cython.Build import cythonize + cythonize(["pluggy/callers/cythonized.pyx"]) + _sdist.run(self) + + +try: + from Cython.Build import cythonize + print("Building Cython extension(s)") + exts = cythonize(["pluggy/callers/cythonized.pyx"]) + cmdclass['sdist'] = sdist +except ImportError: + # When Cython is not installed build from C sources + print("Building C extension(s)") + exts = [Extension("pluggy.callers.cythonized", + ["pluggy/callers/cythonized.c"])] + + def main(): setup( name="pluggy", @@ -45,8 +74,10 @@ def main(): install_requires=['importlib-metadata>=0.12;python_version<"3.8"'], extras_require=EXTRAS_REQUIRE, classifiers=classifiers, - packages=["pluggy"], + packages=['pluggy', 'pluggy.callers'], package_dir={"": "src"}, + ext_modules=exts, + cmdclass=cmdclass, ) From 7bd1ec563a68b73ba75ef033bf8899675b72ee8b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 20 Nov 2017 16:25:33 -0500 Subject: [PATCH 04/11] Include cython in tox benchmark tests --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 18d3f0b2..d6b83998 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ commands=pytest {posargs:testing/benchmark.py} deps= pytest pytest-benchmark + cython [testenv:linting] skip_install = true From 08db6466a7d858651f7e98d24932898c54c4a1ae Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 20 Nov 2017 16:25:47 -0500 Subject: [PATCH 05/11] Install cython in travis env --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f5d074d1..e5513e60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,7 +57,7 @@ jobs: install: - pip install -U pip - - pip install -U --force-reinstall setuptools tox + - pip install -U --force-reinstall setuptools tox cython script: - tox From e003fc466414d2b2c790fc4cc66af90c10c17fdc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Jun 2020 09:02:32 -0400 Subject: [PATCH 06/11] Fix install to include cython --- MANIFEST.in | 4 ++-- pyproject.toml | 1 + setup.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index e059db04..b8815a3c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,5 +5,5 @@ include tox.ini include LICENSE graft testing recursive-exclude * *.pyc *.pyo -include pluggy/callers/cythonized.pyx -include pluggy/callers/cythonized.c +include src/pluggy/callers/cythonized.pyx +include src/pluggy/callers/cythonized.c diff --git a/pyproject.toml b/pyproject.toml index 811cf53e..f9e40879 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ requires = [ "setuptools", "setuptools-scm", "wheel", + "Cython", ] [tool.towncrier] diff --git a/setup.py b/setup.py index 99310114..6171755f 100644 --- a/setup.py +++ b/setup.py @@ -42,20 +42,20 @@ def run(self): # Make sure the compiled Cython files in the distribution # are up-to-date from Cython.Build import cythonize - cythonize(["pluggy/callers/cythonized.pyx"]) + cythonize(["src/pluggy/callers/cythonized.pyx"]) _sdist.run(self) try: from Cython.Build import cythonize print("Building Cython extension(s)") - exts = cythonize(["pluggy/callers/cythonized.pyx"]) + exts = cythonize(["src/pluggy/callers/cythonized.pyx"]) cmdclass['sdist'] = sdist except ImportError: # When Cython is not installed build from C sources print("Building C extension(s)") exts = [Extension("pluggy.callers.cythonized", - ["pluggy/callers/cythonized.c"])] + ["src/pluggy/callers/cythonized.c"])] def main(): From 323ce41c112e495c4e221001b0adc72b05dacb43 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Jun 2020 09:28:04 -0400 Subject: [PATCH 07/11] Move import to bottom of mod; rename c func. --- src/pluggy/callers/__init__.py | 2 ++ src/pluggy/callers/cythonized.pyx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pluggy/callers/__init__.py b/src/pluggy/callers/__init__.py index d66c358e..d35db412 100644 --- a/src/pluggy/callers/__init__.py +++ b/src/pluggy/callers/__init__.py @@ -6,6 +6,8 @@ from ._result import HookCallError, _Result, _raise_wrapfail from .cythonized import _c_multicall +__all__ = ['_multicall', '_c_multicall'] + def _multicall(hook_impls, caller_kwargs, firstresult=False): """Execute a call into multiple python functions/methods and return the diff --git a/src/pluggy/callers/cythonized.pyx b/src/pluggy/callers/cythonized.pyx index 79621cdf..0abc79cf 100644 --- a/src/pluggy/callers/cythonized.pyx +++ b/src/pluggy/callers/cythonized.pyx @@ -7,7 +7,7 @@ import sys from . import _Result, _raise_wrapfail, HookCallError -cpdef _multicall(list hook_impls, dict caller_kwargs, bint firstresult=False): +cpdef _c_multicall(list hook_impls, dict caller_kwargs, bint firstresult=False): """Execute a call into multiple python functions/methods and return the result(s). From 044aa5456c3d3ddbdfbf66a2670edf146458defb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Jun 2020 14:53:17 -0400 Subject: [PATCH 08/11] Revert "Drop `callertype` fixture from benchmark tests" This reverts commit 72948afada60cd9d6474b4631fabb864731de578 since we need this parametrization for testing the cythonized version of `_multicall` aka `pluggy.caller.cythonized._c_multicall()`. --- testing/benchmark.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/testing/benchmark.py b/testing/benchmark.py index cca4a759..4cd7781c 100644 --- a/testing/benchmark.py +++ b/testing/benchmark.py @@ -10,12 +10,12 @@ hookimpl = HookimplMarker("example") -def MC(methods, kwargs, firstresult=False): +def MC(methods, kwargs, callertype, firstresult=False): hookfuncs = [] for method in methods: f = HookImpl(None, "", method, method.example_impl) hookfuncs.append(f) - return _multicall(hookfuncs, kwargs, firstresult=firstresult) + return callertype(hookfuncs, kwargs, firstresult=firstresult) @hookimpl @@ -38,9 +38,14 @@ def wrappers(request): return [wrapper for i in range(request.param)] -def inner_exec(methods): - return MC(methods, {"arg1": 1, "arg2": 2, "arg3": 3}) +@pytest.fixture(params=[_multicall], ids=lambda item: item.__name__) +def callertype(request): + return request.param -def test_hook_and_wrappers_speed(benchmark, hooks, wrappers): - benchmark(inner_exec, hooks + wrappers) +def inner_exec(methods, callertype): + return MC(methods, {"arg1": 1, "arg2": 2, "arg3": 3}, callertype) + + +def test_hook_and_wrappers_speed(benchmark, hooks, wrappers, callertype): + benchmark(inner_exec, hooks + wrappers, callertype) From c17cfeb13e19b618d6bb8f53608b0455e94a4d7a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Jun 2020 14:59:33 -0400 Subject: [PATCH 09/11] Include cythonized multicall in benchmarks --- testing/benchmark.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testing/benchmark.py b/testing/benchmark.py index 4cd7781c..f98fbcca 100644 --- a/testing/benchmark.py +++ b/testing/benchmark.py @@ -4,7 +4,7 @@ import pytest from pluggy import HookspecMarker, HookimplMarker from pluggy.hooks import HookImpl -from pluggy.callers import _multicall +from pluggy.callers import _multicall, _c_multicall hookspec = HookspecMarker("example") hookimpl = HookimplMarker("example") @@ -38,7 +38,10 @@ def wrappers(request): return [wrapper for i in range(request.param)] -@pytest.fixture(params=[_multicall], ids=lambda item: item.__name__) +@pytest.fixture( + params=[_multicall, _c_multicall], + ids=lambda item: item.__name__ +) def callertype(request): return request.param From b826f75d5ceaa1941cee87a1fe4522a9eca20d9f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Jun 2020 15:33:37 -0400 Subject: [PATCH 10/11] Port callers to new _result.py module --- src/pluggy/callers/__init__.py | 2 +- src/pluggy/callers/cythonized.pyx | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pluggy/callers/__init__.py b/src/pluggy/callers/__init__.py index d35db412..2a3e030a 100644 --- a/src/pluggy/callers/__init__.py +++ b/src/pluggy/callers/__init__.py @@ -3,7 +3,7 @@ """ import sys -from ._result import HookCallError, _Result, _raise_wrapfail +from .._result import HookCallError, _Result, _raise_wrapfail from .cythonized import _c_multicall __all__ = ['_multicall', '_c_multicall'] diff --git a/src/pluggy/callers/cythonized.pyx b/src/pluggy/callers/cythonized.pyx index 0abc79cf..952af636 100644 --- a/src/pluggy/callers/cythonized.pyx +++ b/src/pluggy/callers/cythonized.pyx @@ -1,10 +1,14 @@ """ Cynthonized hook call loop. +This is currently maintained as a verbatim copy of +``pluggy.callers._multicall()``. + NOTE: In order to build this source you must have cython installed. """ import sys -from . import _Result, _raise_wrapfail, HookCallError + +from .._result import _Result, _raise_wrapfail, HookCallError cpdef _c_multicall(list hook_impls, dict caller_kwargs, bint firstresult=False): From f5d4eb49890a3f1a013936efe1c113fd1716a095 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 4 Jun 2020 16:03:35 -0400 Subject: [PATCH 11/11] Appease black linter --- setup.py | 10 ++++++---- src/pluggy/callers/__init__.py | 2 +- testing/benchmark.py | 5 +---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 6171755f..0d2cb014 100644 --- a/setup.py +++ b/setup.py @@ -38,24 +38,26 @@ class sdist(_sdist): """Custom sdist building using cython """ + def run(self): # Make sure the compiled Cython files in the distribution # are up-to-date from Cython.Build import cythonize + cythonize(["src/pluggy/callers/cythonized.pyx"]) _sdist.run(self) try: from Cython.Build import cythonize + print("Building Cython extension(s)") exts = cythonize(["src/pluggy/callers/cythonized.pyx"]) - cmdclass['sdist'] = sdist + cmdclass["sdist"] = sdist except ImportError: # When Cython is not installed build from C sources print("Building C extension(s)") - exts = [Extension("pluggy.callers.cythonized", - ["src/pluggy/callers/cythonized.c"])] + exts = [Extension("pluggy.callers.cythonized", ["src/pluggy/callers/cythonized.c"])] def main(): @@ -74,7 +76,7 @@ def main(): install_requires=['importlib-metadata>=0.12;python_version<"3.8"'], extras_require=EXTRAS_REQUIRE, classifiers=classifiers, - packages=['pluggy', 'pluggy.callers'], + packages=["pluggy", "pluggy.callers"], package_dir={"": "src"}, ext_modules=exts, cmdclass=cmdclass, diff --git a/src/pluggy/callers/__init__.py b/src/pluggy/callers/__init__.py index 2a3e030a..38695b27 100644 --- a/src/pluggy/callers/__init__.py +++ b/src/pluggy/callers/__init__.py @@ -6,7 +6,7 @@ from .._result import HookCallError, _Result, _raise_wrapfail from .cythonized import _c_multicall -__all__ = ['_multicall', '_c_multicall'] +__all__ = ["_multicall", "_c_multicall"] def _multicall(hook_impls, caller_kwargs, firstresult=False): diff --git a/testing/benchmark.py b/testing/benchmark.py index f98fbcca..fd048ac8 100644 --- a/testing/benchmark.py +++ b/testing/benchmark.py @@ -38,10 +38,7 @@ def wrappers(request): return [wrapper for i in range(request.param)] -@pytest.fixture( - params=[_multicall, _c_multicall], - ids=lambda item: item.__name__ -) +@pytest.fixture(params=[_multicall, _c_multicall], ids=lambda item: item.__name__) def callertype(request): return request.param