From f8e9b38dbb50fc11e662a62316850e541f9cc4b3 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 26 Jul 2023 18:16:58 -0400 Subject: [PATCH] switch from bundled lazy loading code to the public lazy_loader package (#575) This PR switch from using the internally bundled `_shared/lazy.py` to the public [lazy_loader](https://github.com/scientific-python/lazy_loader) package. Also switches to use `.pyi` [stub files](https://mypy.readthedocs.io/en/stable/stubs.html) in cases where submodules or attributes were being dynamically loaded. closes #489 (probably, but have not yet verified from VS Code) Authors: - Gregory Lee (https://github.com/grlee77) - https://github.com/jakirkham Approvers: - Gigon Bae (https://github.com/gigony) - https://github.com/jakirkham - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cucim/pull/575 --- .../all_cuda-118_arch-x86_64.yaml | 1 + .../all_cuda-120_arch-x86_64.yaml | 1 + conda/recipes/cucim/meta.yaml | 1 + dependencies.yaml | 1 + python/cucim/MANIFEST.in | 2 + python/cucim/ci/requirements.txt | 1 + python/cucim/setup.py | 6 +- python/cucim/src/cucim/__init__.py | 8 +- python/cucim/src/cucim/__init__.pyi | 35 +++++ python/cucim/src/cucim/skimage/__init__.py | 36 ++--- python/cucim/src/cucim/skimage/__init__.pyi | 23 +++ .../cucim/src/cucim/skimage/_shared/lazy.py | 139 ------------------ .../cucim/src/cucim/skimage/data/__init__.py | 10 +- .../cucim/src/cucim/skimage/data/__init__.pyi | 5 + .../src/cucim/skimage/filters/__init__.py | 35 +---- .../src/cucim/skimage/filters/__init__.pyi | 68 +++++++++ 16 files changed, 156 insertions(+), 216 deletions(-) create mode 100644 python/cucim/src/cucim/__init__.pyi create mode 100644 python/cucim/src/cucim/skimage/__init__.pyi delete mode 100644 python/cucim/src/cucim/skimage/_shared/lazy.py create mode 100644 python/cucim/src/cucim/skimage/data/__init__.pyi create mode 100644 python/cucim/src/cucim/skimage/filters/__init__.pyi diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index ffde5c912..b9a4de842 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -18,6 +18,7 @@ dependencies: - imagecodecs>=2021.6.8 - ipython - jbig +- lazy_loader >=0.1 - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libnvjpeg-dev=11.6.0.55 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index ae0339e39..a4cb89dcd 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -19,6 +19,7 @@ dependencies: - imagecodecs>=2021.6.8 - ipython - jbig +- lazy_loader >=0.1 - libcufile-dev - libnvjpeg-dev - libnvjpeg-static diff --git a/conda/recipes/cucim/meta.yaml b/conda/recipes/cucim/meta.yaml index 653e1bc6d..87b5c157c 100644 --- a/conda/recipes/cucim/meta.yaml +++ b/conda/recipes/cucim/meta.yaml @@ -62,6 +62,7 @@ requirements: - {{ pin_compatible('numpy') }} - click - cupy >=12.0.0 + - lazy_loader >=0.1 - libcucim ={{ version }} # - openslide # skipping here but benchmark binary would need openslide library - python diff --git a/dependencies.yaml b/dependencies.yaml index 54a355aad..cdf60e9cf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -162,6 +162,7 @@ dependencies: - click - cupy >=12.0.0 - jbig + - lazy_loader >=0.1 - libwebp-base - numpy >=1.21.3 - scikit-image >=0.19.0,<0.22.0a0 diff --git a/python/cucim/MANIFEST.in b/python/cucim/MANIFEST.in index 4244cebec..3c2e31645 100644 --- a/python/cucim/MANIFEST.in +++ b/python/cucim/MANIFEST.in @@ -22,4 +22,6 @@ include tox.ini .travis.yml .appveyor.yml .readthedocs.yml include versioneer.py include src/cucim/clara/*.so* +recursive-include src/cucim *.py *.pyi *.cu *.h *.npy *.txt *.md + global-exclude *.py[cod] __pycache__/* diff --git a/python/cucim/ci/requirements.txt b/python/cucim/ci/requirements.txt index d7f5177e6..a0f53e12a 100644 --- a/python/cucim/ci/requirements.txt +++ b/python/cucim/ci/requirements.txt @@ -2,3 +2,4 @@ virtualenv>=16.6.0 pip>=19.1.1 setuptools>=18.0.1 six>=1.14.0 +lazy_loader>=0.1 diff --git a/python/cucim/setup.py b/python/cucim/setup.py index 1b7179b6b..214a44f25 100755 --- a/python/cucim/setup.py +++ b/python/cucim/setup.py @@ -39,6 +39,7 @@ def read(*names, **kwargs): url='https://developer.nvidia.com/multidimensional-image-processing', packages=find_packages('src'), package_dir={'cucim': 'src/cucim'}, + package_data={"": ["*.pyi", "*.h", "*.cu"]}, include_package_data=True, zip_safe=False, classifiers=[ @@ -56,11 +57,10 @@ def read(*names, **kwargs): 'Programming Language :: C++', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', # 'Operating System :: OS Independent', # 'Operating System :: Unix', # 'Operating System :: POSIX', @@ -87,7 +87,7 @@ def read(*names, **kwargs): setup_requires=SETUP_REQUIRES, install_requires=[ # TODO: Check cupy dependency based on cuda version - 'click', 'numpy', # 'scipy', 'scikit-image' + 'click', 'numpy', "lazy_loader>=0.1", # 'scipy', 'scikit-image' # eg: 'aspectlib==1.1.1', 'six>=1.7', ], extras_require={ diff --git a/python/cucim/src/cucim/__init__.py b/python/cucim/src/cucim/__init__.py index d51c2cf2e..37bfa2cb3 100644 --- a/python/cucim/src/cucim/__init__.py +++ b/python/cucim/src/cucim/__init__.py @@ -59,13 +59,9 @@ del _version -from .skimage._shared import lazy +import lazy_loader as lazy -__getattr__, __lazy_dir__, _ = lazy.attach( - __name__, - submodules, - submod_attrs, -) +__getattr__, __lazy_dir__, _ = lazy.attach_stub(__name__, __file__) def __dir__(): diff --git a/python/cucim/src/cucim/__init__.pyi b/python/cucim/src/cucim/__init__.pyi new file mode 100644 index 000000000..985551d7e --- /dev/null +++ b/python/cucim/src/cucim/__init__.pyi @@ -0,0 +1,35 @@ +# +# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +submodules = [] + +try: + import cupy + _is_cupy_available = True + submodules += ['core', 'skimage'] +except ImportError: + pass + +try: + from .clara import CuImage, __version__, cli + _is_clara_available = True + submodules += ['clara'] +except ImportError: + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + del _version + +__all__ = submodules + ['__version__', 'is_available'] diff --git a/python/cucim/src/cucim/skimage/__init__.py b/python/cucim/src/cucim/skimage/__init__.py index b48ceace8..c6cf7d4eb 100644 --- a/python/cucim/src/cucim/skimage/__init__.py +++ b/python/cucim/src/cucim/skimage/__init__.py @@ -60,35 +60,17 @@ """ -from ._shared import lazy +import lazy_loader as lazy -submodules = [ - 'color', - 'data', - 'exposure', - 'feature', - 'filters', - 'measure', - 'metrics', - 'morphology', - 'registration', - 'restoration', - 'segmentation', - 'transform', - 'util', -] - - -__getattr__, __lazy_dir__, _ = lazy.attach( - __name__, - submodules, - {'util.dtype': ['dtype_limits', 'img_as_bool', 'img_as_float', - 'img_as_float32', 'img_as_float64', 'img_as_int', - 'img_as_ubyte', 'img_as_uint'], - 'util.lookfor': ['lookfor'], - } -) +__getattr__, __lazy_dir__, _ = lazy.attach_stub(__name__, __file__) def __dir__(): return __lazy_dir__() + + +# Legacy imports into the root namespace; not advertised in __all__ +from .util.dtype import (dtype_limits, img_as_bool, img_as_float, + img_as_float32, img_as_float64, img_as_int, + img_as_ubyte, img_as_uint) +from .util.lookfor import lookfor diff --git a/python/cucim/src/cucim/skimage/__init__.pyi b/python/cucim/src/cucim/skimage/__init__.pyi new file mode 100644 index 000000000..1c3c07ea6 --- /dev/null +++ b/python/cucim/src/cucim/skimage/__init__.pyi @@ -0,0 +1,23 @@ +import lazy_loader as lazy + +submodules = [ + 'color', + 'data', + 'exposure', + 'feature', + 'filters', + 'measure', + 'metrics', + 'morphology', + 'registration', + 'restoration', + 'segmentation', + 'transform', + 'util', +] + +__all__ = submodules + +from . import (color, data, exposure, feature, filters, measure, metrics, + morphology, registration, restoration, segmentation, transform, + util) diff --git a/python/cucim/src/cucim/skimage/_shared/lazy.py b/python/cucim/src/cucim/skimage/_shared/lazy.py deleted file mode 100644 index 2b7270c66..000000000 --- a/python/cucim/src/cucim/skimage/_shared/lazy.py +++ /dev/null @@ -1,139 +0,0 @@ -import importlib -import importlib.util -import os -import sys - - -def attach(package_name, submodules=None, submod_attrs=None): - """Attach lazily loaded submodules, functions, or other attributes. - - Typically, modules import submodules and attributes as follows:: - - import mysubmodule - import anothersubmodule - - from .foo import someattr - - The idea is to replace a package's `__getattr__`, `__dir__`, and - `__all__`, such that all imports work exactly the way they did - before, except that they are only imported when used. - - The typical way to call this function, replacing the above imports, is:: - - __getattr__, __lazy_dir__, __all__ = lazy.attach( - __name__, - ['mysubmodule', 'anothersubmodule'], - {'foo': 'someattr'} - ) - - This functionality requires Python 3.7 or higher. - - Parameters - ---------- - package_name : str - Typically use ``__name__``. - submodules : set - List of submodules to attach. - submod_attrs : dict - Dictionary of submodule -> list of attributes / functions. - These attributes are imported as they are used. - - Returns - ------- - __getattr__, __dir__, __all__ - - """ - if submod_attrs is None: - submod_attrs = {} - - if submodules is None: - submodules = set() - else: - submodules = set(submodules) - - attr_to_modules = { - attr: mod for mod, attrs in submod_attrs.items() for attr in attrs - } - - __all__ = list(submodules | attr_to_modules.keys()) - - def __getattr__(name): - if name in submodules: - return importlib.import_module(f'{package_name}.{name}') - elif name in attr_to_modules: - submod = importlib.import_module( - f'{package_name}.{attr_to_modules[name]}' - ) - return getattr(submod, name) - else: - raise AttributeError(f'No {package_name} attribute {name}') - - def __dir__(): - return __all__ - - eager_import = os.environ.get('EAGER_IMPORT', '') - if eager_import not in ['', '0', 'false']: - for attr in set(attr_to_modules.keys()) | submodules: - __getattr__(attr) - - return __getattr__, __dir__, list(__all__) - - -def load(fullname): - """Return a lazily imported proxy for a module. - - We often see the following pattern:: - - def myfunc(): - import scipy as sp - sp.argmin(...) - .... - - This is to prevent a module, in this case `scipy`, from being - imported at function definition time, since that can be slow. - - This function provides a proxy module that, upon access, imports - the actual module. So the idiom equivalent to the above example is:: - - sp = lazy.load("scipy") - - def myfunc(): - sp.argmin(...) - .... - - The initial import time is fast because the actual import is delayed - until the first attribute is requested. The overall import time may - decrease as well for users that don't make use of large portions - of the library. - - Parameters - ---------- - fullname : str - The full name of the module or submodule to import. For example:: - - sp = lazy.load('scipy') # import scipy as sp - spla = lazy.load('scipy.linalg') # import scipy.linalg as spla - - Returns - ------- - pm : importlib.util._LazyModule - Proxy module. Can be used like any regularly imported module. - Actual loading of the module occurs upon first attribute request. - - """ - try: - return sys.modules[fullname] - except KeyError: - pass - - spec = importlib.util.find_spec(fullname) - if spec is None: - raise ModuleNotFoundError(f"No module name '{fullname}'") - - module = importlib.util.module_from_spec(spec) - sys.modules[fullname] = module - - loader = importlib.util.LazyLoader(spec.loader) - loader.exec_module(module) - - return module diff --git a/python/cucim/src/cucim/skimage/data/__init__.py b/python/cucim/src/cucim/skimage/data/__init__.py index 00c7daddc..62d86f776 100644 --- a/python/cucim/src/cucim/skimage/data/__init__.py +++ b/python/cucim/src/cucim/skimage/data/__init__.py @@ -1,9 +1,3 @@ -from .._shared import lazy +import lazy_loader as lazy -__getattr__, __dir__, __all__ = lazy.attach( - __name__, - submodules={}, - submod_attrs={ - '_binary_blobs': ['binary_blobs'], - } -) +__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__) diff --git a/python/cucim/src/cucim/skimage/data/__init__.pyi b/python/cucim/src/cucim/skimage/data/__init__.pyi new file mode 100644 index 000000000..88deb1e76 --- /dev/null +++ b/python/cucim/src/cucim/skimage/data/__init__.pyi @@ -0,0 +1,5 @@ +__all__ = [ + 'binary_blobs', +] + +from ._binary_blobs import binary_blobs diff --git a/python/cucim/src/cucim/skimage/filters/__init__.py b/python/cucim/src/cucim/skimage/filters/__init__.py index 32cfe78d8..62d86f776 100644 --- a/python/cucim/src/cucim/skimage/filters/__init__.py +++ b/python/cucim/src/cucim/skimage/filters/__init__.py @@ -1,34 +1,3 @@ -from .._shared import lazy +import lazy_loader as lazy -__getattr__, __dir__, __all__ = lazy.attach( - __name__, - # submodules={'rank'}, - submod_attrs={ - 'lpi_filter': [ - 'filter_forward', 'filter_inverse', 'wiener', 'LPIFilter2D' - ], - '_gaussian': ['gaussian', 'difference_of_gaussians'], - 'edges': ['sobel', 'sobel_h', 'sobel_v', - 'scharr', 'scharr_h', 'scharr_v', - 'prewitt', 'prewitt_h', 'prewitt_v', - 'roberts', 'roberts_pos_diag', 'roberts_neg_diag', - 'laplace', - 'farid', 'farid_h', 'farid_v'], - '_rank_order': ['rank_order'], - '_gabor': ['gabor_kernel', 'gabor'], - 'thresholding': [ - 'threshold_local', 'threshold_otsu', 'threshold_yen', - 'threshold_isodata', 'threshold_li', 'threshold_minimum', - 'threshold_mean', 'threshold_triangle', - 'threshold_niblack', 'threshold_sauvola', - 'threshold_multiotsu', 'try_all_threshold', - 'apply_hysteresis_threshold' - ], - 'ridges': ['meijering', 'sato', 'frangi', 'hessian'], - '_median': ['median'], - '_sparse': ['correlate_sparse'], - '_unsharp_mask': ['unsharp_mask'], - '_window': ['window'], - '_fft_based': ['butterworth'] - } -) +__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__) diff --git a/python/cucim/src/cucim/skimage/filters/__init__.pyi b/python/cucim/src/cucim/skimage/filters/__init__.pyi new file mode 100644 index 000000000..c7703e933 --- /dev/null +++ b/python/cucim/src/cucim/skimage/filters/__init__.pyi @@ -0,0 +1,68 @@ +__all__ = [ +'LPIFilter2D', +'apply_hysteresis_threshold', +'butterworth', +'correlate_sparse', +'difference_of_gaussians', +'farid', +'farid_h', +'farid_v', +'filter_forward', +'filter_inverse', +'frangi', +'gabor', +'gabor_kernel', +'gaussian', +'hessian', +'laplace', +'median', +'meijering', +'prewitt', +'prewitt_h', +'prewitt_v', +'rank_order', +'roberts', +'roberts_neg_diag', +'roberts_pos_diag', +'sato', +'scharr', +'scharr_h', +'scharr_v', +'sobel', +'sobel_h', +'sobel_v', +'threshold_isodata', +'threshold_li', +'threshold_local', +'threshold_mean', +'threshold_minimum', +'threshold_multiotsu', +'threshold_niblack', +'threshold_otsu', +'threshold_sauvola', +'threshold_triangle', +'threshold_yen', +'try_all_threshold', +'unsharp_mask', +'wiener', +'window' +] + +from ._fft_based import butterworth +from ._gabor import gabor, gabor_kernel +from ._gaussian import difference_of_gaussians, gaussian +from ._median import median +from ._rank_order import rank_order +from ._sparse import correlate_sparse +from ._unsharp_mask import unsharp_mask +from ._window import window +from .edges import (farid, farid_h, farid_v, laplace, prewitt, prewitt_h, + prewitt_v, roberts, roberts_neg_diag, roberts_pos_diag, + scharr, scharr_h, scharr_v, sobel, sobel_h, sobel_v) +from .lpi_filter import LPIFilter2D, filter_forward, filter_inverse, wiener +from .ridges import frangi, hessian, meijering, sato +from .thresholding import (apply_hysteresis_threshold, threshold_isodata, + threshold_li, threshold_local, threshold_mean, + threshold_minimum, threshold_multiotsu, + threshold_niblack, threshold_otsu, threshold_sauvola, + threshold_triangle, threshold_yen, try_all_threshold)