Merge pull request #529 from analysiscenter/requirements

requirements.txt

.github/workflows/pr.yml

@@ -28,3 +28,32 @@ jobs:
  run: |
  pip3 install -U codecov
  codecov -t ${{ secrets.CODECOV_TOKEN }}
+
+ test_requirements:
+
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ python-version: [3.6, 3.7]
+
+ steps:
+ - uses: actions/checkout@v1
+
+ - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+ uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install batchflow requirements
+ run: |
+ pip install -r requirements.txt
+ pip install pytest
+ 
+ - name: Run 'import batchflow' in installed environemnt
+ run: python -c 'import batchflow' 
+
+ - name: Run basic tests
+ run: |
+ cd batchflow/tests
+ pytest --disable-pytest-warnings -v dataset_test.py filesindex_test.py datasetindex_test.py

batchflow/batch_image.py

@@ -4,9 +4,6 @@
 from numbers import Number
 
 import numpy as np
-from skimage.transform import resize # pylint: disable=unused-import
-import scipy.ndimage
-
 import PIL
 import PIL.ImageOps
 import PIL.ImageChops
@@ -18,60 +15,6 @@
 from .dsindex import FilesIndex
 
 
-def get_scipy_transforms():
- """ Returns ``dict`` {'function_name' : function} of functions from scipy.ndimage.
-
- Function is included if it has 'input : ndarray' or 'input : array_like' in its docstring.
- """
- scipy_transformations = {}
- ref_counter = 1001
- hooks = ['input : ndarray', 'input : array_like']
-
- for function_name in scipy.ndimage.__dict__['__all__']:
- function = getattr(scipy.ndimage, function_name)
- doc = getattr(function, '__doc__')
- if doc is not None and (hooks[0] in doc or hooks[1] in doc):
- # Re-enumerate references in docs and add missing links
- for i in range(5):
- ref = '[{}]'.format(i)
-
- if ref in doc[doc.find('References'):]:
- place = doc.find('Parameters') - 6
- doc = '\n'.join([doc[:place],
- '\n See [{}]_ for more details.'.format(ref_counter),
- doc[place:]])
- if ref in doc:
- doc = doc.replace(ref, '[{}]'.format(ref_counter))
- ref_counter += 1
- function.__doc__ = doc
-
- scipy_transformations[function_name] = function
- return scipy_transformations
-
-
-def add_methods(transformations=None, prefix='_', suffix='_'):
- """ Bounds given functions to a decorated class
-
- All bounded methods' names will be extended with ``prefix`` and ``suffix``.
- For example, if ``transformations``={'method_name': method}, ``suffix``='_all' and ``prefix``='_'
- then a decorated class will have '_method_name_all' method.
-
- Parameters
- ----------
- transformations : dict
- dict of the form {'method_name' : function_to_bound} -- functions to bound to a class
- prefix : str
- suffix : str
- """
- def _decorator(cls):
- for func_name, func in transformations.items():
- method_name = ''.join((prefix, func_name, suffix))
- added_method = staticmethod(func)
- setattr(cls, method_name, added_method)
- return cls
- return _decorator
-
-
 class BaseImagesBatch(Batch):
  """ Batch class for 2D images.
 
@@ -204,9 +147,6 @@ def dump(self, *args, dst=None, fmt=None, components="images", **kwargs):
  return super().dump(dst=dst, fmt=fmt, components=components, *args, **kwargs)
 
 
-@add_methods(transformations={**get_scipy_transforms(),
- 'pad': np.pad,
- 'resize': resize}, prefix='_sp_', suffix='_')
 class ImagesBatch(BaseImagesBatch):
  """ Batch class for 2D images.
 

batchflow/components.py

@@ -1,10 +1,11 @@
 """ Contains classes to handle batch data components """
 import copy as cp
+
 import numpy as np
 try:
  import pandas as pd
 except ImportError:
- import _fake as pd
+ from . import _fake as pd
 
 from .utils import is_iterable
 

batchflow/decorators.py

@@ -382,9 +382,9 @@ def parallel(*args, use_self=None, **kwargs):
  return inbatch_parallel(*args, _use_self=use_self, **kwargs)
 
 
-def njit(nogil=True):
+def njit(nogil=True, parallel=True): # pylint: disable=redefined-outer-name
  """ Fake njit decorator to use when numba is not installed """
- _ = nogil
+ _, _ = nogil, parallel
  def njit_fake_decorator(method):
  """ Return a decorator """
  @functools.wraps(method)

batchflow/models/metrics/utils.py

@@ -1,12 +1,17 @@
 """ Contains utility function for metrics evaluation """
 import numpy as np
 import numpy.ma as ma
+try:
+ from scipy.ndimage import measurements
+except ImportError:
+ pass
+try:
+ from numba import njit
+except ImportError:
+ from ...decorators import njit
 
-from numba import njit
-from scipy.ndimage import measurements
 
-
-@njit(nogil=True)
+@njit(nogil=True, parallel=True)
 def binarize(inputs, threshold=.5):
  """ Create a binary mask from probabilities with a given threshold.
 
@@ -26,7 +31,7 @@ def binarize(inputs, threshold=.5):
  return inputs >= threshold
 
 
-@njit(nogil=True)
+@njit(nogil=True, parallel=True)
 def sigmoid(arr):
  return 1. / (1. + np.exp(-arr))
 

batchflow/models/tf/__init__.py

@@ -1,4 +1,8 @@
-""" Contains tensorflow models and functions """
+""" Contains tensorflow models and functions.
+
+.. note::
+ This module requires TensorFlow package.
+ """
 import sys
 import tensorflow as tf_
 

batchflow/models/torch/__init__.py

@@ -1,4 +1,8 @@
-""" Various Torch models. """
+""" Various Torch models and blocks.
+
+.. note::
+ This module requries PyTorch package.
+"""
 from .base import TorchModel
 from .encoder_decoder import Encoder, Decoder, EncoderDecoder, AutoEncoder, VariationalAutoEncoder
 from .vgg import VGG, VGG7, VGG16, VGG19

batchflow/models/torch/encoder_decoder.py

@@ -6,7 +6,7 @@
 
 from .base import TorchModel
 from .utils import get_shape
-from .layers import ConvBlock, Upsample, Combine, Crop
+from .layers import ConvBlock, Upsample, Combine
 from .blocks import DefaultBlock
 from ..utils import unpack_args
 

batchflow/models/torch/layers/conv_block.py

@@ -354,7 +354,6 @@ def _make_modules(self, inputs):
  shape_before = [str((None, *shape[1:])) for shape in shape_before]
  shape_after = (None, *shape_after[1:])
  layer_desc = 'Layer {}, combine "{}": {}'.format(i, letter, shape_before[0])
- starter = ('\n' + ' '*len(layer_desc))
  for shape in shape_before[1:]:
  layer_desc += '\n' + ' '*(len(layer_desc) - len(shape)) + shape
  layer_desc += ' -> {}'.format(shape_after)

batchflow/monitor.py

@@ -4,10 +4,12 @@
 from math import ceil
 from multiprocessing import Process, Manager, Queue
 
-import psutil
 import numpy as np
 import matplotlib.pyplot as plt
-
+try:
+ import psutil
+except ImportError:
+ pass
 try:
  import nvidia_smi
 except ImportError:

batchflow/notifier.py

@@ -7,8 +7,11 @@
 from tqdm.auto import tqdm as tqdm_auto
 
 import numpy as np
-from IPython import display
 import matplotlib.pyplot as plt
+try:
+ from IPython import display
+except ImportError:
+ pass
 
 from .monitor import ResourceMonitor, MONITOR_ALIASES
 from .named_expr import NamedExpression, eval_expr

batchflow/opensets/imagenette.py

@@ -11,7 +11,6 @@
 import PIL
 import tqdm
 import numpy as np
-from sklearn.preprocessing import LabelEncoder
 
 from . import ImagesOpenset
 
@@ -68,7 +67,7 @@ def _is_file_rgb(archive, member):
  def _gather_extracted(archive, files):
  images = np.array([_extract(archive, file) for file in files], dtype=object)
  labels = np.array([_image_class(file.name) for file in files])
- labels_encoded = LabelEncoder().fit_transform(labels)
+ _, labels_encoded = np.unique(labels, return_inverse=True)
  return images, labels_encoded
 
  if path is None:

batchflow/pipeline.py

@@ -13,7 +13,10 @@
 from pstats import Stats
 import queue as q
 import numpy as np
-import pandas as pd
+try:
+ import pandas as pd
+except ImportError:
+ from . import _fake as pd
 
 from .base import Baseset
 from .config import Config

batchflow/research/__init__.py

@@ -1,4 +1,8 @@
-""" Research module. """
+""" Research module.
+
+.. note::
+ This module requries multiprocess package <http://multiprocess.rtfd.io/>`_.
+"""
 from .domain import KV, Domain, Option, ConfigAlias
 from .distributor import Distributor
 from .logger import BaseLogger, FileLogger, PrintLogger, TelegramLogger

batchflow/sampler.py

@@ -3,7 +3,10 @@
 
 from copy import copy
 import numpy as np
-import scipy.stats as ss
+try:
+ import scipy.stats as ss
+except ImportError:
+ pass
 
 # if empirical probability of truncation region is less than
 # this number, truncation throws a ValueError

batchflow/tests/config_pass_test.py

@@ -4,7 +4,6 @@
 import pytest
 
 from batchflow import Config
-from batchflow.models.tf import TFModel
 
 
 
@@ -15,6 +14,7 @@
 @pytest.fixture()
 def single_config():
  """ Fixture that returns the simplest config for single-input model. """
+ from batchflow.models.tf import TFModel
 
  class SingleModel(TFModel):
  model_args = Config()
@@ -66,6 +66,7 @@ def head(cls, inputs, name='head', **kwargs):
 @pytest.fixture()
 def multi_config():
  """ Fixture that returns the simplest config for multi-input model. """
+ from batchflow.models.tf import TFModel
 
  class MultiModel(TFModel):
  model_args = Config()

batchflow/utils.py

@@ -4,7 +4,10 @@
 import itertools
 
 import numpy as np
-import pandas as pd
+try:
+ import pandas as pd
+except ImportError:
+ from . import _fake as pd
 from matplotlib import pyplot as plt
 from matplotlib import colors as mcolors
 

batchflow/utils_notebook.py

@@ -2,10 +2,8 @@
 import os
 import re
 import json
-import requests
-
-# Additionally imports `ipykernel`, `notebook`, `nbconvert` and `pylint`, if needed
 
+# Additionally imports 'requests`, 'ipykernel`, `notebook`, `nbconvert` and `pylint`, if needed
 
 
 def in_notebook():
@@ -29,6 +27,7 @@ def get_notebook_path():
  if not in_notebook():
  return None
 
+ import requests
  import ipykernel
  from notebook.notebookapp import list_running_servers
 

docs/api/batchflow.models.tf.rst

@@ -2,6 +2,8 @@
 batchflow.models.tf
 ===================
 
+.. automodule:: batchflow.models.tf
+
 .. toctree::
 
  batchflow.models.tf.models

docs/api/batchflow.models.torch.rst

@@ -2,6 +2,8 @@
 batchflow.models.torch
 ======================
 
+.. automodule:: batchflow.models.torch
+
 .. toctree::
 
  batchflow.models.torch.models

docs/api/batchflow.research.rst

@@ -2,6 +2,7 @@
 batchflow.research
 ==================
 
+.. automodule:: batchflow.research
 
 Research
 --------

docs/index.rst

@@ -127,6 +127,22 @@ Relative import is also possible::
  from .batchflow import Dataset
 
 
+Requirements
+-----------
+
+BatchFlow has the following strict requirments:
+
+* numpy
+* dill
+* tqdm
+* matplotlib
+
+Some BatchFlow modules also depend on other mandatory packages:
+
+* pandas and multiprocess for ``batchflow.research``
+* torch for ``batchflow.models.torch``
+* tensorflow ``for batchflow.models.tf``
+
 
 Citing BatchFlow
 ================