Merge pull request #155 from ExaScience/parallel_ci

Parallel CI

.github/workflows/conda.yml

@@ -8,11 +8,12 @@ on:
 
 jobs:
   conda-build:
-    name: Build Conda Package on ${{ matrix.os }}
+    name: Build Conda Package on ${{ matrix.os }} for Python ${{ matrix.python }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
           os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+          python: [ "3.8", "3.9", "3.10", "3.11"]
     defaults:
       run:
         shell: bash -el {0}
@@ -26,14 +27,7 @@ jobs:
           channels: conda-forge
           activate-environment: devtools
           environment-file: conda-recipes/devtools.yml
-      - run: conda info
-      - run: conda list
-      - run: conda config --show
-      - name: Conda Info
-        run: |
-          conda info
-      - run: git describe --tags
       - name: Run 'conda build'
         run: |
-          conda build -c vanderaa smurff
+          conda build -c vanderaa --python ${{ matrix.python }} smurff
         working-directory: conda-recipes

.github/workflows/wheels.yml

@@ -13,8 +13,7 @@ jobs:
     strategy:
       matrix:
           os: [ubuntu-latest, macos-13, macos-14]
-          # os: [ubuntu-latest, windows-latest, macos-13, macos-14]
-
+          pyver: [cp39, cp310, cp311, cp312]
     steps:
       - uses: actions/checkout@v4
 
@@ -25,6 +24,8 @@ jobs:
         run: python -m pip install cibuildwheel==2.18.1
 
       - name: Build wheels
+        env:
+          CIBW_BUILD: ${{matrix.pyver}}-*
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v4

ci/docker/Dockerfile.alpine

@@ -0,0 +1,20 @@
+FROM alpine:3.19
+
+RUN apk add \
+        binutils gcc g++ gfortran \
+        cmake ninja make \
+	wget curl util-linux tar \
+        git \
+        eigen-dev openblas-dev hdf5-dev boost-dev catch2-3 \
+        py3-pybind11-dev python3-dev py3-pip py3-pytest py3-parameterized py3-pytest-xdist
+
+#install HighFive
+RUN wget -O HighFive.tar.gz https://github.com/BlueBrain/HighFive/archive/v2.2.2.tar.gz && \
+    tar xzf HighFive.tar.gz && \
+    rm HighFive.tar.gz && \
+    cd HighFive* && \
+    cmake -S . -B build -GNinja .. -DHIGHFIVE_USE_BOOST=OFF && \
+    cmake --build build && \
+    cmake --install build && \
+    cd ../.. && \
+    rm -r HighFive*

ci/docker/Dockerfile.musllinux

@@ -1,6 +1,6 @@
 FROM quay.io/pypa/musllinux_1_2_x86_64
 
-RUN apk add wget eigen openblas hdf5-dev
+RUN apk add wget eigen-dev openblas-dev hdf5-dev
 
 #install HighFive
 RUN wget -O HighFive.tar.gz https://github.com/BlueBrain/HighFive/archive/v2.2.2.tar.gz && \

ci/docker/Dockerfile.ubuntu

@@ -21,7 +21,9 @@ RUN apt-get update && \
        python3-scipy python3-pandas \
        python3-joblib python3-sklearn \
        python3-h5py \
-       python3-pytest python3-parameterized \
+       python3-pytest \
+       python3-parameterized \
+       python3-pytest-xdist \
     && rm -rf /var/lib/apt/lists/*
 
 

ci/docker/build_script.sh

@@ -18,11 +18,13 @@ git config --global --add safe.directory /smurff/.git
 git clone /smurff
 
 cd smurff
-cmake -S . -B build
+cmake -S . -B build -GNinja
 cmake --build build
 cmake --install build
+smurff --bist
 
-python3 -m pip install .
+python3 -m venv .venv
+. .venv/bin/activate
+pip install -v .
 
-smurff --bist
-pytest-3 python/test
+pytest-3 -n auto -v python/test

conda-recipes/smurff/build.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export CMAKE_GENERATOR="Ninja"
+
 if [ "$blas_impl" == "mkl" ]
 then
     SKBUILD_CMAKE_ARGS="-DENABLE_MKL=ON -DENABLE_OPENBLAS=OFF"

conda-recipes/smurff/meta.yaml

@@ -13,6 +13,7 @@ build:
 requirements:
   build:
     - cmake
+    - ninja
     - {{ compiler('cxx') }}
     - {{ compiler('c') }}
     - llvm-openmp             # [osx]
@@ -45,8 +46,9 @@ requirements:
 test:
   requires:
     - setuptools
-    - parameterized
     - pytest
+    - parameterized
+    - pytest-xdist
   source_files:
     - python/test/*.py
 

conda-recipes/smurff/run_test.bat

@@ -1,4 +1,4 @@
 %CONDA_PREFIX%\Scripts\smurff --bist ~[random]
 if errorlevel 1 exit 1
-%PYTHON% -m pytest
+%PYTHON% -m pytest -n auto -v
 if errorlevel 1 exit 1

conda-recipes/smurff/run_test.sh

@@ -1,2 +1,2 @@
 $PREFIX/bin/smurff --bist
-$PYTHON -m pytest -v python/test
+$PYTHON -m pytest -n auto -v python/test

pyproject.toml

@@ -4,8 +4,6 @@ requires = [
     "setuptools_scm",
     "pybind11",
     "scikit-build-core",
-    "cmake>=3.18",
-    "ninja",
 ]
 build-backend = "scikit_build_core.build"
 
@@ -73,12 +71,10 @@ musllinux-x86_64-image = "vanderaa/musllinux_1_2_x86_64_smurff"
 # - CPython 3.6: unsupported by scikit_build_core
 # - CPython 3.7: unsupported by h5sparse
 # - CPython 3.8: removed from manylinux/musllinux
-# - CPython 3.12: removed pkg_resources in h5sparse
 # - i686 and win32: we do not care about 32bit
-skip = "pp* cp36-* cp37-* cp38-* cp312-* *-win32 *i686"
-test-command = 'pytest {project}/python/test'
-test-requires = 'parameterized pytest'
-build-verbosity = 3
+skip = "pp* cp36-* cp37-* cp38-* *musl* *-win32 *i686"
+test-command = 'pytest -n auto {project}/python/test'
+test-requires = 'parameterized pytest pytest-xdist'
 
 [tool.cibuildwheel.macos.config-settings]
 "cmake.define.HDF5_ROOT" = "/usr/local/hdf5"

python/test/test_gfa.py

@@ -16,27 +16,27 @@ class TestGFA(unittest.TestCase):
     def test_gfa_1view(self):
         Y = scipy.sparse.rand(10, 20, 0.2)
         Y, Ytest = smurff.make_train_test(Y, 0.5)
-        predictions = smurff.gfa([Y], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5)
+        predictions = smurff.gfa([Y], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5)
         self.assertEqual(Ytest.nnz, len(predictions))
 
     def test_gfa_2view(self):
         Y = scipy.sparse.rand(10, 20, 0.2)
         Y, Ytest = smurff.make_train_test(Y, 0.5)
-        predictions = smurff.gfa([Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5)
+        predictions = smurff.gfa([Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5)
         self.assertEqual(Ytest.nnz, len(predictions))
 
     def test_gfa_3view(self):
         Y = scipy.sparse.rand(10, 20, 0.2)
         Y, Ytest = smurff.make_train_test(Y, 0.5)
-        predictions = smurff.gfa([Y, Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5)
+        predictions = smurff.gfa([Y, Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5)
         self.assertEqual(Ytest.nnz, len(predictions))
 
     def test_gfa_mixedview(self):
         Y = scipy.sparse.rand(10, 20, 0.2)
         Y, Ytest = smurff.make_train_test(Y, 0.5)
         D1 = np.random.randn(10, 2)
         D2 = scipy.sparse.rand(10, 5, 0.2)
-        predictions = smurff.gfa([Y, D1, D2], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5)
+        predictions = smurff.gfa([Y, D1, D2], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5)
         self.assertEqual(Ytest.nnz, len(predictions))
 
 

python/test/test_macau.py

@@ -29,6 +29,7 @@ def test_macau(self):
                                 direct=True,
                                 num_latent=4,
                                 verbose=verbose,
+                                num_threads=1,
                                 burnin=200,
                                 nsamples=200)
 
@@ -46,7 +47,9 @@ def test_macau_side_bin(self):
                       num_latent=5,
                       burnin=200,
                       nsamples=200,
-                      verbose=verbose)
+                      verbose=verbose,
+                      num_threads=1,
+                      )
 
     def test_macau_dense(self):
         Y  = scipy.sparse.rand(15, 10, 0.2)
@@ -59,7 +62,9 @@ def test_macau_dense(self):
                       num_latent=5,
                       burnin=200,
                       nsamples=200,
-                      verbose=verbose)
+                      verbose=verbose,
+                      num_threads=1
+                      )
 
     def test_macau_univariate(self):
         Y = scipy.sparse.rand(10, 20, 0.2)
@@ -73,13 +78,14 @@ def test_macau_univariate(self):
                                 univariate = True,
                                 num_latent=4,
                                 verbose=verbose,
+                                num_threads=1,
                                 burnin=200,
                                 nsamples=200)
         self.assertEqual(Ytest.nnz, len(predictions))
 
     def test_macau_tensor(self):
         shape = [30, 4, 2]
-        
+
         A = np.random.randn(shape[0], 2)
         B = np.random.randn(shape[1], 2)
         C = np.random.randn(shape[2], 2)
@@ -97,6 +103,7 @@ def test_macau_tensor(self):
              direct=True,
 			 num_latent = 4,
 			 verbose=verbose,
+             num_threads=1,
 			 burnin=200,
 			 nsamples=200)
 
@@ -125,6 +132,7 @@ def test_macau_tensor_univariate(self):
                                 univariate = True,
                                 num_latent=4,
                                 verbose=verbose,
+                                num_threads=1,
                                 burnin=200,
                                 nsamples=2000)
 

python/test/test_noisemodels.py

@@ -59,7 +59,7 @@ def test_noise_model(density, nmodes, side_info, noise_model):
     if si is not None:
         priors[0] = 'macau'
 
-    trainSession = smurff.TrainSession(priors = priors, num_latent=8, burnin=20, nsamples=20, threshold=.0, seed=seed, verbose=verbose)
+    trainSession = smurff.TrainSession(priors = priors, num_latent=8, burnin=20, nsamples=20, threshold=.0, seed=seed, verbose=verbose, num_threads=1)
 
     trainSession.addTrainAndTest(Ytrain, Ytest, nm)
     if not si is None:

python/test/test_pp.py

@@ -35,7 +35,7 @@ def test_bmf_pp(self):
         Y = scipy.sparse.rand(30, 20, 0.2)
         Y, Ytest = smurff.make_train_test(Y, 0.5, seed=seed)
         trainSession = smurff.BPMFSession(Y, is_scarce = True, Ytest=Ytest,
-                num_latent=4, verbose=verbose, burnin=20, nsamples=20, save_freq=1,
+                num_latent=4, verbose=verbose, num_threads = 1, burnin=20, nsamples=20, save_freq=1,
                 seed = seed, save_name=smurff.helper.temp_savename())
         trainSession.run()
         predict_session = trainSession.makePredictSession()

python/test/test_predict.py

@@ -16,13 +16,13 @@ class TestPredictSession(unittest.TestCase):
     __name__ = "TestPredictSession"
 
     def run_train_session(self, nmodes, density):
-        shape = range(5, nmodes+5) # 5, 6, 7, ... 
+        shape = range(5, nmodes+5) # 5, 6, 7, ...
         Y, X = smurff.generate.gen_tensor(shape, 3, density)
         self.Ytrain, self.Ytest = smurff.make_train_test(Y, 0.1)
         priors = ['normal'] * nmodes
 
         trainSession = smurff.TrainSession(priors = priors, num_latent=4,
-                burnin=10, nsamples=nsamples, verbose=verbose,
+                burnin=10, nsamples=nsamples, verbose=verbose, num_threads=1,
                 save_freq = 1, save_name = smurff.helper.temp_savename())
 
         trainSession.addTrainAndTest(self.Ytrain, self.Ytest)
@@ -47,7 +47,7 @@ def assert_almost_equal_sparse(self, A, B):
         c1,v1 = smurff.find(A)
         c2,v2 = smurff.find(B)
         assert np.array_equal(c1,c2)
-        assert np.allclose(v1,v2, atol=0.01) 
+        assert np.allclose(v1,v2, atol=0.01)
 
     def run_predict_some_all_one(self, train_session, predict_session):
         coords, _ = smurff.find(self.Ytest)
@@ -83,7 +83,7 @@ def run_predict_predict(self, predict_session, X):
         """ Test the PredictSession.predict function """
 
         def run_n_samples(samples, expected_nsamples):
-            operand_and_sizes = [ 
+            operand_and_sizes = [
                 [
                     ( Ellipsis   , x.shape[0] ),
                     ( slice(3)   , 3          ),

python/test/test_pybind.py

@@ -4,7 +4,14 @@
 import scipy.sparse as sp
 
 def test_pybind():
-    trainSession = smurff.TrainSession(priors = ["normal", "normal"], verbose = 2 )
+    trainSession = smurff.TrainSession(
+        priors = ["normal", "normal"],
+        burnin = 10,
+        nsamples = 10,
+        num_latent = 4,
+        verbose = 2,
+        num_threads = 1,
+        )
 
     Y = np.array([[1.,2.],[3.,4.]])
     trainSession.setTrain(Y)

python/test/test_scarce.py

@@ -22,16 +22,16 @@ class TestScarce(unittest.TestCase):
     def test_simple(self):
         matrix = matrix_with_explicit_zeros()
         self.assertTrue(matrix.nnz == 6)
-        
+
         matrix.eliminate_zeros()
         self.assertTrue(matrix.nnz == 3)
 
     def test_smurff(self):
         matrix = matrix_with_explicit_zeros()
         self.assertTrue(matrix.nnz == 6)
 
-        predictions = smurff.bpmf(matrix, Ytest=matrix, num_latent=4, burnin=5, nsamples=5)
+        predictions = smurff.bpmf(matrix, Ytest=matrix, num_latent=4, burnin=5, nsamples=5, num_threads=1)
         self.assertEqual(len(predictions), 6)
-        
+
 if __name__ == '__main__':
     unittest.main()