diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 4b327d0b..e29b6387 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -8,11 +8,12 @@ on: jobs: conda-build: - name: Build Conda Package on ${{ matrix.os }} + name: Build Conda Package on ${{ matrix.os }} for Python ${{ matrix.python }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, windows-latest, macos-13, macos-14] + python: [ "3.8", "3.9", "3.10", "3.11"] defaults: run: shell: bash -el {0} @@ -26,14 +27,7 @@ jobs: channels: conda-forge activate-environment: devtools environment-file: conda-recipes/devtools.yml - - run: conda info - - run: conda list - - run: conda config --show - - name: Conda Info - run: | - conda info - - run: git describe --tags - name: Run 'conda build' run: | - conda build -c vanderaa smurff + conda build -c vanderaa --python ${{ matrix.python }} smurff working-directory: conda-recipes \ No newline at end of file diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a313f522..d2b8d00a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -13,8 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-13, macos-14] - # os: [ubuntu-latest, windows-latest, macos-13, macos-14] - + pyver: [cp39, cp310, cp311, cp312] steps: - uses: actions/checkout@v4 @@ -25,6 +24,8 @@ jobs: run: python -m pip install cibuildwheel==2.18.1 - name: Build wheels + env: + CIBW_BUILD: ${{matrix.pyver}}-* run: python -m cibuildwheel --output-dir wheelhouse - uses: actions/upload-artifact@v4 diff --git a/ci/docker/Dockerfile.alpine b/ci/docker/Dockerfile.alpine new file mode 100644 index 00000000..d0dfef9a --- /dev/null +++ b/ci/docker/Dockerfile.alpine @@ -0,0 +1,20 @@ +FROM alpine:3.19 + +RUN apk add \ + binutils gcc g++ gfortran \ + cmake ninja make \ + wget curl util-linux tar \ + git \ + eigen-dev openblas-dev hdf5-dev boost-dev catch2-3 \ + py3-pybind11-dev python3-dev py3-pip py3-pytest py3-parameterized py3-pytest-xdist + +#install HighFive +RUN wget -O HighFive.tar.gz https://github.com/BlueBrain/HighFive/archive/v2.2.2.tar.gz && \ + tar xzf HighFive.tar.gz && \ + rm HighFive.tar.gz && \ + cd HighFive* && \ + cmake -S . -B build -GNinja .. -DHIGHFIVE_USE_BOOST=OFF && \ + cmake --build build && \ + cmake --install build && \ + cd ../.. && \ + rm -r HighFive* diff --git a/ci/docker/Dockerfile.musllinux b/ci/docker/Dockerfile.musllinux index 22ae4cd3..5ab15ec4 100644 --- a/ci/docker/Dockerfile.musllinux +++ b/ci/docker/Dockerfile.musllinux @@ -1,6 +1,6 @@ FROM quay.io/pypa/musllinux_1_2_x86_64 -RUN apk add wget eigen openblas hdf5-dev +RUN apk add wget eigen-dev openblas-dev hdf5-dev #install HighFive RUN wget -O HighFive.tar.gz https://github.com/BlueBrain/HighFive/archive/v2.2.2.tar.gz && \ diff --git a/ci/docker/Dockerfile.ubuntu b/ci/docker/Dockerfile.ubuntu index c6576204..ece0b1a5 100644 --- a/ci/docker/Dockerfile.ubuntu +++ b/ci/docker/Dockerfile.ubuntu @@ -21,7 +21,9 @@ RUN apt-get update && \ python3-scipy python3-pandas \ python3-joblib python3-sklearn \ python3-h5py \ - python3-pytest python3-parameterized \ + python3-pytest \ + python3-parameterized \ + python3-pytest-xdist \ && rm -rf /var/lib/apt/lists/* diff --git a/ci/docker/build_script.sh b/ci/docker/build_script.sh index b7ee9622..1fc3feab 100755 --- a/ci/docker/build_script.sh +++ b/ci/docker/build_script.sh @@ -18,11 +18,13 @@ git config --global --add safe.directory /smurff/.git git clone /smurff cd smurff -cmake -S . -B build +cmake -S . -B build -GNinja cmake --build build cmake --install build +smurff --bist -python3 -m pip install . +python3 -m venv .venv +. .venv/bin/activate +pip install -v . -smurff --bist -pytest-3 python/test +pytest-3 -n auto -v python/test diff --git a/conda-recipes/smurff/build.sh b/conda-recipes/smurff/build.sh index b63222cc..f6ce2242 100644 --- a/conda-recipes/smurff/build.sh +++ b/conda-recipes/smurff/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +export CMAKE_GENERATOR="Ninja" + if [ "$blas_impl" == "mkl" ] then SKBUILD_CMAKE_ARGS="-DENABLE_MKL=ON -DENABLE_OPENBLAS=OFF" diff --git a/conda-recipes/smurff/meta.yaml b/conda-recipes/smurff/meta.yaml index 19a95785..86d98c5c 100644 --- a/conda-recipes/smurff/meta.yaml +++ b/conda-recipes/smurff/meta.yaml @@ -13,6 +13,7 @@ build: requirements: build: - cmake + - ninja - {{ compiler('cxx') }} - {{ compiler('c') }} - llvm-openmp # [osx] @@ -45,8 +46,9 @@ requirements: test: requires: - setuptools - - parameterized - pytest + - parameterized + - pytest-xdist source_files: - python/test/*.py diff --git a/conda-recipes/smurff/run_test.bat b/conda-recipes/smurff/run_test.bat index 67130bce..e090fb38 100644 --- a/conda-recipes/smurff/run_test.bat +++ b/conda-recipes/smurff/run_test.bat @@ -1,4 +1,4 @@ %CONDA_PREFIX%\Scripts\smurff --bist ~[random] if errorlevel 1 exit 1 -%PYTHON% -m pytest +%PYTHON% -m pytest -n auto -v if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda-recipes/smurff/run_test.sh b/conda-recipes/smurff/run_test.sh index 4dcf42ab..ad44672b 100644 --- a/conda-recipes/smurff/run_test.sh +++ b/conda-recipes/smurff/run_test.sh @@ -1,2 +1,2 @@ $PREFIX/bin/smurff --bist -$PYTHON -m pytest -v python/test \ No newline at end of file +$PYTHON -m pytest -n auto -v python/test \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3958e765..4ec9f267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,6 @@ requires = [ "setuptools_scm", "pybind11", "scikit-build-core", - "cmake>=3.18", - "ninja", ] build-backend = "scikit_build_core.build" @@ -73,12 +71,10 @@ musllinux-x86_64-image = "vanderaa/musllinux_1_2_x86_64_smurff" # - CPython 3.6: unsupported by scikit_build_core # - CPython 3.7: unsupported by h5sparse # - CPython 3.8: removed from manylinux/musllinux -# - CPython 3.12: removed pkg_resources in h5sparse # - i686 and win32: we do not care about 32bit -skip = "pp* cp36-* cp37-* cp38-* cp312-* *-win32 *i686" -test-command = 'pytest {project}/python/test' -test-requires = 'parameterized pytest' -build-verbosity = 3 +skip = "pp* cp36-* cp37-* cp38-* *musl* *-win32 *i686" +test-command = 'pytest -n auto {project}/python/test' +test-requires = 'parameterized pytest pytest-xdist' [tool.cibuildwheel.macos.config-settings] "cmake.define.HDF5_ROOT" = "/usr/local/hdf5" diff --git a/python/test/test_gfa.py b/python/test/test_gfa.py index 3e3b291f..3deceb4b 100644 --- a/python/test/test_gfa.py +++ b/python/test/test_gfa.py @@ -16,19 +16,19 @@ class TestGFA(unittest.TestCase): def test_gfa_1view(self): Y = scipy.sparse.rand(10, 20, 0.2) Y, Ytest = smurff.make_train_test(Y, 0.5) - predictions = smurff.gfa([Y], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5) + predictions = smurff.gfa([Y], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5) self.assertEqual(Ytest.nnz, len(predictions)) def test_gfa_2view(self): Y = scipy.sparse.rand(10, 20, 0.2) Y, Ytest = smurff.make_train_test(Y, 0.5) - predictions = smurff.gfa([Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5) + predictions = smurff.gfa([Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5) self.assertEqual(Ytest.nnz, len(predictions)) def test_gfa_3view(self): Y = scipy.sparse.rand(10, 20, 0.2) Y, Ytest = smurff.make_train_test(Y, 0.5) - predictions = smurff.gfa([Y, Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5) + predictions = smurff.gfa([Y, Y, Y], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5) self.assertEqual(Ytest.nnz, len(predictions)) def test_gfa_mixedview(self): @@ -36,7 +36,7 @@ def test_gfa_mixedview(self): Y, Ytest = smurff.make_train_test(Y, 0.5) D1 = np.random.randn(10, 2) D2 = scipy.sparse.rand(10, 5, 0.2) - predictions = smurff.gfa([Y, D1, D2], Ytest=Ytest, num_latent=4, verbose=verbose, burnin=5, nsamples=5) + predictions = smurff.gfa([Y, D1, D2], Ytest=Ytest, num_latent=4, verbose=verbose, num_threads=1, burnin=5, nsamples=5) self.assertEqual(Ytest.nnz, len(predictions)) diff --git a/python/test/test_macau.py b/python/test/test_macau.py index e6dae931..bb99c8d6 100644 --- a/python/test/test_macau.py +++ b/python/test/test_macau.py @@ -29,6 +29,7 @@ def test_macau(self): direct=True, num_latent=4, verbose=verbose, + num_threads=1, burnin=200, nsamples=200) @@ -46,7 +47,9 @@ def test_macau_side_bin(self): num_latent=5, burnin=200, nsamples=200, - verbose=verbose) + verbose=verbose, + num_threads=1, + ) def test_macau_dense(self): Y = scipy.sparse.rand(15, 10, 0.2) @@ -59,7 +62,9 @@ def test_macau_dense(self): num_latent=5, burnin=200, nsamples=200, - verbose=verbose) + verbose=verbose, + num_threads=1 + ) def test_macau_univariate(self): Y = scipy.sparse.rand(10, 20, 0.2) @@ -73,13 +78,14 @@ def test_macau_univariate(self): univariate = True, num_latent=4, verbose=verbose, + num_threads=1, burnin=200, nsamples=200) self.assertEqual(Ytest.nnz, len(predictions)) def test_macau_tensor(self): shape = [30, 4, 2] - + A = np.random.randn(shape[0], 2) B = np.random.randn(shape[1], 2) C = np.random.randn(shape[2], 2) @@ -97,6 +103,7 @@ def test_macau_tensor(self): direct=True, num_latent = 4, verbose=verbose, + num_threads=1, burnin=200, nsamples=200) @@ -125,6 +132,7 @@ def test_macau_tensor_univariate(self): univariate = True, num_latent=4, verbose=verbose, + num_threads=1, burnin=200, nsamples=2000) diff --git a/python/test/test_noisemodels.py b/python/test/test_noisemodels.py index 6079add7..388abfdf 100644 --- a/python/test/test_noisemodels.py +++ b/python/test/test_noisemodels.py @@ -59,7 +59,7 @@ def test_noise_model(density, nmodes, side_info, noise_model): if si is not None: priors[0] = 'macau' - trainSession = smurff.TrainSession(priors = priors, num_latent=8, burnin=20, nsamples=20, threshold=.0, seed=seed, verbose=verbose) + trainSession = smurff.TrainSession(priors = priors, num_latent=8, burnin=20, nsamples=20, threshold=.0, seed=seed, verbose=verbose, num_threads=1) trainSession.addTrainAndTest(Ytrain, Ytest, nm) if not si is None: diff --git a/python/test/test_pp.py b/python/test/test_pp.py index 77aa4c87..c9ca6a6e 100644 --- a/python/test/test_pp.py +++ b/python/test/test_pp.py @@ -35,7 +35,7 @@ def test_bmf_pp(self): Y = scipy.sparse.rand(30, 20, 0.2) Y, Ytest = smurff.make_train_test(Y, 0.5, seed=seed) trainSession = smurff.BPMFSession(Y, is_scarce = True, Ytest=Ytest, - num_latent=4, verbose=verbose, burnin=20, nsamples=20, save_freq=1, + num_latent=4, verbose=verbose, num_threads = 1, burnin=20, nsamples=20, save_freq=1, seed = seed, save_name=smurff.helper.temp_savename()) trainSession.run() predict_session = trainSession.makePredictSession() diff --git a/python/test/test_predict.py b/python/test/test_predict.py index 6560a716..4c94ba25 100644 --- a/python/test/test_predict.py +++ b/python/test/test_predict.py @@ -16,13 +16,13 @@ class TestPredictSession(unittest.TestCase): __name__ = "TestPredictSession" def run_train_session(self, nmodes, density): - shape = range(5, nmodes+5) # 5, 6, 7, ... + shape = range(5, nmodes+5) # 5, 6, 7, ... Y, X = smurff.generate.gen_tensor(shape, 3, density) self.Ytrain, self.Ytest = smurff.make_train_test(Y, 0.1) priors = ['normal'] * nmodes trainSession = smurff.TrainSession(priors = priors, num_latent=4, - burnin=10, nsamples=nsamples, verbose=verbose, + burnin=10, nsamples=nsamples, verbose=verbose, num_threads=1, save_freq = 1, save_name = smurff.helper.temp_savename()) trainSession.addTrainAndTest(self.Ytrain, self.Ytest) @@ -47,7 +47,7 @@ def assert_almost_equal_sparse(self, A, B): c1,v1 = smurff.find(A) c2,v2 = smurff.find(B) assert np.array_equal(c1,c2) - assert np.allclose(v1,v2, atol=0.01) + assert np.allclose(v1,v2, atol=0.01) def run_predict_some_all_one(self, train_session, predict_session): coords, _ = smurff.find(self.Ytest) @@ -83,7 +83,7 @@ def run_predict_predict(self, predict_session, X): """ Test the PredictSession.predict function """ def run_n_samples(samples, expected_nsamples): - operand_and_sizes = [ + operand_and_sizes = [ [ ( Ellipsis , x.shape[0] ), ( slice(3) , 3 ), diff --git a/python/test/test_pybind.py b/python/test/test_pybind.py index 1fbdd85d..52fc6398 100644 --- a/python/test/test_pybind.py +++ b/python/test/test_pybind.py @@ -4,7 +4,14 @@ import scipy.sparse as sp def test_pybind(): - trainSession = smurff.TrainSession(priors = ["normal", "normal"], verbose = 2 ) + trainSession = smurff.TrainSession( + priors = ["normal", "normal"], + burnin = 10, + nsamples = 10, + num_latent = 4, + verbose = 2, + num_threads = 1, + ) Y = np.array([[1.,2.],[3.,4.]]) trainSession.setTrain(Y) diff --git a/python/test/test_scarce.py b/python/test/test_scarce.py index 3cc4c0cf..0922e1b8 100644 --- a/python/test/test_scarce.py +++ b/python/test/test_scarce.py @@ -22,7 +22,7 @@ class TestScarce(unittest.TestCase): def test_simple(self): matrix = matrix_with_explicit_zeros() self.assertTrue(matrix.nnz == 6) - + matrix.eliminate_zeros() self.assertTrue(matrix.nnz == 3) @@ -30,8 +30,8 @@ def test_smurff(self): matrix = matrix_with_explicit_zeros() self.assertTrue(matrix.nnz == 6) - predictions = smurff.bpmf(matrix, Ytest=matrix, num_latent=4, burnin=5, nsamples=5) + predictions = smurff.bpmf(matrix, Ytest=matrix, num_latent=4, burnin=5, nsamples=5, num_threads=1) self.assertEqual(len(predictions), 6) - + if __name__ == '__main__': unittest.main() diff --git a/python/test/test_smurff.py b/python/test/test_smurff.py index 9272dc45..9d86db49 100644 --- a/python/test/test_smurff.py +++ b/python/test/test_smurff.py @@ -6,7 +6,7 @@ import itertools import collections -verbose = 0 +verbose = 1 class TestSmurff(unittest.TestCase): @@ -21,6 +21,7 @@ def test_bpmf(self): priors=['normal', 'normal'], num_latent=4, verbose=verbose, + num_threads=1, burnin=50, nsamples=50) self.assertEqual(Ytest.nnz, len(predictions)) @@ -35,7 +36,9 @@ def test_bpmf_numerictest(self): num_latent=10, burnin=10, nsamples=15, - verbose=verbose) + verbose=verbose, + num_threads=1, + ) def test_macau(self): Ydense = np.random.rand(10, 20) @@ -54,6 +57,7 @@ def test_macau(self): # side_info_noises=[[('fixed', 1.0, None, None, None)], [('adaptive', None, 0.5, 1.0, None)]], num_latent=4, verbose=verbose, + num_threads=1, burnin=50, nsamples=50) #self.assertEqual(Ytest.nnz, len(predictions)) @@ -71,7 +75,8 @@ def test_macau_side_bin(self): num_latent=5, burnin=10, nsamples=5, - verbose=verbose) + verbose=verbose, + num_threads=1) def test_macau_dense(self): Y = scipy.sparse.rand(15, 10, 0.2) @@ -85,7 +90,8 @@ def test_macau_dense(self): num_latent=5, burnin=10, nsamples=5, - verbose=verbose) + verbose=verbose, + num_threads = 1) def test_macau_dense_probit(self): A = np.random.randn(25, 2) @@ -97,13 +103,15 @@ def test_macau_dense_probit(self): Ytrain, Ytest = smurff.make_train_test(df, 0.2) threshold = 0.5 # since we sample from mu(0,1) - + trainSession = smurff.TrainSession(priors=['macau', 'normal'], num_latent=4, threshold=threshold, burnin=200, nsamples=200, - verbose=False) + verbose=0, + num_threads=1, + ) trainSession.addTrainAndTest(Ytrain, Ytest, smurff.ProbitNoise(threshold)) trainSession.addSideInfo(0, A, direct=True) @@ -127,6 +135,7 @@ def test_macau_univariate(self): direct=True, num_latent=4, verbose=verbose, + num_threads=1, burnin=50, nsamples=50) self.assertEqual(Ytest.nnz, len(predictions)) @@ -137,7 +146,9 @@ def test_too_many_sides(self): smurff.smurff(Y, priors=['normal', 'normal', 'normal'], side_info=[None, None, None], - verbose = False) + verbose = 0, + num_threads=1, + ) def test_bpmf_emptytest(self): X = scipy.sparse.rand(15, 10, 0.2) @@ -146,7 +157,9 @@ def test_bpmf_emptytest(self): num_latent=10, burnin=10, nsamples=15, - verbose=verbose) + verbose=verbose, + num_threads=1, + ) def test_bpmf_emptytest_probit(self): X = scipy.sparse.rand(15, 10, 0.2) @@ -156,7 +169,9 @@ def test_bpmf_emptytest_probit(self): num_latent=10, burnin=10, nsamples=15, - verbose=verbose) + verbose=verbose, + num_threads=1 + ) def test_make_train_test(self): X = scipy.sparse.rand(15, 10, 0.2) @@ -206,6 +221,7 @@ def test_bpmf_tensor(self): priors=['normal', 'normal', 'normal'], num_latent=4, verbose=verbose, + num_threads=1, burnin=50, nsamples=50) @@ -224,6 +240,7 @@ def test_bpmf_tensor2(self): priors=['normal', 'normal', 'normal'], num_latent=4, verbose=verbose, + num_threads=1, burnin=20, nsamples=20) @@ -247,6 +264,7 @@ def test_bpmf_tensor3(self): priors=['normal', 'normal', 'normal'], num_latent=4, verbose=verbose, + num_threads=1, burnin=20, nsamples=20) @@ -271,7 +289,9 @@ def test_macau_tensor_empty(self): num_latent=2, burnin=5, nsamples=5, - verbose=verbose) + verbose=verbose, + num_threads=1, + ) self.assertFalse(predictions)