Pytest hanging fix (#119)

* Updated actions versions

* Update changelog

* switch back to concurrent.futures due to pytest

.github/workflows/build.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Generate env yaml
         run: |
@@ -37,7 +37,7 @@ jobs:
 
       - name: Load cache
         id: cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             /usr/share/miniconda3

.github/workflows/cache.yml

@@ -30,10 +30,10 @@ jobs:
     # note absence of cucurrency, this one should only be run one at a time
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Miniforge
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
             python-version: ${{ matrix.python-version }}
             miniforge-variant: Miniforge3
@@ -54,7 +54,7 @@ jobs:
           PYTHONPYCACHEPREFIX=~/pycache mamba run -n fiat_test python -m pytest --verbose --cov --cov-report xml
 
       - name: Upload cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: |
@@ -82,10 +82,10 @@ jobs:
 
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Miniforge3
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
             python-version: ${{ matrix.python-version }}
             miniforge-variant: Miniforge3
@@ -104,7 +104,7 @@ jobs:
     #     run: PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt sphinx-build -M html ./docs ./docs/_build
 
       - name: Upload cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: |
@@ -132,10 +132,10 @@ jobs:
 
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Miniforge3
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
             python-version: ${{ matrix.python-version }}
             miniforge-variant: Miniforge3
@@ -149,7 +149,7 @@ jobs:
           mamba env create -f environment.yml
 
       - name: Upload cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: |
@@ -177,10 +177,10 @@ jobs:
 
     steps:
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Miniforge3
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
             python-version: ${{ matrix.python-version }}
             miniforge-variant: Miniforge3
@@ -194,7 +194,7 @@ jobs:
           mamba env create -f environment.yml
 
       - name: Upload cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: |

.github/workflows/docs.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Generate env yaml
         run: |
@@ -37,7 +37,7 @@ jobs:
 
       - name: Load cache
         id: cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             /usr/share/miniconda3

.github/workflows/linting.yml

@@ -10,6 +10,6 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v3
       - uses: pre-commit/[email protected]

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -61,7 +61,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/download-artifact@v3
         with:
@@ -78,7 +78,7 @@ jobs:
 
       - name: load from cache
         id: cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             /usr/share/miniconda3

.github/workflows/purge_cache.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Cleanup caches
         run: |

.github/workflows/sonar.yml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/test.yml

@@ -30,14 +30,14 @@ jobs:
 
     steps:
       - name: Check-out source
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Generate env yaml
         run: pip install tomli && python make_env.py test -p ${{ matrix.python-version }}.*
 
       - name: load from cache
         id: cache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             /usr/share/miniconda3

docs/changelog.qmd

@@ -19,7 +19,6 @@ This contains the unreleased changes to Delft-FIAT.
 ### Changed
 - Better version of `BufferHandler`
 - Made read methods of `BaseModel`, `GeomModel` and `GridModel` public (removed underscore)
-- Parallelization now depends solely on `multiprocessing` instead of on `concurrent.futures`
 - Testing of workers (not properly caught due to using `multiprocessing`)
 - Testing only based on integers
 

src/fiat/models/util.py

@@ -1,5 +1,6 @@
 """The FIAT model workers."""
 
+from concurrent.futures import ProcessPoolExecutor, wait
 from itertools import product
 from multiprocessing.context import SpawnContext
 from pathlib import Path
@@ -106,19 +107,20 @@ def execute_pool(
     # If there are more threads needed however
     processes = []
     # Setup the multiprocessing pool
-    pool = ctx.Pool(processes=threads)
+    pool = ProcessPoolExecutor(
+        max_workers=threads,
+        mp_context=ctx,
+    )
 
     # Go through all the jobs
     for job in jobs:
-        pr = pool.apply_async(
-            func=func,
-            kwds=job,
+        pr = pool.submit(
+            func,
+            **job,
         )
         processes.append(pr)
 
     # wait for all jobs to conclude
-    for pr in processes:
-        pr.get()
+    wait(processes)
 
-    pool.close()
-    pool.join()
+    pool.shutdown(wait=False)