Merge branch 'main' into py310-collections-sequence

NVIDIA-Merlin · Jul 1, 2023 · 42e0c35 · 42e0c35
2 parents b87b99d + b89f66e
commit 42e0c35
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 35 deletions.
diff --git a/.github/workflows/cpu-horovod.yml b/.github/workflows/cpu-horovod.yml
@@ -72,4 +72,4 @@ jobs:
           if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
               extra_pytest_markers="and changed"
           fi
-          EXTRA_PYTEST_MARKERS="$extra_pytest_markers" MERLIN_BRANCH="$merlin_branch" COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-horovod-cpu
+          EXTRA_PYTEST_MARKERS="$extra_pytest_markers" MERLIN_BRANCH="$merlin_branch" COMPARE_BRANCH=${{ github.base_ref }} tox -e horovod-cpu
diff --git a/.github/workflows/cpu-nvtabular.yml b/.github/workflows/cpu-nvtabular.yml
@@ -64,4 +64,4 @@ jobs:
       - name: Run tests
         run: |
           merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
-          MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e py38-nvtabular-cpu
+          MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e nvtabular-cpu
diff --git a/.github/workflows/cpu-systems.yml b/.github/workflows/cpu-systems.yml
@@ -64,4 +64,4 @@ jobs:
       - name: Run tests
         run: |
           merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
-          MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e py38-systems-cpu
+          MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e systems-cpu
diff --git a/.github/workflows/cpu-t4r.yml b/.github/workflows/cpu-t4r.yml
@@ -60,4 +60,4 @@ jobs:
       - name: Run tests
         run: |
           merlin_branch="${{ steps.get-branch-name.outputs.branch }}"
-          MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e py38-transformers4rec-cpu
+          MERLIN_BRANCH="$merlin_branch" GIT_COMMIT=$(git rev-parse HEAD) tox -e transformers4rec-cpu
diff --git a/.github/workflows/gpu-multi.yml b/.github/workflows/gpu-multi.yml
@@ -56,4 +56,4 @@ jobs:
           if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
               extra_pytest_markers="and changed"
           fi
-          cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py38-multi-gpu
+          cd ${{ github.workspace }}; EXTRA_PYTEST_MARKERS=$extra_pytest_markers MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e multi-gpu
diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml
@@ -34,7 +34,7 @@ jobs:
           if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
               extra_pytest_markers="and changed"
           fi
-          cd ${{ github.workspace }}; PYTEST_MARKERS="unit and not (examples or integration or notebook) $extra_pytest_markers" MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py310-gpu
+          cd ${{ github.workspace }}; PYTEST_MARKERS="unit and not (examples or integration or notebook) $extra_pytest_markers" MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e gpu
 
   tests-examples:
     runs-on: 1GPU
@@ -55,4 +55,4 @@ jobs:
           if [[ "${{ github.ref }}" != 'refs/heads/main' ]]; then
               extra_pytest_markers="and changed"
           fi
-          cd ${{ github.workspace }}; PYTEST_MARKERS="(examples or notebook) $extra_pytest_markers" MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e py310-gpu
+          cd ${{ github.workspace }}; PYTEST_MARKERS="(examples or notebook) $extra_pytest_markers" MERLIN_BRANCH=$branch COMPARE_BRANCH=${{ github.base_ref }} tox -e gpu
diff --git a/README.md b/README.md
@@ -111,7 +111,7 @@ You can find more details and information about a low-level API in our overview
 
 ### Notebook Examples and Tutorials
 
-View the example notebooks in the [documentation](https://nvidia-merlin.github.io/models/stable/examples/README.html) to help you become familiar with Merlin Models.
+View the example notebooks in the [documentation](https://nvidia-merlin.github.io/models/stable/examples/) to help you become familiar with Merlin Models.
 
 The same notebooks are available in the `examples` directory from the [Merlin Models](https://github.com/NVIDIA-Merlin/models) GitHub repository.
 

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -20,13 +20,15 @@
 import platform
 import warnings
 from pathlib import Path
+from unittest.mock import patch
 
 import distributed
 import psutil
 import pytest
 from asvdb import BenchmarkInfo, utils
 
 from merlin.core.utils import Distributed
+from merlin.dataloader.loader_base import LoaderBase
 from merlin.datasets.synthetic import generate_data
 from merlin.io import Dataset
 from merlin.models.utils import ci_utils
@@ -145,3 +147,17 @@ def get_benchmark_info():
         arch=uname.machine,
         ram="%d" % psutil.virtual_memory().total,
     )
+
+
+@pytest.fixture(scope="function", autouse=True)
+def cleanup_dataloader():
+    """After each test runs. Call .stop() on any dataloaders created during the test.
+    The avoids issues with background threads hanging around and interfering with subsequent tests.
+    This happens when a dataloader is partially consumed (not all batches are iterated through).
+    """
+    with patch.object(
+        LoaderBase, "__iter__", side_effect=LoaderBase.__iter__, autospec=True
+    ) as patched:
+        yield
+        for call in patched.call_args_list:
+            call.args[0].stop()
diff --git a/tox.ini b/tox.ini
@@ -2,33 +2,34 @@
 ; .github/workflows/cpu-ci.yml for the workflow definition.
 
 [tox]
-envlist = py310-gpu,py310-multi-gpu
+envlist = gpu,multi-gpu,horovod-cpu,nvtabular-cpu,systems-cpu,transformers4rec-cpu,docs,docs-multi
 
 [testenv]
 commands =
     pip install --upgrade pip
     pip install -e .[all]
 
-[testenv:py310-gpu]
+[testenv:gpu]
 ; Runs in: Github Actions
 ; Runs GPU-based tests.
 allowlist_externals =
     bash
 deps =
-    --no-deps -rrequirements/test.txt
+    -rrequirements/test.txt
+    git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH}
 passenv =
     OPAL_PREFIX
 setenv =
     TF_GPU_ALLOCATOR=cuda_malloc_async
 sitepackages=true
 commands =
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{env:MERLIN_BRANCH:main}
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH:main}
-    bash -c 'python -m pytest --cov-report term --cov merlin -m "{env:PYTEST_MARKERS}" -rxs tests/ || ([ $? = 5 ] && exit 0 || exit $?)'
+    bash -c 'python -m pytest --cov-report term --cov merlin -m "{env:PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
+
 
-[testenv:py310-multi-gpu]
+[testenv:multi-gpu]
 ; Runs in: Github Actions
 ; Runs GPU-based tests.
 allowlist_externals =
@@ -43,30 +44,32 @@ setenv =
     LD_LIBRARY_PATH=${envdir}/hugectr/include/lib{:}/usr/local/lib/python3.10/dist-packages/tensorflow{:}{env:LD_LIBRARY_PATH}
     LIBRARY_PATH=${envdir}/hugectr/lib{:}{env:LIBRARY_PATH}
 sitepackages=true
+deps =
+    git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
 commands =
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{env:MERLIN_BRANCH:main}
     sh examples/usecases/multi-gpu/install_sparse_operation_kit.sh {envdir}
-    bash -c 'horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh python -m pytest -m "horovod {env:EXTRA_PYTEST_MARKERS}" -rxs tests/unit || ([ $? = 5 ] && exit 0 || exit $?)'
+    bash -c 'horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh python -m pytest -m "unit and horovod {env:EXTRA_PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
 
-[testenv:py310-horovod-cpu]
+[testenv:horovod-cpu]
 setenv =
     HOROVOD_WITH_MPI=1
     HOROVOD_WITH_TENSORFLOW=1
     PATH={env:PATH}{:}{envdir}/env/bin
     LD_LIBRARY_PATH={env:LD_LIBRARY_PATH}{:}{envdir}/env/lib
+deps =
+    git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
+    git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
 commands =
     conda update --yes --name base --channel defaults conda
     conda env create --prefix {envdir}/env --file requirements/horovod-cpu-environment.yml --force
     {envdir}/env/bin/python -m pip install 'horovod==0.27.0' --no-cache-dir
     {envdir}/env/bin/horovodrun --check-build
-    {envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
-    {envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
-    {envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{env:MERLIN_BRANCH:main}
-    {envdir}/env/bin/horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh pytest -m "horovod {env:EXTRA_PYTEST_MARKERS}" -rxs tests/unit
+    {envdir}/env/bin/horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh pytest -m "unit and horovod {env:EXTRA_PYTEST_MARKERS}" -rxs {posargs:tests}
 
-[testenv:py310-nvtabular-cpu]
+[testenv:nvtabular-cpu]
 passenv=GIT_COMMIT
 allowlist_externals = git
 deps =
@@ -82,7 +85,7 @@ commands =
     python -m pip install .
     python -m pytest nvtabular-{env:GIT_COMMIT}/tests/unit
 
-[testenv:py310-systems-cpu]
+[testenv:systems-cpu]
 passenv=GIT_COMMIT
 allowlist_externals = git
 deps =
@@ -99,7 +102,7 @@ commands =
     python -m pip install .
     python -m pytest -m "not notebook" systems-{env:GIT_COMMIT}/tests/unit
 
-[testenv:py310-transformers4rec-cpu]
+[testenv:transformers4rec-cpu]
 passenv=GIT_COMMIT
 allowlist_externals = git
 commands =
@@ -120,10 +123,10 @@ changedir = {toxinidir}
 deps =
     -rrequirements/docs.txt
     -rrequirements/test.txt
+    git+https://github.com/NVIDIA-Merlin/core.git
+    git+https://github.com/NVIDIA-Merlin/dataloader.git
+    git+https://github.com/NVIDIA-Merlin/NVTabular.git
 commands =
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git
     python -m sphinx.cmd.build -E -P -b html docs/source docs/build/html
 
 [testenv:docs-multi]
@@ -132,9 +135,9 @@ changedir = {toxinidir}
 deps =
     -rrequirements/docs.txt
     -rrequirements/test.txt
+    git+https://github.com/NVIDIA-Merlin/core.git
+    git+https://github.com/NVIDIA-Merlin/dataloader.git
+    git+https://github.com/NVIDIA-Merlin/NVTabular.git
 commands =
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git
-    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git
     sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
     sphinx-multiversion docs/source docs/build/html