nvidia base upgrade to 12.4.1

.github/workflows/docker-publish.yaml

@@ -3,7 +3,7 @@ name: Build and upload Docker
 on:
   push:
      branches:
-      - main
+      - nvidia-base-upgrade12-4-1
      tags:
       - v*
   schedule:
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v2
         if: github.event_name == 'schedule'
         with:
-          ref: main
+          ref: nvidia-base-upgrade12-4-1
 
       - uses: actions/checkout@v2
         if: github.event_name != 'schedule'

devops/Dockerfile

@@ -13,7 +13,6 @@ RUN set -ex \
  && git config --global user.name "Wild Me" \
  && find /wbia/wbia* -name '.git' -type d -print0 | xargs -0 -i /bin/bash -c \
     'cd {} && cd .. && echo $(pwd) && git reset --hard origin/main && git pull' \
-
  && cd /wbia/wbia-plugin-kaggle7/wbia_kaggle7 \
  && git reset --hard origin/main \
  && git pull \
@@ -131,9 +130,10 @@ RUN set -ex \
      opencv-python-headless \
      opencv-contrib-python-headless \
  && /virtualenv/env3/bin/pip install --no-cache-dir --upgrade \
-     opencv-contrib-python-headless \
- && /virtualenv/env3/bin/pip uninstall -y \
-     nvidia_cublas_cu11
+     opencv-contrib-python-headless 
+ #\
+ #&& /virtualenv/env3/bin/pip uninstall -y \
+ #    nvidia_cublas_cu11
 
 # Run smoke tests
 # && /virtualenv/env3/bin/python -c "import wbia;             from wbia.__main__ import smoke_test; smoke_test()" \

devops/_config/deprecated.sh

@@ -17,8 +17,8 @@ git clone https://github.com/WildMeOrg/wbia-plugin-pie.git
  && /bin/bash run_developer_setup.sh'
 
 /virtualenv/env3/bin/pip install --no-cache-dir \
-    'tensorflow-gpu==1.15.5' \
-    'keras==2.2.5'
+    'tensorflow' \
+    'keras'
 
 /virtualenv/env3/bin/pip install --no-cache-dir --upgrade \
     numpy

devops/base/Dockerfile

@@ -1,4 +1,10 @@
-ARG WBIA_UBUNTU_IMAGE=nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04
+ARG WBIA_UBUNTU_IMAGE=nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
+#nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+#nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
+#nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04
+#docker pull nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04
+#nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+#nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04
 
 FROM ${WBIA_UBUNTU_IMAGE} as org.wildme.wbia.base
 
@@ -22,7 +28,7 @@ RUN set -ex \
 
 # Install package updates and dependencies
 RUN set -ex \
- && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub \
+ && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         software-properties-common \
@@ -36,11 +42,11 @@ RUN set -ex \
         build-essential \
         lsb-release \
         pkg-config \
-        # Install Python 3.7
-        python3.7 \
-        python3.7-dev \
-        python3.7-gdbm \
-        python3.7-distutils \
+        # Install Python 3.8
+        python3.8 \
+        python3.8-dev \
+        python3.8-gdbm \
+        python3.8-distutils \
         python3-pip \
         python3-setuptools \
         python3-venv \
@@ -117,18 +123,18 @@ RUN set -ex \
 
 # Create virtualenvs for Python3
 RUN set -ex \
- && virtualenv --system-site-packages -p $(which python3.7) /virtualenv/env3
+ && virtualenv --system-site-packages -p $(which python3.8) /virtualenv/env3
 
 # Install PyPI packages
 RUN set -ex \
  && /virtualenv/env3/bin/pip install --upgrade \
         pip \
- && /virtualenv/env3/bin/pip install setuptools==59.5.0 \
+ && /virtualenv/env3/bin/pip install setuptools \
  && /virtualenv/env3/bin/pip install  --upgrade  \
         # Install build requirements
         cmake \
         ninja \
-        setuptools==59.5.0 \
+        setuptools \
         scikit-build \    
         setuptools_scm[toml] \
         cython \
@@ -146,12 +152,12 @@ RUN set -ex \
     && wait-for google.com:80 -- echo "success"
 
 # Set CUDA-specific environment paths
-ENV PATH "/usr/local/cuda/bin:${PATH}"
-
-ENV LD_LIBRARY_PATH "/usr/local/cuda/lib64:/virtualenv/env3/lib:${LD_LIBRARY_PATH}"
+ENV PATH "/usr/local/nvidia/bin:${PATH}"
 
+ENV LD_LIBRARY_PATH "/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/virtualenv/env3/lib:${LD_LIBRARY_PATH}"
+
 ENV CUDA_HOME "/usr/local/cuda"
 
-ENV CUDA_TOOLKIT_ROOT_DIR "/usr/local/cuda"
+ENV CUDA_TOOLKIT_ROOT_DIR "/usr/local/nvidia/cuda"
 
 CMD ["/bin/bash", "-c", "Xvfb :1 -screen 0 1024x768x16 &>/tmp/xvfb.log & /bin/bash"]

devops/develop/Dockerfile

@@ -11,11 +11,13 @@ RUN /bin/bash -xc '. /virtualenv/env3/bin/activate \
  && pip install --upgrade pip setuptools-scm \
  && pip install --no-cache-dir -r /wbia/wildbook-ia/requirements/tests.txt \
  && pip install --no-cache-dir -r /wbia/wildbook-ia/requirements/runtime.txt \
+ && pip install --no-cache-dir -r /wbia/wildbook-ia/requirements/torch.txt \
+ && pip install --no-cache-dir -r /wbia/wildbook-ia/requirements/runtime-post.txt \
  && pip install --no-cache-dir -r /wbia/wildbook-ia/requirements/postgres.txt \
  && pip install --no-cache-dir -e .'
 
 # Temporary fix for ARM64
-ENV LD_PRELOAD "/virtualenv/env3/lib/python3.7/site-packages/torch/lib/libgomp-d22c30c5.so.1"
+ENV LD_PRELOAD "/virtualenv/env3/lib/python3.8/site-packages/torch/lib/libgomp-d22c30c5.so.1"
 
 RUN set -ex \
 

devops/install.ubuntu.sh

@@ -69,7 +69,7 @@ export OPENCV_VERSION=3.4.11
 
 pip3 install --no-cache-dir virtualenv
 
-virtualenv -p $(which python3.7) ${VENV}
+virtualenv -p $(which python3.8) ${VENV}
 
 source ${VENV}/bin/activate
 
@@ -78,8 +78,8 @@ pip install --no-cache-dir --upgrade \
     'cmake!=3.18.2' \
     ninja \
     scikit-build \
-    'setuptools==59.5.0' \
-    'setuptools_scm[toml]>=3.4' \
+    'setuptools' \
+    'setuptools_scm[toml]' \
     cython==3.0.0 \
     numpy \
     ipython
@@ -178,7 +178,7 @@ pip install --no-cache-dir -e .
 
 pip install --no-cache-dir pygraphviz --install-option="--include-path=/usr/include/graphviz" --install-option="--library-path=/usr/lib/graphviz/"
 
-cp -r ${VIRTUAL_ENV}/lib/python3.7/site-packages/cv2 /tmp/cv2
+cp -r ${VIRTUAL_ENV}/lib/python3.8/site-packages/cv2 /tmp/cv2
 
 cd ${CODE}
 git clone --branch develop https://github.com/WildMeOrg/wildbook-ia.git
@@ -307,8 +307,8 @@ pip install --no-cache-dir \
     tensorflow-gpu==1.15.4 \
     keras==2.2.5
 
-rm -rf ${VIRTUAL_ENV}/lib/python3.7/site-packages/cv2*
-cp -r /tmp/cv2 ${VIRTUAL_ENV}/lib/python3.7/site-packages/cv2
+rm -rf ${VIRTUAL_ENV}/lib/python3.8/site-packages/cv2*
+cp -r /tmp/cv2 ${VIRTUAL_ENV}/lib/python3.8/site-packages/cv2
 rm -rf /tmp/cv2
 
 # python -c "import wbia;            from wbia.__main__ import smoke_test; smoke_test()"

pyproject.toml

@@ -1,2 +1,2 @@
 [build-system]
-requires = ["setuptools==59.5.0", "wheel", "setuptools_scm[toml]==6.3.2", "scikit-build", "cmake", "ninja"]
+requires = ["setuptools", "wheel", "setuptools_scm[toml]", "scikit-build", "cmake", "ninja"]

requirements.txt

@@ -1,4 +1,5 @@
 -r requirements/build.txt
 -r requirements/runtime.txt
+-r requirements/torch.txt
 -r requirements/pinned.txt
 -r requirements/tests.txt

requirements/runtime-post.txt

@@ -0,0 +1,13 @@
+tornado==6.2
+tqdm==4.64.0
+ubelt==1.2.1
+visdom==0.1.8.9
+PyQt5==5.15.9
+wbia-lightnet
+wbia-pydarknet
+wbia-pyflann
+wbia-utool @ git+https://github.com/WildMeOrg/wbia-utool@main
+wbia-vtool
+wbia-pyhesaff
+# webargs==5.5.3
+# werkzeug==0.15.6

requirements/runtime.txt

@@ -32,12 +32,12 @@ grad-cam==1.4.6
 imgaug==0.4.0
 ipython==7.34.0
 jupyter==1.0.0
-keras==2.2.5
+keras
 line-profiler==3.5.1
 lockfile==0.12.2
 mako==1.2.2
 itsdangerous>=0.24,<2.0
-markupsafe<2.0.0
+markupsafe==2.0.1
 # marshmallow==2.21.0
 # marshmallow-sqlalchemy==0.12.1
 matplotlib==3.5.3
@@ -79,19 +79,4 @@ SQLAlchemy-Utils==0.38.3
 tensorboard_logger==0.1.0
 Theano==1.0.5
 timm==0.6.12
-torch==1.13.1
-torchvision==0.14.1
-torchaudio==0.13.1
-tornado==6.2
-tqdm==4.64.0
-ubelt==1.2.1
-visdom==0.1.8.9
-PyQt5==5.15.9
-wbia-lightnet
-wbia-pydarknet
-wbia-pyflann
-wbia-utool @ git+https://github.com/WildMeOrg/wbia-utool@main
-wbia-vtool
-wbia-pyhesaff
-# webargs==5.5.3
-# werkzeug==0.15.6
+

requirements/torch.txt

@@ -0,0 +1,4 @@
+--index-url https://download.pytorch.org/whl/cu121
+torch==2.1.0
+torchvision==0.16.0
+torchaudio==2.1.0

setup.py

@@ -257,7 +257,7 @@ def gen_packages_items():
         'Environment :: Console',
         'Environment :: Web Environment',
         'Environment :: GPU',
-        'Environment :: GPU :: NVIDIA CUDA :: 11.0',
+        'Environment :: GPU :: NVIDIA CUDA :: 12.2',
         'Natural Language :: English',
         'License :: OSI Approved :: Apache Software License',
         'Intended Audience :: Developers',