Auto docker image building on release (#494)

* + Add workflows to publish docker images and pypi packages automatically when a release is published

* * update setuptools before installing ram to avoid ParseException

* - remove comments in the requirement files when installing them

* * build docker image with a newer version of python

* * try to build on self-hosted runner

* * try to build on self-hosted runner

* * try to build on self-hosted runner

* * try to build on self-hosted runner

* * try to build on self-hosted runner

* * try to build on self-hosted runner

* * try to build on self-hosted runner

* * build docker on a new runner

* * fix wrong build path

* * build from cuda

* * install python first

* * install torch first

* * install torch first

* * use cuda 12.4 base image

* - remove cache for built images

.github/workflows/publish-docker.yml

@@ -12,11 +12,12 @@ on:
 
 env:
   IMAGE_NAME: datajuicer/data-juicer
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: [docker]
     permissions:
       contents: read
       packages: write
@@ -27,7 +28,9 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
+        with:
+          path: dj-${{ github.run_id }}
 
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer
@@ -40,12 +43,12 @@ jobs:
       # multi-platform images and export cache
       # https://github.com/docker/setup-buildx-action
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -64,12 +67,10 @@ jobs:
         id: build-and-push
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: dj-${{ github.run_id }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
 
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker

.github/workflows/unit-test.yml

@@ -18,7 +18,7 @@ env:
 
 jobs:
   unittest-single:
-    runs-on: [self-hosted, linux]
+    runs-on: [self-hosted, linux, unittest]
     environment: Testing
     steps:
     - uses: actions/checkout@v3

Dockerfile

@@ -1,32 +1,43 @@
 # The data-juicer image includes all open-source contents of data-juicer,
 # and it will be instaled in editable mode.
 
-FROM python:3.8.18
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+
+# install python 3.10
+RUN apt-get update \
+    && apt-get install -y git curl vim wget python3.10 libpython3.10-dev python3-pip \
+    && apt-get install -y libgl1-mesa-glx libglib2.0-0 \
+    && ln -sf /usr/bin/python3.10  /usr/bin/python3 \
+    && ln -sf /usr/bin/python3.10  /usr/bin/python \
+    && apt-get autoclean && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade pip
+
+# install 3rd-party system dependencies
+RUN apt-get update \
+    && apt-get install ffmpeg libsm6 libxext6 software-properties-common build-essential cmake  -y
 
 # prepare the java env
 WORKDIR /opt
 # download jdk
-RUN wget https://aka.ms/download-jdk/microsoft-jdk-17.0.9-linux-x64.tar.gz -O jdk.tar.gz && \
-    tar -xzf jdk.tar.gz && \
-    rm -rf jdk.tar.gz && \
-    mv jdk-17.0.9+8 jdk
+RUN wget https://aka.ms/download-jdk/microsoft-jdk-17.0.9-linux-x64.tar.gz -O jdk.tar.gz \
+    && tar -xzf jdk.tar.gz \
+    && rm -rf jdk.tar.gz \
+    && mv jdk-17.0.9+8 jdk
 
 # set the environment variable
 ENV JAVA_HOME=/opt/jdk
 
 WORKDIR /data-juicer
 
 # install requirements which need to be installed from source
-RUN pip install git+https://github.com/xinyu1205/recognize-anything.git --default-timeout 1000
+RUN pip install --upgrade setuptools==69.5.1 setuptools_scm \
+    && pip install git+https://github.com/xinyu1205/recognize-anything.git --default-timeout 1000
 
 # install requirements first to better reuse installed library cache
 COPY environments/ environments/
-RUN cat environments/* | xargs pip install --default-timeout 1000
+RUN cat environments/* | grep -v '^#' | xargs pip install --default-timeout 1000
 
 # install data-juicer then
 COPY . .
 RUN pip install -v -e .[all]
 RUN pip install -v -e .[sandbox]
-
-# install 3rd-party system dependencies
-RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y