CICD / Docker image (#13)

* New CICD workflow

.github/workflows/CI-build-test.yml

@@ -0,0 +1,27 @@
+name: Vision Build and Test
+
+on:
+  workflow_call:
+
+jobs:
+  build-and-test:
+    name: Build/Test 
+    runs-on: ubuntu-20.04
+    steps:    
+    - uses: actions/checkout@v4
+
+    - name: Use Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11.6'
+        cache: 'pip'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements-test.txt
+
+    - name: Test with pytest
+      run: |
+        coverage run -m pytest -s && coverage report --show-missing

.github/workflows/CI-pre-build.yml

@@ -0,0 +1,14 @@
+name: Pre Build
+
+on:
+  workflow_call:
+
+jobs:
+  pre-build:
+    name: Pre-build
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: fkirc/skip-duplicate-actions@master
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        cancel_others: true

.github/workflows/CICD-dev.yml

@@ -7,39 +7,53 @@ on:
       - '!main'
 
 jobs:
+  pre-build:
+    uses: ./.github/workflows/CI-pre-build.yml
+
   build-and-test:
-    name: Build/Test
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v4
+    needs: pre-build
+    uses: ./.github/workflows/CI-build-test.yml
 
   build-and-push-dev-docker-image:
-    name: Build/Push Dev Docker Image
+    name: Build/Push Dev Docker Image 
+    needs: pre-build
     runs-on: ubuntu-20.04
-    steps:
+    steps:    
     - uses: actions/checkout@v4
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
+        password: ${{ secrets.DOCKERHUB_TOKEN }}  
+    
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-
-    - name: Build/Push dev vision-api
+    
+    - name: Build/Push Dev inatvisionapi
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.ref_name }}
+        tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.ref_name }}
 
-    - name: Build/Push dev vision-cleanup
+    - name: Build/Push Dev inatvisionapi-cleanup
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile-cleanup
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.ref_name }}
+        tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.ref_name }}
+
+  notify-slack:
+    name: Notify Slack
+    needs: build-and-push-dev-docker-image
+    if: ${{ success() || failure() }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: iRoachie/[email protected]
+      if: env.SLACK_WEBHOOK_URL != null
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/CICD-main.yml

@@ -6,39 +6,57 @@ on:
       - main
 
 jobs:
+  pre-build:
+    uses: ./.github/workflows/CI-pre-build.yml
+
   build-and-test:
-    name: Build/Test
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v4
+    needs: pre-build
+    uses: ./.github/workflows/CI-build-test.yml
 
-  build-and-push-staging-docker-image:
-    name: Build/Push Staging Docker Image
+  build-and-push-main-docker-image:
+    name: Build/Push Main Docker Image 
+    needs: pre-build
     runs-on: ubuntu-20.04
-    steps:
+    steps:    
     - uses: actions/checkout@v4
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-
+        password: ${{ secrets.DOCKERHUB_TOKEN }}  
+    
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-
-    - name: Build/Push staging vision-api
+    
+    - name: Build/Push Main inatvisionapi
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.sha }}
+        tags: |
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.sha }}
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:latest
 
-    - name: Build/Push staging vision-cleanup
+    - name: Build/Push Main inatvisionapi-cleanup
       uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile-cleanup
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.sha }}
+        tags: |
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.sha }}
+          ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:latest
+
+  notify-slack:
+    name: Notify Slack
+    needs: build-and-push-main-docker-image
+    if: ${{ success() || failure() }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: iRoachie/[email protected]
+      if: env.SLACK_WEBHOOK_URL != null
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/CICD-pr.yml

@@ -0,0 +1,20 @@
+name: Vision CI/CD Pull Request
+
+on:
+  pull_request:
+
+jobs:
+  build-and-test:
+    uses: ./.github/workflows/CI-build-test.yml
+
+  notify-slack:
+    name: Notify Slack
+    needs: build-and-test
+    if: ${{ success() || failure() }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: iRoachie/[email protected]
+      if: env.SLACK_WEBHOOK_URL != null
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -9,13 +9,13 @@ ENV PATH="/home/inaturalist/.local/bin:${PATH}"
 
 RUN pip install --upgrade pip
 
-# set the working directory in the container
+# Set the working directory in the container
 WORKDIR /home/inaturalist/vision
 
-# copy the dependencies file to the working directory
-COPY --chown=inaturalist:inaturalist ./requirements-production.txt /home/inaturalist/vision/requirements.txt
+# Copy the dependencies file to the working directory
+COPY --chown=inaturalist:inaturalist ./requirements.txt /home/inaturalist/vision/requirements.txt
 
-# install dependencies
+# Install dependencies
 RUN UWSGI_EMBED_PLUGINS=stats_pusher_statsd pip install -r requirements.txt
 
 # Copy app and libs

requirements-test.txt

@@ -0,0 +1,4 @@
+coverage
+pytest
+pytest-cov
+pytest-mock

requirements.txt

@@ -1,4 +1,3 @@
-coverage
 flake8
 Flask
 Flask-WTF
@@ -12,17 +11,14 @@ numpy
 pandas
 Pillow
 prison
-pytest
-pytest-cov
-pytest-mock
 python-magic
 pyyaml
 scikit_learn
 scipy
 six
 tensorflow
 tifffile
-torch
 umap_learn
+uWSGI
 werkzeug
 WTForms