From 5af2e589a3db49dcb7cc972f77d1ca5e15f45d94 Mon Sep 17 00:00:00 2001 From: Sylvain Morin <56630013+sylvain-morin@users.noreply.github.com> Date: Tue, 5 Mar 2024 19:18:09 +0100 Subject: [PATCH] CICD / Docker image (#13) * New CICD workflow --- .github/workflows/CI-build-test.yml | 27 +++++++++++++++++ .github/workflows/CI-pre-build.yml | 14 +++++++++ .github/workflows/CI.yml | 34 --------------------- .github/workflows/CICD-dev.yml | 40 +++++++++++++++++-------- .github/workflows/CICD-main.yml | 46 ++++++++++++++++++++--------- .github/workflows/CICD-pr.yml | 20 +++++++++++++ Dockerfile | 8 ++--- requirements-production.txt | 26 ---------------- requirements-test.txt | 4 +++ requirements.txt | 6 +--- 10 files changed, 129 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/CI-build-test.yml create mode 100644 .github/workflows/CI-pre-build.yml delete mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/CICD-pr.yml delete mode 100644 requirements-production.txt create mode 100644 requirements-test.txt diff --git a/.github/workflows/CI-build-test.yml b/.github/workflows/CI-build-test.yml new file mode 100644 index 0000000..6485dd8 --- /dev/null +++ b/.github/workflows/CI-build-test.yml @@ -0,0 +1,27 @@ +name: Vision Build and Test + +on: + workflow_call: + +jobs: + build-and-test: + name: Build/Test + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Use Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.6' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-test.txt + + - name: Test with pytest + run: | + coverage run -m pytest -s && coverage report --show-missing diff --git a/.github/workflows/CI-pre-build.yml b/.github/workflows/CI-pre-build.yml new file mode 100644 index 0000000..00d4055 --- /dev/null +++ b/.github/workflows/CI-pre-build.yml @@ -0,0 +1,14 @@ +name: Pre Build + +on: + workflow_call: + +jobs: + pre-build: + name: Pre-build + runs-on: ubuntu-20.04 + steps: + - uses: fkirc/skip-duplicate-actions@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + cancel_others: true diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index 9bd7a08..0000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: inatVisionAPI CI - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v4 - - name: Use Python - uses: actions/setup-python@v4 - with: - python-version: '3.11.6' - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Test with pytest - run: | - coverage run -m pytest -s && coverage report --show-missing - - notify: - name: Notify Slack - needs: build - if: ${{ success() || failure() }} - runs-on: ubuntu-20.04 - steps: - - uses: iRoachie/slack-github-actions@v2.3.2 - if: env.SLACK_WEBHOOK_URL != null - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CICD-dev.yml b/.github/workflows/CICD-dev.yml index 5bddd82..23b70de 100644 --- a/.github/workflows/CICD-dev.yml +++ b/.github/workflows/CICD-dev.yml @@ -7,39 +7,53 @@ on: - '!main' jobs: + pre-build: + uses: ./.github/workflows/CI-pre-build.yml + build-and-test: - name: Build/Test - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 + needs: pre-build + uses: ./.github/workflows/CI-build-test.yml build-and-push-dev-docker-image: - name: Build/Push Dev Docker Image + name: Build/Push Dev Docker Image + needs: pre-build runs-on: ubuntu-20.04 - steps: + steps: - uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Build/Push dev vision-api + + - name: Build/Push Dev inatvisionapi uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.ref_name }} + tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.ref_name }} - - name: Build/Push dev vision-cleanup + - name: Build/Push Dev inatvisionapi-cleanup uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile-cleanup push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.ref_name }} + tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.ref_name }} + + notify-slack: + name: Notify Slack + needs: build-and-push-dev-docker-image + if: ${{ success() || failure() }} + runs-on: ubuntu-20.04 + steps: + - uses: iRoachie/slack-github-actions@v2.3.2 + if: env.SLACK_WEBHOOK_URL != null + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CICD-main.yml b/.github/workflows/CICD-main.yml index e000de7..6a8a434 100644 --- a/.github/workflows/CICD-main.yml +++ b/.github/workflows/CICD-main.yml @@ -6,39 +6,57 @@ on: - main jobs: + pre-build: + uses: ./.github/workflows/CI-pre-build.yml + build-and-test: - name: Build/Test - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 + needs: pre-build + uses: ./.github/workflows/CI-build-test.yml - build-and-push-staging-docker-image: - name: Build/Push Staging Docker Image + build-and-push-main-docker-image: + name: Build/Push Main Docker Image + needs: pre-build runs-on: ubuntu-20.04 - steps: + steps: - uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Build/Push staging vision-api + + - name: Build/Push Main inatvisionapi uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.sha }} + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.sha }} + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:latest - - name: Build/Push staging vision-cleanup + - name: Build/Push Main inatvisionapi-cleanup uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile-cleanup push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.sha }} + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.sha }} + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:latest + + notify-slack: + name: Notify Slack + needs: build-and-push-main-docker-image + if: ${{ success() || failure() }} + runs-on: ubuntu-20.04 + steps: + - uses: iRoachie/slack-github-actions@v2.3.2 + if: env.SLACK_WEBHOOK_URL != null + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CICD-pr.yml b/.github/workflows/CICD-pr.yml new file mode 100644 index 0000000..c333aef --- /dev/null +++ b/.github/workflows/CICD-pr.yml @@ -0,0 +1,20 @@ +name: Vision CI/CD Pull Request + +on: + pull_request: + +jobs: + build-and-test: + uses: ./.github/workflows/CI-build-test.yml + + notify-slack: + name: Notify Slack + needs: build-and-test + if: ${{ success() || failure() }} + runs-on: ubuntu-20.04 + steps: + - uses: iRoachie/slack-github-actions@v2.3.2 + if: env.SLACK_WEBHOOK_URL != null + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 43f2d18..683b0c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,13 @@ ENV PATH="/home/inaturalist/.local/bin:${PATH}" RUN pip install --upgrade pip -# set the working directory in the container +# Set the working directory in the container WORKDIR /home/inaturalist/vision -# copy the dependencies file to the working directory -COPY --chown=inaturalist:inaturalist ./requirements-production.txt /home/inaturalist/vision/requirements.txt +# Copy the dependencies file to the working directory +COPY --chown=inaturalist:inaturalist ./requirements.txt /home/inaturalist/vision/requirements.txt -# install dependencies +# Install dependencies RUN UWSGI_EMBED_PLUGINS=stats_pusher_statsd pip install -r requirements.txt # Copy app and libs diff --git a/requirements-production.txt b/requirements-production.txt deleted file mode 100644 index 97255a1..0000000 --- a/requirements-production.txt +++ /dev/null @@ -1,26 +0,0 @@ -flake8 -Flask -Flask-WTF -h3 -h3pandas -itsdangerous -jinja2 -keras -matplotlib -numpy -pandas -Pillow -prison -python-magic -pyyaml -scikit_learn -scipy -six -tensorflow -tifffile -umap_learn -uWSGI -werkzeug -WTForms -#-f https://download.pytorch.org/whl/torch_stable.html -#torch==2.1.0+cpu diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..e439c39 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +coverage +pytest +pytest-cov +pytest-mock diff --git a/requirements.txt b/requirements.txt index a4d3300..6de8913 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -coverage flake8 Flask Flask-WTF @@ -12,9 +11,6 @@ numpy pandas Pillow prison -pytest -pytest-cov -pytest-mock python-magic pyyaml scikit_learn @@ -22,7 +18,7 @@ scipy six tensorflow tifffile -torch umap_learn +uWSGI werkzeug WTForms