From 5af2e589a3db49dcb7cc972f77d1ca5e15f45d94 Mon Sep 17 00:00:00 2001 From: Sylvain Morin <56630013+sylvain-morin@users.noreply.github.com> Date: Tue, 5 Mar 2024 19:18:09 +0100 Subject: [PATCH 1/3] CICD / Docker image (#13) * New CICD workflow --- .github/workflows/CI-build-test.yml | 27 +++++++++++++++++ .github/workflows/CI-pre-build.yml | 14 +++++++++ .github/workflows/CI.yml | 34 --------------------- .github/workflows/CICD-dev.yml | 40 +++++++++++++++++-------- .github/workflows/CICD-main.yml | 46 ++++++++++++++++++++--------- .github/workflows/CICD-pr.yml | 20 +++++++++++++ Dockerfile | 8 ++--- requirements-production.txt | 26 ---------------- requirements-test.txt | 4 +++ requirements.txt | 6 +--- 10 files changed, 129 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/CI-build-test.yml create mode 100644 .github/workflows/CI-pre-build.yml delete mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/CICD-pr.yml delete mode 100644 requirements-production.txt create mode 100644 requirements-test.txt diff --git a/.github/workflows/CI-build-test.yml b/.github/workflows/CI-build-test.yml new file mode 100644 index 0000000..6485dd8 --- /dev/null +++ b/.github/workflows/CI-build-test.yml @@ -0,0 +1,27 @@ +name: Vision Build and Test + +on: + workflow_call: + +jobs: + build-and-test: + name: Build/Test + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Use Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.6' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-test.txt + + - name: Test with pytest + run: | + coverage run -m pytest -s && coverage report --show-missing diff --git a/.github/workflows/CI-pre-build.yml b/.github/workflows/CI-pre-build.yml new file mode 100644 index 0000000..00d4055 --- /dev/null +++ b/.github/workflows/CI-pre-build.yml @@ -0,0 +1,14 @@ +name: Pre Build + +on: + workflow_call: + +jobs: + pre-build: + name: Pre-build + runs-on: ubuntu-20.04 + steps: + - uses: fkirc/skip-duplicate-actions@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + cancel_others: true diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index 9bd7a08..0000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: inatVisionAPI CI - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v4 - - name: Use Python - uses: actions/setup-python@v4 - with: - python-version: '3.11.6' - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Test with pytest - run: | - coverage run -m pytest -s && coverage report --show-missing - - notify: - name: Notify Slack - needs: build - if: ${{ success() || failure() }} - runs-on: ubuntu-20.04 - steps: - - uses: iRoachie/slack-github-actions@v2.3.2 - if: env.SLACK_WEBHOOK_URL != null - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CICD-dev.yml b/.github/workflows/CICD-dev.yml index 5bddd82..23b70de 100644 --- a/.github/workflows/CICD-dev.yml +++ b/.github/workflows/CICD-dev.yml @@ -7,39 +7,53 @@ on: - '!main' jobs: + pre-build: + uses: ./.github/workflows/CI-pre-build.yml + build-and-test: - name: Build/Test - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 + needs: pre-build + uses: ./.github/workflows/CI-build-test.yml build-and-push-dev-docker-image: - name: Build/Push Dev Docker Image + name: Build/Push Dev Docker Image + needs: pre-build runs-on: ubuntu-20.04 - steps: + steps: - uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Build/Push dev vision-api + + - name: Build/Push Dev inatvisionapi uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.ref_name }} + tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.ref_name }} - - name: Build/Push dev vision-cleanup + - name: Build/Push Dev inatvisionapi-cleanup uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile-cleanup push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.ref_name }} + tags: ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.ref_name }} + + notify-slack: + name: Notify Slack + needs: build-and-push-dev-docker-image + if: ${{ success() || failure() }} + runs-on: ubuntu-20.04 + steps: + - uses: iRoachie/slack-github-actions@v2.3.2 + if: env.SLACK_WEBHOOK_URL != null + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CICD-main.yml b/.github/workflows/CICD-main.yml index e000de7..6a8a434 100644 --- a/.github/workflows/CICD-main.yml +++ b/.github/workflows/CICD-main.yml @@ -6,39 +6,57 @@ on: - main jobs: + pre-build: + uses: ./.github/workflows/CI-pre-build.yml + build-and-test: - name: Build/Test - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 + needs: pre-build + uses: ./.github/workflows/CI-build-test.yml - build-and-push-staging-docker-image: - name: Build/Push Staging Docker Image + build-and-push-main-docker-image: + name: Build/Push Main Docker Image + needs: pre-build runs-on: ubuntu-20.04 - steps: + steps: - uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Build/Push staging vision-api + + - name: Build/Push Main inatvisionapi uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-api:${{ github.sha }} + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:${{ github.sha }} + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi:latest - - name: Build/Push staging vision-cleanup + - name: Build/Push Main inatvisionapi-cleanup uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile-cleanup push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision-cleanup:${{ github.sha }} + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:${{ github.sha }} + ${{ secrets.DOCKERHUB_USERNAME }}/inatvisionapi-cleanup:latest + + notify-slack: + name: Notify Slack + needs: build-and-push-main-docker-image + if: ${{ success() || failure() }} + runs-on: ubuntu-20.04 + steps: + - uses: iRoachie/slack-github-actions@v2.3.2 + if: env.SLACK_WEBHOOK_URL != null + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CICD-pr.yml b/.github/workflows/CICD-pr.yml new file mode 100644 index 0000000..c333aef --- /dev/null +++ b/.github/workflows/CICD-pr.yml @@ -0,0 +1,20 @@ +name: Vision CI/CD Pull Request + +on: + pull_request: + +jobs: + build-and-test: + uses: ./.github/workflows/CI-build-test.yml + + notify-slack: + name: Notify Slack + needs: build-and-test + if: ${{ success() || failure() }} + runs-on: ubuntu-20.04 + steps: + - uses: iRoachie/slack-github-actions@v2.3.2 + if: env.SLACK_WEBHOOK_URL != null + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 43f2d18..683b0c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,13 @@ ENV PATH="/home/inaturalist/.local/bin:${PATH}" RUN pip install --upgrade pip -# set the working directory in the container +# Set the working directory in the container WORKDIR /home/inaturalist/vision -# copy the dependencies file to the working directory -COPY --chown=inaturalist:inaturalist ./requirements-production.txt /home/inaturalist/vision/requirements.txt +# Copy the dependencies file to the working directory +COPY --chown=inaturalist:inaturalist ./requirements.txt /home/inaturalist/vision/requirements.txt -# install dependencies +# Install dependencies RUN UWSGI_EMBED_PLUGINS=stats_pusher_statsd pip install -r requirements.txt # Copy app and libs diff --git a/requirements-production.txt b/requirements-production.txt deleted file mode 100644 index 97255a1..0000000 --- a/requirements-production.txt +++ /dev/null @@ -1,26 +0,0 @@ -flake8 -Flask -Flask-WTF -h3 -h3pandas -itsdangerous -jinja2 -keras -matplotlib -numpy -pandas -Pillow -prison -python-magic -pyyaml -scikit_learn -scipy -six -tensorflow -tifffile -umap_learn -uWSGI -werkzeug -WTForms -#-f https://download.pytorch.org/whl/torch_stable.html -#torch==2.1.0+cpu diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..e439c39 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +coverage +pytest +pytest-cov +pytest-mock diff --git a/requirements.txt b/requirements.txt index a4d3300..6de8913 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -coverage flake8 Flask Flask-WTF @@ -12,9 +11,6 @@ numpy pandas Pillow prison -pytest -pytest-cov -pytest-mock python-magic pyyaml scikit_learn @@ -22,7 +18,7 @@ scipy six tensorflow tifffile -torch umap_learn +uWSGI werkzeug WTForms From b5ab344f195ebe0e41346d182b0ca7532e2a3dd7 Mon Sep 17 00:00:00 2001 From: Alex Shepard Date: Tue, 19 Mar 2024 14:08:15 -0700 Subject: [PATCH 2/3] resize_with_crop_or_pad gives better overall results see https://github.com/inaturalist/iNaturalistMLWork/issues/24 for some results --- lib/inat_inferrer.py | 5 +++-- lib/vision_testing.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/inat_inferrer.py b/lib/inat_inferrer.py index 2ae5e92..f9eb0aa 100644 --- a/lib/inat_inferrer.py +++ b/lib/inat_inferrer.py @@ -364,8 +364,9 @@ def prepare_image_for_inference(file_path): image = tf.io.read_file(file_path) image = tf.image.decode_jpeg(image, channels=3) image = tf.image.convert_image_dtype(image, tf.float32) - image = tf.image.central_crop(image, 0.875) - image = tf.image.resize(image, [299, 299], tf.image.ResizeMethod.NEAREST_NEIGHBOR) + image = tf.image.resize_with_crop_or_pad( + image, 299, 299 + ) return tf.expand_dims(image, 0) @staticmethod diff --git a/lib/vision_testing.py b/lib/vision_testing.py index 73f21ce..54a8638 100644 --- a/lib/vision_testing.py +++ b/lib/vision_testing.py @@ -153,8 +153,9 @@ def prepare_image_for_inference(self, cache_path): image = tf.io.read_file(cache_path) image = tf.image.decode_jpeg(image, channels=3) image = tf.image.convert_image_dtype(image, tf.float32) - image = tf.image.central_crop(image, 0.875) - image = tf.image.resize(image, [299, 299], tf.image.ResizeMethod.NEAREST_NEIGHBOR) + image = tf.image.resize_with_crop_or_pad( + image, 299, 299 + ) return tf.expand_dims(image, 0) def assess_top_results(self, observation, top_results): From 533695ca3112fac73dcca904e90665562ad7c46e Mon Sep 17 00:00:00 2001 From: Patrick Leary Date: Thu, 21 Mar 2024 17:20:23 -0400 Subject: [PATCH 3/3] specify requirements versions; prefer double quotes with flake8; test on multiple python versions --- .flake8 | 2 +- .github/workflows/CI-build-test.yml | 11 ++++-- requirements-test.txt | 8 ++-- requirements.txt | 57 +++++++++++++++++------------ 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/.flake8 b/.flake8 index 998a753..4b8cb1b 100644 --- a/.flake8 +++ b/.flake8 @@ -11,4 +11,4 @@ exclude = test-obs*, venv max-complexity = 10 - +inline-quotes = " diff --git a/.github/workflows/CI-build-test.yml b/.github/workflows/CI-build-test.yml index 6485dd8..b0cdb7a 100644 --- a/.github/workflows/CI-build-test.yml +++ b/.github/workflows/CI-build-test.yml @@ -7,13 +7,18 @@ jobs: build-and-test: name: Build/Test runs-on: ubuntu-20.04 - steps: + strategy: + matrix: + python-version: + - 3.8.10 + - 3.11.6 + steps: - uses: actions/checkout@v4 - - name: Use Python + - name: Use Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: '3.11.6' + python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install dependencies diff --git a/requirements-test.txt b/requirements-test.txt index e439c39..cc92d30 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -coverage -pytest -pytest-cov -pytest-mock +coverage==7.4.4 +pytest==8.1.1 +pytest-cov==4.1.0 +pytest-mock==3.12.0 diff --git a/requirements.txt b/requirements.txt index 6de8913..8992f7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,33 @@ -flake8 -Flask -Flask-WTF -h3 -h3pandas -itsdangerous -jinja2 -keras -matplotlib -numpy -pandas -Pillow -prison -python-magic -pyyaml -scikit_learn -scipy -six -tensorflow -tifffile -umap_learn -uWSGI -werkzeug -WTForms +flake8==7.0.0 +flake8-quotes==3.4.0 +Flask==3.0.2 +Flask-WTF==1.2.1 +h3==3.7.7 +h3pandas==0.2.6 +itsdangerous==2.1.2 +Jinja2==3.1.3 +keras==2.14.0;python_version>="3.11" +keras==2.12.0;python_version=="3.8" +matplotlib==3.8.3;python_version>="3.11" +matplotlib==3.7.5;python_version=="3.8" +numpy==1.26.4;python_version>="3.11" +numpy==1.23.5;python_version=="3.8" +pandas==2.1.2;python_version>="3.11" +pandas==2.0.3;python_version=="3.8" +pillow==10.2.0 +prison==0.2.1 +python-magic==0.4.27 +PyYAML==6.0.1 +scikit-learn==1.4.1.post1;python_version>="3.11" +scikit-learn==1.3.2;python_version=="3.8" +scipy==1.12.0;python_version>="3.11" +scipy==1.10.1;python_version=="3.8" +six==1.16.0 +tensorflow==2.14.0;python_version>="3.11" +tensorflow==2.12.0;python_version=="3.8" +tifffile==2024.2.12;python_version>="3.11" +tifffile==2023.7.10;python_version=="3.8" +umap-learn==0.5.5 +uWSGI==2.0.24 +Werkzeug==3.0.1 +WTForms==3.1.2