Merge pull request #12 from inaturalist/docker-pleary

Docker pleary

.dockerignore

@@ -0,0 +1,6 @@
+__pyache__
+log/*
+static/*
+venv
+config.yml
+models

.github/workflows/CI.yml

@@ -0,0 +1,27 @@
+name: inatVisionAPI CI
+
+on: [push, pull_request]
+
+jobs:
+  build-push-docker-image:
+    name: Build and Push Docker Image
+    runs-on: ubuntu-20.04
+    steps:    
+    - uses: actions/checkout@v3
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}  
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./Dockerfile
+        push: true
+        tags: ${{ secrets.DOCKERHUB_USERNAME }}/vision:latest

.gitignore

@@ -99,3 +99,4 @@ models/*
 test-obs*
 lib/vision_cache/*
 config.yml
+env.list

Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.11.6 as base
+
+RUN apt-get update && apt-get install -y libgdal-dev uwsgi-plugin-python3
+
+RUN useradd -ms /bin/bash inaturalist
+USER inaturalist
+
+ENV PATH="/home/inaturalist/.local/bin:${PATH}"
+
+RUN pip install --upgrade pip
+
+# set the working directory in the container
+WORKDIR /home/inaturalist/vision
+
+# copy the dependencies file to the working directory
+COPY --chown=inaturalist:inaturalist ./requirements-production.txt /home/inaturalist/vision/requirements.txt
+
+# install dependencies
+RUN UWSGI_EMBED_PLUGINS=stats_pusher_statsd pip install -r requirements.txt
+
+# Copy app and libs
+COPY --chown=inaturalist:inaturalist app.py /home/inaturalist/vision
+COPY --chown=inaturalist:inaturalist lib /home/inaturalist/vision/lib
+
+# Create directories for the log and static content
+RUN mkdir /home/inaturalist/vision/log
+RUN mkdir /home/inaturalist/vision/static
+
+# Development target
+FROM base AS development
+
+# Run with built-in Flask server
+CMD [ "python", "app.py" ]
+
+# Production target with uwsgi
+FROM development AS production
+
+# Configure uwsgi
+ENV UWSGI_PLUGIN_DIR /usr/lib/uwsgi/plugins
+RUN mkdir /home/inaturalist/vision/uwsgi
+COPY docker/uwsgi.ini /home/inaturalist/vision/uwsgi.ini
+
+# Run with uwsgi
+CMD ["uwsgi", "--ini", "/home/inaturalist/vision/uwsgi.ini", "--stats", ":1717", "--stats-http"]

Dockerfile-cleanup

@@ -0,0 +1,20 @@
+FROM alpine:latest
+
+RUN apk add --no-cache logrotate
+
+RUN mkdir -p /var/vision/log
+RUN mkdir -p /var/vision/static
+RUN mkdir -p /var/vision/script
+
+# Configure logrotate hourly
+COPY docker/logrotate.conf /var/vision/script/logrotate.conf
+COPY docker/logrotate-cron.sh /var/vision/script/logrotate-cron.sh
+RUN chmod 400 /var/vision/script/logrotate.conf
+RUN crontab -l | { cat; echo "0 * * * * sh /var/vision/script/logrotate-cron.sh"; } | crontab -
+
+# Configure job to clean static folder
+COPY docker/clean-static-cron.sh /var/vision/script/clean-static-cron.sh
+RUN crontab -l | { cat; echo "*/10 * * * * sh /var/vision/script/clean-static-cron.sh"; } | crontab -
+
+# Run cron
+CMD ["/bin/sh", "-c", "crond -f"]

app.py

@@ -1,9 +1,26 @@
 import sys
 import yaml
+import os
 sys.path.append("./lib")
 from inat_vision_api import InatVisionAPI  # noqa: E402
 
-CONFIG = yaml.safe_load(open("config.yml"))
+if "APP_SECRET" in os.environ:
+    CONFIG = {
+        "app_secret": os.environ["APP_SECRET"],
+        "models": [{
+            "name": os.environ["MODEL_NAME"],
+            "vision_model_path": os.environ["VISION_MODEL_PATH"],
+            "taxonomy_path": os.environ["TAXONOMY_PATH"],
+            "tf_geo_elevation_model_path": os.environ["TF_GEO_MODEL_PATH"],
+            "elevation_h3_r4": os.environ["ELEVATION_H3_R4_PATH"],
+            "tf_elev_thresholds": os.environ["GEO_THRESHOLDS_PATH"],
+            "taxon_ranges_path": os.environ["TAXON_RANGES_PATH"]
+        }]
+    }
+    if "GEO_MIN" in os.environ:
+        CONFIG["models"][0]["geo_min"] = os.environ["GEO_MIN"]
+else:
+    CONFIG = yaml.safe_load(open("config.yml"))
 
 api = InatVisionAPI(CONFIG)
 app = api.app

config.yml.example

@@ -1,6 +1,10 @@
 app_secret: "somesecret"
-vision_model_path: "models/.../model.h5"
-taxonomy_path: "models/.../taxonomy.csv"
-tf_geo_model_path: "models/.../tf_gpmodel.h5"
-pt_geo_model_path: "models/.../model.pth.tar"
-use_pt_gp_model: False
+models:
+  - name: "ModelGenerationName"
+    vision_model_path: "models/.../vision_model.h5"
+    taxonomy_path: "models/.../taxonomy.csv"
+    tf_geo_elevation_model_path: "models/.../tf_gpmodel.h5"
+    elevation_h3_r4: "models/.../elevation_r4_5m.csv"
+    tf_elev_thresholds: "models/.../thresholds.csv"
+    taxon_ranges_path: "models/.../taxon_ranges"
+    geo_min: 0.005

docker/clean-static-cron.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+find /var/vision/static -type f -mmin +10 -exec rm {} \;

docker/logrotate-cron.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+/usr/sbin/logrotate -s /var/vision/script/logrotate.status /var/vision/script/logrotate.conf 

docker/logrotate.conf

@@ -0,0 +1,10 @@
+/var/vision/log/logstash.log {
+  hourly
+  rotate 5
+  size 10M
+  missingok
+  compress
+  delaycompress
+  notifempty
+  copytruncate
+}

docker/uwsgi.ini

@@ -0,0 +1,14 @@
+[uwsgi]
+wsgi-file = /home/inaturalist/vision/app.py
+protocol = uwsgi
+callable = app
+chmod-socket = 777
+socket= :8888
+master = true
+lazy-apps = true
+processes = 1
+threads = 1
+uid = www-data
+gid = www-data
+skip-atexit = true
+skip-atexit-teardown = true

env.list.example

@@ -0,0 +1,9 @@
+APP_SECRET=somesecret
+MODEL_NAME=ModelGenerationName
+VISION_MODEL_PATH=models/.../vision_model.h5
+TAXONOMY_PATH=models/.../taxonomy.csv
+TF_GEO_MODEL_PATH=models/.../tf_gpmodel.h5
+ELEVATION_H3_R4_PATH=models/.../elevation_r4_5m.csv
+GEO_THRESHOLDS_PATH=models/.../thresholds.csv
+TAXON_RANGES_PATH=models/.../taxon_ranges
+GEO_MIN=0.005

requirements-production.txt

@@ -0,0 +1,26 @@
+flake8
+Flask
+Flask-WTF
+h3
+h3pandas
+itsdangerous
+jinja2
+keras
+matplotlib
+numpy
+pandas
+Pillow
+prison
+python-magic
+pyyaml
+scikit_learn
+scipy
+six
+tensorflow
+tifffile
+umap_learn
+uWSGI
+werkzeug
+WTForms
+#-f https://download.pytorch.org/whl/torch_stable.html
+#torch==2.1.0+cpu

requirements.txt

@@ -1,23 +1,24 @@
-scipy
-six
+flake8
+Flask
+Flask-WTF
+h3
+h3pandas
+itsdangerous
+jinja2
+keras
 matplotlib
 numpy
-tifffile
-umap_learn
 pandas
-torch
-Flask
 Pillow
-scikit_learn
-pyyaml
-Flask-WTF
+prison
 python-magic
-keras
-jinja2
-itsdangerous
+pyyaml
+scikit_learn
+scipy
+six
+tensorflow
+tifffile
+torch
+umap_learn
 werkzeug
 WTForms
-flake8
-h3
-h3pandas
-prison