Merge pull request #55 from nimh-dsst/docker-speedup

use a base image for docker

.dockerignore

@@ -1 +1,2 @@
-.terraform
+*/.terraform
+tempdata/*

.gitignore

@@ -1,5 +1,6 @@
 # Ignore the virtual environment directory
 _version.py
+dashboard_data
 node_modules
 venv/
 *coverage*

Dockerfile.base

@@ -0,0 +1,14 @@
+FROM tiangolo/uvicorn-gunicorn:python3.11
+
+WORKDIR /app
+RUN mkdir -p /opt/data
+RUN mkdir -p /opt/osm
+
+RUN pip install poetry fastapi[standard] uvicorn
+# Install requirements first in a separate layer to avoid re-installing them on
+# every code changes
+COPY pyproject.toml /opt/osm
+RUN bash -c 'FORCE_OSM_VERSION=0.0.0 pip install -r <(poetry export --without-hashes --format=requirements.txt)'
+# Install the package
+COPY osm /opt/osm/osm
+RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install --no-dependencies -e /opt/osm

compose.development.override.yaml

@@ -1,13 +1,34 @@
 name: local-osm
 services:
+  base_image:
+    image: nimhdsst/osm_base
+    build:
+      context: .
+      dockerfile: Dockerfile.base
+
   rtransparent:
     build:
       context: .
       dockerfile: ./external_components/rtransparent/Dockerfile
     volumes:
       - ./external_components/rtransparent:/app
 
+  llm_extraction:
+    image: nimhdsst/osm_base
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY:-NOKEY}
+    build:
+      context: .
+      dockerfile: ./external_components/llm_extraction/Dockerfile
+    volumes:
+      - ./external_components/llm_extraction:/app
+    depends_on:
+      - base_image
+
+
+
   web_api:
+    image: nimhdsst/osm_base
     container_name: web_api
     environment:
       - MONGODB_URI=mongodb://db:27017/osm
@@ -22,21 +43,27 @@ services:
     working_dir: /app/app
     command: ["fastapi","dev","--host","0.0.0.0","--port","80"]
     depends_on:
-        - db
+      - db
+      - base_image
 
   dashboard:
+    image: nimhdsst/osm_base
     container_name: dashboard
+    environment:
+      - MONGODB_URI=mongodb://db:27017/osm
     build:
       context: .
       dockerfile: ./web/dashboard/Dockerfile
-    environment:
-      - MONGODB_URI=mongodb://db:27017/osm
-    working_dir: /app
     ports:
       - "8501:8501"
     volumes:
       - ./web/dashboard:/app
       - ./osm:/opt/osm/osm
+    working_dir: /app
+    command: ["python", "app.py"]
+    depends_on:
+      - db
+      - base_image
 
   db:
     container_name: db

compose.yaml

@@ -9,3 +9,8 @@ services:
     image: nimhdsst/rtransparent:staging
     ports:
       - "8071:8071"
+  llm_extraction:
+    container_name: llm_extraction
+    image: nimhdsst/llm_extraction:staging
+    ports:
+      - "8072:8072"

external_components/llm_extraction/Dockerfile

@@ -0,0 +1,7 @@
+FROM nimhdsst/osm_base
+RUN pip install llama-index llama-index-llms-openai llama-index-program-openai
+
+# # Copy the project files and install the package
+COPY external_components/llm_extraction/app.py /app
+
+CMD ["fastapi", "dev", "--host", "0.0.0.0", "--port", "8072"]

osm/__init__.py

@@ -3,6 +3,8 @@
 
 def get_version():
     try:
+        if os.environ.get("FORCE_OSM_VERSION"):
+            return os.environ["FORCE_OSM_VERSION"]
         from . import _version
 
         return _version.version

pyproject.toml

@@ -12,7 +12,6 @@ keywords = [
 dynamic = ["version"]
 dependencies = [
   "dill",
-  "colorcet",
   "pandas",
   "pyarrow",
   "pydantic",

web/api/Dockerfile

@@ -1,14 +1,5 @@
-FROM tiangolo/uvicorn-gunicorn:python3.11
+FROM nimhdsst/osm_base
 
-WORKDIR /app
-RUN pip install fastapi[standard]
-# Consider installing from pypi
-RUN mkdir -p /opt/osm
-COPY pyproject.toml /opt/osm
-COPY osm /opt/osm/osm
-ARG PSEUDO_VERSION=0.0.1 # strongly recommended to update based on git describe
-RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=${PSEUDO_VERSION} pip install -e /opt/osm
-RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install -e /opt/osm
 COPY ./web/api/main.py /app/app/main.py
 
 CMD ["fastapi", "run", "--host", "0.0.0.0", "--port", "80", "--root-path", "/api"]

web/dashboard/Dockerfile

@@ -1,22 +1,12 @@
-FROM tiangolo/uvicorn-gunicorn:python3.11
-
-WORKDIR /app
-
-
-# Create the environment
-RUN pip install panel pymongo odmantic pandas pydantic[email] pyarrow
-
-RUN mkdir -p /opt/data
-ENV LOCAL_DATA_PATH=/opt/data/matches.parquet
-COPY ./tempdata/matches.parquet /opt/data/matches.parquet
-
-RUN mkdir -p /opt/osm
-COPY pyproject.toml /opt/osm
-COPY osm /opt/osm/osm
-ARG PSEUDO_VERSION=0.0.1 # strongly recommended to update based on git describe
-RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=${PSEUDO_VERSION} pip install -e /opt/osm
-RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install -e /opt/osm
-
+FROM nimhdsst/osm_base
+RUN pip install colorcet panel
 # # Copy the project files and install the package
 COPY web/dashboard/ /app
 CMD ["python", "app.py"]
+
+
+# # Create the environment
+# RUN pip install panel pymongo odmantic pandas pydantic[email] pyarrow
+
+# ENV LOCAL_DATA_PATH=/opt/data/matches.parquet
+# COPY ./tempdata/matches.parquet /opt/data/matches.parquet