Merge pull request #57 from nimh-dsst/docker-speedup

Docker speedup

.dockerignore

@@ -1,2 +1,21 @@
-*/.terraform
+**/.terraform
 tempdata/*
+.venv
+.DS_Store
+.env
+.gitattributes
+.github
+.gitignore
+.pre-commit-config.yaml
+.public_dns
+.pytest_cache
+.ruff_cache
+LICENSE
+README.md
+build
+draft_of_specification_2024-05-06.md
+*.egg-info
+osm_output
+scripts
+tests
+uv.lock

.gitignore

@@ -1,4 +1,5 @@
 # Ignore the virtual environment directory
+.git
 _version.py
 dashboard_data
 node_modules
@@ -23,3 +24,4 @@ osm_output
 .terraform.lock.hcl
 .public_dns
 tempdata
+.aider*

Dockerfile.base

@@ -1,14 +1,20 @@
 FROM tiangolo/uvicorn-gunicorn:python3.11
-
+SHELL ["/bin/bash", "--login", "-c"]
+COPY --from=ghcr.io/astral-sh/uv:0.4.0 /uv /bin/uv
 WORKDIR /app
 RUN mkdir -p /opt/data
-RUN mkdir -p /opt/osm
-
-RUN pip install poetry fastapi[standard] uvicorn
+RUN mkdir -p /opt/osm/osm
+ENV UV_SYSTEM_PYTHON=1
+# ENV UV_PYTHON=/usr/local/bin/python
 # Install requirements first in a separate layer to avoid re-installing them on
 # every code changes
 COPY pyproject.toml /opt/osm
-RUN bash -c 'FORCE_OSM_VERSION=0.0.0 pip install -r <(poetry export --without-hashes --format=requirements.txt)'
-# Install the package
-COPY osm /opt/osm/osm
-RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install --no-dependencies -e /opt/osm
+RUN --mount=type=cache,target=/root/.cache/uv \
+    SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=9.9.9; \
+    cd /opt/osm; \
+    uv pip install --system  -r <(uv pip compile --all-extras pyproject.toml )
+COPY ./osm /opt/osm/osm
+RUN --mount=source=.git,target=/opt/osm/.git,type=bind \
+    --mount=type=cache,target=/root/.cache/uv \
+    cd /opt/osm; \
+    uv pip install  --system -r <(uv pip compile --all-extras pyproject.toml ) -e  .

compose.development.override.yaml

@@ -1,75 +1,78 @@
 name: local-osm
 services:
-  base_image:
-    image: nimhdsst/osm_base
-    build:
-      context: .
-      dockerfile: Dockerfile.base
 
   rtransparent:
+    container_name: rtransparent
     build:
       context: .
       dockerfile: ./external_components/rtransparent/Dockerfile
     volumes:
       - ./external_components/rtransparent:/app
 
-  llm_extraction:
-    image: nimhdsst/osm_base
-    environment:
-      - OPENAI_API_KEY=${OPENAI_API_KEY:-NOKEY}
-    build:
-      context: .
-      dockerfile: ./external_components/llm_extraction/Dockerfile
-    volumes:
-      - ./external_components/llm_extraction:/app
-    depends_on:
-      - base_image
-
 
+############ Development images ############
 
-  web_api:
-    image: nimhdsst/osm_base
-    container_name: web_api
+  db:
+  # use old version of mongo to avoid Apple Instruction set error
+    image: mongo:4.4.6
+    container_name: db
+    ports:
+      - 27017:27017
     environment:
-      - MONGODB_URI=mongodb://db:27017/osm
+      - MONGO_INITDB_DATABASE=osm
+
+  base:
+    command: ["echo", "base image"]
+    image: nimhdsst/osm_base:latest
     build:
       context: .
-      dockerfile: ./web/api/Dockerfile
-    ports:
-      - 80:80
-    volumes:
-      - ./web/api:/app/app
-      - ./osm:/opt/osm/osm
-    working_dir: /app/app
-    command: ["fastapi","dev","--host","0.0.0.0","--port","80"]
-    depends_on:
-      - db
-      - base_image
+      dockerfile: Dockerfile.base
 
   dashboard:
-    image: nimhdsst/osm_base
     container_name: dashboard
     environment:
       - MONGODB_URI=mongodb://db:27017/osm
+    image: nimhdsst/osm_dashboard:latest
     build:
       context: .
       dockerfile: ./web/dashboard/Dockerfile
     ports:
       - "8501:8501"
     volumes:
       - ./web/dashboard:/app
-      - ./osm:/opt/osm/osm
-    working_dir: /app
-    command: ["python", "app.py"]
+      - ./dashboard_data:/opt/data
     depends_on:
       - db
-      - base_image
+      - base
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./osm
+          target: /opt/osm/osm
+          ignore:
+            - __pycache__
 
-  db:
-    container_name: db
-  # use old version of mongo to avoid Apple Instruction set error
-    image: mongo:4.4.6
-    ports:
-      - 27017:27017
+  web_api:
+    container_name: web_api
+    working_dir: /app/app
+    command: ["fastapi","dev","--host","0.0.0.0","--port","80"]
     environment:
-      - MONGO_INITDB_DATABASE=osm
+      - MONGODB_URI=mongodb://db:27017/osm
+    image: nimhdsst/osm_web_api:latest
+    build:
+      context: .
+      dockerfile: ./web/api/Dockerfile
+    ports:
+      - 80:80
+    volumes:
+      - ./web/api:/app/app
+    depends_on:
+      - db
+      - base
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./osm
+          target: /opt/osm/osm
+          ignore:
+            - __pycache__

compose.yaml

@@ -9,8 +9,3 @@ services:
     image: nimhdsst/rtransparent:staging
     ports:
       - "8071:8071"
-  llm_extraction:
-    container_name: llm_extraction
-    image: nimhdsst/llm_extraction:staging
-    ports:
-      - "8072:8072"

external_components/llm_extraction/Dockerfile

@@ -1,7 +1,10 @@
 FROM nimhdsst/osm_base
-RUN pip install llama-index llama-index-llms-openai llama-index-program-openai
 
-# # Copy the project files and install the package
 COPY external_components/llm_extraction/app.py /app
 
+# Install the package
+COPY osm /opt/osm/osm
+RUN --mount=type=cache,target=/root/.cache/uv \
+    export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=9.9.9; \
+    cd /opt/osm ; uv pip install -e .
 CMD ["fastapi", "dev", "--host", "0.0.0.0", "--port", "8072"]

external_components/rtransparent/environment.yaml

@@ -25,10 +25,10 @@ dependencies:
   - r-lubridate
   - r-magrittr
   - r-pbapply
-  - r-pdftools
+  #- r-pdftools
   - r-plyr
   - r-purrr
-  - r-qpdf
+  #- r-qpdf
   - r-readr
   # - r-rentrez
   - r-rlang

osm/__init__.py

@@ -3,8 +3,6 @@
 
 def get_version():
     try:
-        if os.environ.get("FORCE_OSM_VERSION"):
-            return os.environ["FORCE_OSM_VERSION"]
         from . import _version
 
         return _version.version
@@ -18,7 +16,10 @@ def get_version():
 def generate_version_file():
     import pkg_resources
 
-    version = pkg_resources.get_distribution("osm").version
+    if os.get("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM"):
+        version = os.environ["SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM"]
+    else:
+        version = pkg_resources.get_distribution("osm").version
     version_file_content = f"version = '{version}'\n"
 
     version_file_path = os.path.join(os.path.dirname(__file__), "_version.py")

pyproject.toml

@@ -9,6 +9,7 @@ keywords = [
   "scientific transparency",
   "reproducibility"
 ]
+
 dynamic = ["version"]
 dependencies = [
   "dill",
@@ -21,6 +22,28 @@ dependencies = [
   "requests",
 ]
 
+[project.optional-dependencies]
+dev = [
+  "pdbpp",
+  "ipython",
+  "ruff",
+  "pytest",
+]
+server = [
+  "fastapi[standard]",
+  "uvicorn",
+  "gunicorn",
+]
+llm = [
+  "llama-index",
+  "llama-index-llms-openai",
+  "llama-index-program-openai",
+  "llama-index-llms-openrouter"
+]
+plot = [
+  "colorcet",
+  "panel",
+]
 [project.urls]
 homepage = "https://website"
 source = "https://github.com/nimh-dsst/osm"
@@ -39,10 +62,6 @@ where = ["."]
 [project.scripts]
 osm = "osm.cli:main"
 
-[tool.black]
-line-length = 88
-target-version = ["py38", "py39", "py310"]
-
 [tool.ruff]
 line-length = 88
 indent-width = 4

web/api/Dockerfile

@@ -1,5 +1,3 @@
 FROM nimhdsst/osm_base
-
 COPY ./web/api/main.py /app/app/main.py
-
 CMD ["fastapi", "run", "--host", "0.0.0.0", "--port", "80", "--root-path", "/api"]

web/dashboard/Dockerfile

@@ -1,12 +1,5 @@
+
 FROM nimhdsst/osm_base
-RUN pip install colorcet panel
-# # Copy the project files and install the package
 COPY web/dashboard/ /app
+ENV LOCAL_DATA_PATH=/opt/data/matches.parquet
 CMD ["python", "app.py"]
-
-
-# # Create the environment
-# RUN pip install panel pymongo odmantic pandas pydantic[email] pyarrow
-
-# ENV LOCAL_DATA_PATH=/opt/data/matches.parquet
-# COPY ./tempdata/matches.parquet /opt/data/matches.parquet