From 3a89806a4eb36ed4853ae04dc838a69d06874a20 Mon Sep 17 00:00:00 2001 From: leej3 Date: Wed, 28 Aug 2024 20:29:10 +0100 Subject: [PATCH] use a base image for docker --- .dockerignore | 3 +- .gitignore | 1 + Dockerfile.base | 14 ++++++++ compose.development.override.yaml | 35 ++++++++++++++++--- compose.yaml | 5 +++ external_components/llm_extraction/Dockerfile | 7 ++++ osm/__init__.py | 2 ++ pyproject.toml | 1 - web/api/Dockerfile | 11 +----- web/dashboard/Dockerfile | 28 +++++---------- 10 files changed, 72 insertions(+), 35 deletions(-) create mode 100644 Dockerfile.base create mode 100644 external_components/llm_extraction/Dockerfile diff --git a/.dockerignore b/.dockerignore index 3fa8c86b..6641a07d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -.terraform +*/.terraform +tempdata/* diff --git a/.gitignore b/.gitignore index e6f0a174..d921bad8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore the virtual environment directory _version.py +dashboard_data node_modules venv/ *coverage* diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 00000000..c78d50d9 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,14 @@ +FROM tiangolo/uvicorn-gunicorn:python3.11 + +WORKDIR /app +RUN mkdir -p /opt/data +RUN mkdir -p /opt/osm + +RUN pip install poetry fastapi[standard] uvicorn +# Install requirements first in a separate layer to avoid re-installing them on +# every code changes +COPY pyproject.toml /opt/osm +RUN bash -c 'FORCE_OSM_VERSION=0.0.0 pip install -r <(poetry export --without-hashes --format=requirements.txt)' +# Install the package +COPY osm /opt/osm/osm +RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install --no-dependencies -e /opt/osm diff --git a/compose.development.override.yaml b/compose.development.override.yaml index 933fd03c..1d2e4194 100644 --- a/compose.development.override.yaml +++ b/compose.development.override.yaml @@ -1,5 +1,11 @@ name: local-osm services: + base_image: + image: nimhdsst/osm_base + build: + context: . + dockerfile: Dockerfile.base + rtransparent: build: context: . @@ -7,7 +13,22 @@ services: volumes: - ./external_components/rtransparent:/app + llm_extraction: + image: nimhdsst/osm_base + environment: + - OPENAI_API_KEY=${OPENAI_API_KEY:-NOKEY} + build: + context: . + dockerfile: ./external_components/llm_extraction/Dockerfile + volumes: + - ./external_components/llm_extraction:/app + depends_on: + - base_image + + + web_api: + image: nimhdsst/osm_base container_name: web_api environment: - MONGODB_URI=mongodb://db:27017/osm @@ -22,21 +43,27 @@ services: working_dir: /app/app command: ["fastapi","dev","--host","0.0.0.0","--port","80"] depends_on: - - db + - db + - base_image dashboard: + image: nimhdsst/osm_base container_name: dashboard + environment: + - MONGODB_URI=mongodb://db:27017/osm build: context: . dockerfile: ./web/dashboard/Dockerfile - environment: - - MONGODB_URI=mongodb://db:27017/osm - working_dir: /app ports: - "8501:8501" volumes: - ./web/dashboard:/app - ./osm:/opt/osm/osm + working_dir: /app + command: ["python", "app.py"] + depends_on: + - db + - base_image db: container_name: db diff --git a/compose.yaml b/compose.yaml index 037890bf..b4d384e2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,3 +9,8 @@ services: image: nimhdsst/rtransparent:staging ports: - "8071:8071" + llm_extraction: + container_name: llm_extraction + image: nimhdsst/llm_extraction:staging + ports: + - "8072:8072" diff --git a/external_components/llm_extraction/Dockerfile b/external_components/llm_extraction/Dockerfile new file mode 100644 index 00000000..f87ea512 --- /dev/null +++ b/external_components/llm_extraction/Dockerfile @@ -0,0 +1,7 @@ +FROM nimhdsst/osm_base +RUN pip install llama-index llama-index-llms-openai llama-index-program-openai + +# # Copy the project files and install the package +COPY external_components/llm_extraction/app.py /app + +CMD ["fastapi", "dev", "--host", "0.0.0.0", "--port", "8072"] diff --git a/osm/__init__.py b/osm/__init__.py index 9d177dbb..1c12630c 100644 --- a/osm/__init__.py +++ b/osm/__init__.py @@ -3,6 +3,8 @@ def get_version(): try: + if os.environ.get("FORCE_OSM_VERSION"): + return os.environ["FORCE_OSM_VERSION"] from . import _version return _version.version diff --git a/pyproject.toml b/pyproject.toml index f13b6a4d..1808d06b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ keywords = [ dynamic = ["version"] dependencies = [ "dill", - "colorcet", "pandas", "pyarrow", "pydantic", diff --git a/web/api/Dockerfile b/web/api/Dockerfile index 7c7223d9..835873ac 100644 --- a/web/api/Dockerfile +++ b/web/api/Dockerfile @@ -1,14 +1,5 @@ -FROM tiangolo/uvicorn-gunicorn:python3.11 +FROM nimhdsst/osm_base -WORKDIR /app -RUN pip install fastapi[standard] -# Consider installing from pypi -RUN mkdir -p /opt/osm -COPY pyproject.toml /opt/osm -COPY osm /opt/osm/osm -ARG PSEUDO_VERSION=0.0.1 # strongly recommended to update based on git describe -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=${PSEUDO_VERSION} pip install -e /opt/osm -RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install -e /opt/osm COPY ./web/api/main.py /app/app/main.py CMD ["fastapi", "run", "--host", "0.0.0.0", "--port", "80", "--root-path", "/api"] diff --git a/web/dashboard/Dockerfile b/web/dashboard/Dockerfile index 7daf4b25..ff72a35d 100644 --- a/web/dashboard/Dockerfile +++ b/web/dashboard/Dockerfile @@ -1,22 +1,12 @@ -FROM tiangolo/uvicorn-gunicorn:python3.11 - -WORKDIR /app - - -# Create the environment -RUN pip install panel pymongo odmantic pandas pydantic[email] pyarrow - -RUN mkdir -p /opt/data -ENV LOCAL_DATA_PATH=/opt/data/matches.parquet -COPY ./tempdata/matches.parquet /opt/data/matches.parquet - -RUN mkdir -p /opt/osm -COPY pyproject.toml /opt/osm -COPY osm /opt/osm/osm -ARG PSEUDO_VERSION=0.0.1 # strongly recommended to update based on git describe -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=${PSEUDO_VERSION} pip install -e /opt/osm -RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install -e /opt/osm - +FROM nimhdsst/osm_base +RUN pip install colorcet panel # # Copy the project files and install the package COPY web/dashboard/ /app CMD ["python", "app.py"] + + +# # Create the environment +# RUN pip install panel pymongo odmantic pandas pydantic[email] pyarrow + +# ENV LOCAL_DATA_PATH=/opt/data/matches.parquet +# COPY ./tempdata/matches.parquet /opt/data/matches.parquet