From 212cbda938d61e58a40593e304570db8efc15251 Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:40:45 +0100 Subject: [PATCH 01/10] refactor(docker): move docker to a folder --- .env.example => docker/.env.example | 0 Dockerfile => docker/Dockerfile | 0 .../apm}/grafana/dashboards/dashboard.json | 0 .../apm}/grafana/dashboards/dashboard.yml | 0 .../apm}/grafana/datasources/datasource.yml | 0 {apm => docker/apm}/prometheus.yml | 0 .../docker-compose.prod.yml | 6 +- .../docker-compose.test.yml | 0 docker/docker-compose.yml | 69 +++++++++++++++++++ 9 files changed, 71 insertions(+), 4 deletions(-) rename .env.example => docker/.env.example (100%) rename Dockerfile => docker/Dockerfile (100%) rename {apm => docker/apm}/grafana/dashboards/dashboard.json (100%) rename {apm => docker/apm}/grafana/dashboards/dashboard.yml (100%) rename {apm => docker/apm}/grafana/datasources/datasource.yml (100%) rename {apm => docker/apm}/prometheus.yml (100%) rename docker-compose.yml => docker/docker-compose.prod.yml (97%) rename docker-compose.test.yml => docker/docker-compose.test.yml (100%) create mode 100644 docker/docker-compose.yml diff --git a/.env.example b/docker/.env.example similarity index 100% rename from .env.example rename to docker/.env.example diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/apm/grafana/dashboards/dashboard.json b/docker/apm/grafana/dashboards/dashboard.json similarity index 100% rename from apm/grafana/dashboards/dashboard.json rename to docker/apm/grafana/dashboards/dashboard.json diff --git a/apm/grafana/dashboards/dashboard.yml b/docker/apm/grafana/dashboards/dashboard.yml similarity index 100% rename from apm/grafana/dashboards/dashboard.yml rename to docker/apm/grafana/dashboards/dashboard.yml diff --git a/apm/grafana/datasources/datasource.yml b/docker/apm/grafana/datasources/datasource.yml similarity index 100% rename from apm/grafana/datasources/datasource.yml rename to docker/apm/grafana/datasources/datasource.yml diff --git a/apm/prometheus.yml b/docker/apm/prometheus.yml similarity index 100% rename from apm/prometheus.yml rename to docker/apm/prometheus.yml diff --git a/docker-compose.yml b/docker/docker-compose.prod.yml similarity index 97% rename from docker-compose.yml rename to docker/docker-compose.prod.yml index 282ff83..293cc6b 100644 --- a/docker-compose.yml +++ b/docker/docker-compose.prod.yml @@ -2,10 +2,8 @@ version: '3.7' services: backend: - build: . + image: quackai/contribution-api:latest command: uvicorn app.main:app --reload --host 0.0.0.0 --port 8050 --proxy-headers - volumes: - - ./src/:/app/ ports: - "8050:8050" environment: @@ -19,7 +17,7 @@ services: - GH_OAUTH_ID=${GH_OAUTH_ID} - GH_OAUTH_SECRET=${GH_OAUTH_SECRET} - SUPPORT_EMAIL=${SUPPORT_EMAIL} - - DEBUG=true + - DEBUG=false depends_on: db: condition: service_healthy diff --git a/docker-compose.test.yml b/docker/docker-compose.test.yml similarity index 100% rename from docker-compose.test.yml rename to docker/docker-compose.test.yml diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..7a46700 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,69 @@ +version: '3.7' + +services: + backend: + build: + context: .. + dockerfile: ./docker/Dockerfile + command: uvicorn app.main:app --reload --host 0.0.0.0 --port 8050 --proxy-headers + volumes: + - ../src/:/app/ + ports: + - "8050:8050" + environment: + - POSTGRES_URL=postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB} + - OLLAMA_ENDPOINT=http://ollama:11434 + - OLLAMA_MODEL=${OLLAMA_MODEL} + - SECRET_KEY=${SECRET_KEY} + - SUPERADMIN_GH_PAT=${SUPERADMIN_GH_PAT} + - SUPERADMIN_LOGIN=${SUPERADMIN_LOGIN} + - SUPERADMIN_PWD=${SUPERADMIN_PWD} + - GH_OAUTH_ID=${GH_OAUTH_ID} + - GH_OAUTH_SECRET=${GH_OAUTH_SECRET} + - SUPPORT_EMAIL=${SUPPORT_EMAIL} + - DEBUG=true + depends_on: + db: + condition: service_healthy + ollama: + condition: service_healthy + + ollama: + image: ollama/ollama:0.1.29 + command: serve + volumes: + - "$HOME/.ollama:/root/.ollama" + expose: + - 11434 + healthcheck: + test: ["CMD-SHELL", "ollama pull '${OLLAMA_MODEL}'"] + interval: 5s + timeout: 1m + retries: 3 + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: 1 + # capabilities: [gpu] + + db: + image: postgres:15-alpine + volumes: + - postgres_data:/var/lib/postgresql/data/ + expose: + - 5432 + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + healthcheck: + test: ["CMD-SHELL", "sh -c 'pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}'"] + interval: 10s + timeout: 3s + retries: 3 + +volumes: + postgres_data: + ollama: From c66d7c893d30c878d5c0c2d2f01dec46717b77e6 Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:40:58 +0100 Subject: [PATCH 02/10] docs(makefile): update makefile --- Makefile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 2cc2984..cdccdf8 100644 --- a/Makefile +++ b/Makefile @@ -16,35 +16,35 @@ lock: # Build the docker build: poetry export -f requirements.txt --without-hashes --output requirements.txt - docker build . -t quackai/contribution-api:python3.9-alpine3.14 + docker build -f docker/Dockerfile . -t quackai/contribution-api:python3.9-alpine3.14 # Run the docker run: poetry export -f requirements.txt --without-hashes --output requirements.txt - docker compose up -d --build + docker compose -f docker/docker-compose.yml up -d --build # Run the docker stop: - docker compose down + docker compose -f docker/docker-compose.yml down run-dev: poetry export -f requirements.txt --without-hashes --with test --output requirements.txt - docker compose -f docker-compose.test.yml up -d --build + docker compose -f docker/docker-compose.test.yml up -d --build stop-dev: - docker compose -f docker-compose.test.yml down + docker compose -f docker/docker-compose.test.yml down # Run tests for the library test: poetry export -f requirements.txt --without-hashes --with test --output requirements.txt - docker compose -f docker-compose.test.yml up -d --build + docker compose -f docker/docker-compose.test.yml up -d --build docker compose exec -T backend pytest --cov=app - docker compose -f docker-compose.test.yml down + docker compose -f docker/docker-compose.test.yml down # Run tests for the library e2e: poetry export -f requirements.txt --without-hashes --output requirements.txt - docker compose -f docker-compose.test.yml up -d --build + docker compose -f docker/docker-compose.test.yml up -d --build sleep 5 python scripts/test_e2e.py - docker compose -f docker-compose.test.yml down + docker compose -f docker/docker-compose.test.yml down From 180f6471b2ee5e0d7cc76670186e9aeac194b16d Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:41:17 +0100 Subject: [PATCH 03/10] ci(github): update docker command --- .github/labeler.yml | 10 +++++----- .github/workflows/builds.yml | 6 +++--- .github/workflows/push.yml | 2 +- .github/workflows/scripts.yml | 2 +- .github/workflows/tests.yml | 20 ++++++++++---------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 7f5c07f..cb2936f 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -8,7 +8,7 @@ 'ext: apm': - changed-files: - - any-glob-to-any-file: apm/* + - any-glob-to-any-file: docker/apm/* 'module: core': - changed-files: @@ -79,10 +79,10 @@ 'topic: docker': - changed-files: - any-glob-to-any-file: - - docker-compose.yml - - docker-compose.test.yml - - Dockerfile - - traefik/* + - docker/docker-compose.yml + - docker/docker-compose.test.yml + - docker/Dockerfile + - docker/.env.example 'topic: docs': - changed-files: diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 3ac51a9..ed148e0 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -33,7 +33,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: pg_pwd OLLAMA_MODEL: tinydolphin:1.1b-v2.8-q4_0 - run: docker compose up -d --build + run: docker compose -f docker/docker-compose.yml up -d --build - name: Docker sanity check env: SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} @@ -46,7 +46,7 @@ jobs: POSTGRES_PASSWORD: pg_pwd OLLAMA_MODEL: tinydolphin:1.1b-v2.8-q4_0 run: | - sleep 20 && docker compose logs backend + sleep 20 && docker compose -f docker/docker-compose.yml logs backend nc -vz localhost 8050 - docker compose logs + docker compose -f docker/docker-compose.yml logs curl http://localhost:8050/docs diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5b8f229..410fe3a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -18,7 +18,7 @@ jobs: - name: Resolve dependencies run: poetry export -f requirements.txt --without-hashes --output requirements.txt - name: Build docker - run: docker build . -t quackai/contribution-api:latest + run: docker build -f docker/Dockerfile . -t quackai/contribution-api:latest - name: Login to DockerHub uses: docker/login-action@v3 with: diff --git a/.github/workflows/scripts.yml b/.github/workflows/scripts.yml index 150b1a6..20acd77 100644 --- a/.github/workflows/scripts.yml +++ b/.github/workflows/scripts.yml @@ -27,7 +27,7 @@ jobs: SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} - run: docker compose -f docker-compose.test.yml up -d --build + run: docker compose -f docker/docker-compose.test.yml up -d --build - name: Docker sanity check run: sleep 20 && nc -vz localhost 8050 - name: Install dependencies diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 933bed3..93b7e3c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,15 +24,15 @@ jobs: SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} - run: docker compose -f docker-compose.test.yml up -d --build + run: docker compose -f docker/docker-compose.test.yml up -d --build - name: Run docker test env: SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} run: | - docker compose -f docker-compose.test.yml exec -T backend pytest --cov=app --cov-report xml tests/ - docker compose -f docker-compose.test.yml cp backend:/app/coverage.xml ./coverage-src.xml + docker compose -f docker/docker-compose.test.yml exec -T backend pytest --cov=app --cov-report xml tests/ + docker compose -f docker/docker-compose.test.yml cp backend:/app/coverage.xml ./coverage-src.xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: @@ -57,17 +57,17 @@ jobs: SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} - run: docker compose -f docker-compose.test.yml up -d --build + run: docker compose -f docker/docker-compose.test.yml up -d --build - name: Debug run: | - docker compose -f docker-compose.test.yml exec -T backend alembic current - docker compose -f docker-compose.test.yml exec -T backend alembic history --verbose + docker compose -f docker/docker-compose.test.yml exec -T backend alembic current + docker compose -f docker/docker-compose.test.yml exec -T backend alembic history --verbose - name: Run migrations run: | - docker compose -f docker-compose.test.yml exec -T backend alembic stamp head - docker compose -f docker-compose.test.yml exec -T backend alembic history --verbose - docker compose -f docker-compose.test.yml exec -T backend alembic downgrade -1 - docker compose -f docker-compose.test.yml exec -T backend alembic upgrade +1 + docker compose -f docker/docker-compose.test.yml exec -T backend alembic stamp head + docker compose -f docker/docker-compose.test.yml exec -T backend alembic history --verbose + docker compose -f docker/docker-compose.test.yml exec -T backend alembic downgrade -1 + docker compose -f docker/docker-compose.test.yml exec -T backend alembic upgrade +1 headers: runs-on: ${{ matrix.os }} From 69f9aa4d110d97ec90b10260e7619d2fa7387896 Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:44:52 +0100 Subject: [PATCH 04/10] docs(readme): update .env.example ref --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44cb558..62ed4da 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Optionally, the following information can be added: _** marks the deprecated values._ -So your `.env` file should look like something similar to [`.env.example`](.env.example) +So your `.env` file should look like something similar to [`.env.example`](./docker/.env.example) The file should be placed in the folder of your `./docker-compose.yml`. ## Contributing From aa66f3a20ddee25db162797d6c9c0ec9f9d51935 Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:53:32 +0100 Subject: [PATCH 05/10] build(docker): fix docker test command --- Makefile | 4 ++-- docker/docker-compose.test.yml | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index cdccdf8..950aa21 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lock: # Build the docker build: poetry export -f requirements.txt --without-hashes --output requirements.txt - docker build -f docker/Dockerfile . -t quackai/contribution-api:python3.9-alpine3.14 + docker build -f docker/Dockerfile . -t quackai/contribution-api:latest # Run the docker run: @@ -38,7 +38,7 @@ stop-dev: test: poetry export -f requirements.txt --without-hashes --with test --output requirements.txt docker compose -f docker/docker-compose.test.yml up -d --build - docker compose exec -T backend pytest --cov=app + docker compose -f docker/docker-compose.test.yml exec -T backend pytest --cov=app docker compose -f docker/docker-compose.test.yml down # Run tests for the library diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml index 9722869..7e692c7 100644 --- a/docker/docker-compose.test.yml +++ b/docker/docker-compose.test.yml @@ -2,10 +2,12 @@ version: '3.7' services: backend: - build: . + build: + context: .. + dockerfile: ./docker/Dockerfile command: uvicorn app.main:app --host 0.0.0.0 --port 8050 volumes: - - ./src/:/app/ + - ../src/:/app/ ports: - "8050:8050" environment: @@ -50,13 +52,13 @@ services: interval: 5s timeout: 1m retries: 3 - # deploy: - # resources: - # reservations: - # devices: - # - driver: nvidia - # count: 1 - # capabilities: [gpu] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] volumes: ollama: From 9de44c2b8c65e030e507dea4b2a54423622dd96e Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:55:40 +0100 Subject: [PATCH 06/10] build(docker): bump python to 3.11 --- docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0cd516d..eda7770 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.9-alpine3.14 +FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim WORKDIR /app @@ -11,8 +11,7 @@ ENV PYTHONPATH "${PYTHONPATH}:/app" COPY requirements.txt /app/requirements.txt # install dependencies -RUN set -eux \ - && pip install --no-cache-dir uv \ +RUN pip install --no-cache-dir uv \ && uv pip install --no-cache --system -r /app/requirements.txt \ && rm -rf /root/.cache From bce66172bbe336714aad7b56d07712b1de1d028d Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:12:08 +0100 Subject: [PATCH 07/10] buidl(docker): revert python update --- docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index eda7770..0cd516d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim +FROM tiangolo/uvicorn-gunicorn-fastapi:python3.9-alpine3.14 WORKDIR /app @@ -11,7 +11,8 @@ ENV PYTHONPATH "${PYTHONPATH}:/app" COPY requirements.txt /app/requirements.txt # install dependencies -RUN pip install --no-cache-dir uv \ +RUN set -eux \ + && pip install --no-cache-dir uv \ && uv pip install --no-cache --system -r /app/requirements.txt \ && rm -rf /root/.cache From 63b7799879b6ec38f900ac5c798cc3d49bbf9a3e Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:12:23 +0100 Subject: [PATCH 08/10] docs(contributing): update codebase description --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e68db12..ea043bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,7 @@ Whatever the way you wish to contribute to the project, please respect the [code - [`src/app`](https://github.com/quack-ai/contribution-api/blob/main/src/app) - The actual API codebase - [`src/tests`](https://github.com/quack-ai/contribution-api/blob/main/src/tests) - The API unit tests - [`.github`](https://github.com/quack-ai/contribution-api/blob/main/.github) - Configuration for CI (GitHub Workflows) +- [`docker`](https://github.com/quack-ai/contribution-api/blob/main/docker) - Docker-related configurations - [`docs`](https://github.com/quack-ai/contribution-api/blob/main/docs) - Everything related to documentation - [`scripts`](https://github.com/quack-ai/contribution-api/blob/main/scripts) - Custom scripts From 87fab967c9f528812a413b4add424664d827db26 Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:12:43 +0100 Subject: [PATCH 09/10] docs(readme): update configuration --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 62ed4da..ee455c0 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,11 @@ The project was designed so that everything runs with Docker orchestration (stan ### Configuration In order to run the project, you will need to specific some information, which can be done using a `.env` file. +Copy the default environement variables from [`.env.example`](./docker/.env.example): +```shell +cp docker/.env.example .env +``` + This file will have to hold the following information: - `POSTGRES_DB`*: a name for the [PostgreSQL](https://www.postgresql.org/) database that will be created - `POSTGRES_USER`*: a login for the PostgreSQL database @@ -133,12 +138,7 @@ Optionally, the following information can be added: - `SLACK_CHANNEL`: the Slack channel where your bot will post events (defaults to `#general`, you have to invite the App to your channel). - `SUPPORT_EMAIL`: the email used for support of your API. - `DEBUG`: if set to false, silence debug logs. -- `OPENAI_API_KEY`**: your API key for Open AI (Create new secret key on [OpenAI](https://platform.openai.com/api-keys)) - -_** marks the deprecated values._ -So your `.env` file should look like something similar to [`.env.example`](./docker/.env.example) -The file should be placed in the folder of your `./docker-compose.yml`. ## Contributing From 6a7fa2472bce7847613efcc09ed2f46569cc6d4f Mon Sep 17 00:00:00 2001 From: F-G Fernandez <26927750+frgfm@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:15:15 +0100 Subject: [PATCH 10/10] build(docker): update GPU access in docker compose --- docker/docker-compose.prod.yml | 14 +++++++------- docker/docker-compose.test.yml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 293cc6b..1a0d40b 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -36,13 +36,13 @@ services: interval: 5s timeout: 1m retries: 3 - # deploy: - # resources: - # reservations: - # devices: - # - driver: nvidia - # count: 1 - # capabilities: [gpu] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] db: image: postgres:15-alpine diff --git a/docker/docker-compose.test.yml b/docker/docker-compose.test.yml index 7e692c7..1ed5f8b 100644 --- a/docker/docker-compose.test.yml +++ b/docker/docker-compose.test.yml @@ -52,13 +52,13 @@ services: interval: 5s timeout: 1m retries: 3 - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu] + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: 1 + # capabilities: [gpu] volumes: ollama: