From 49a41c05ea3d0f4fe9898c7d7a5df1359f3e7fe6 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 09:27:23 +0100
Subject: [PATCH 01/41] add x-airflow-common

---
 development/docker-compose.yml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 8998078..f8abace 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -9,6 +9,27 @@ x-logging:
       loki-retries: 5
       loki-batch-size: 400
 
+x-airflow-common:
+  &airflow-common
+  image: ghcr.io/togethercrew/airflow-dags:main
+  env_file:
+    &airflow-common-env
+    - ./.env.airflow
+  volumes:
+    - ./dags:/opt/airflow/dags:ro
+    - airflow_logs:/opt/airflow/logs
+    - airflow_config:/opt/airflow/config
+    - airflow_plugins:/opt/airflow/plugins
+  user: "50000:0"
+  depends_on:
+    &airflow-common-depends-on
+    airflow-redis:
+      condition: service_healthy
+    postgres:
+      condition: service_healthy
+    neo4j-dev:
+      condition: service_healthy
+
 services:
   mongodb-dev:
     image: mongo:7
@@ -412,6 +433,12 @@ services:
 
   # DISCOURSE SERVICES - END
 
+  # AIRFLOW SERVICES - START
+
+
+
+  # AIRFLOW SERVICES - END
+
 volumes:
   mongodb_data_container:
   rmq_data_container:
@@ -423,6 +450,11 @@ volumes:
   neo4j_import:
   neo4j_plugins:
   hivemind_vector_store:
+  postgres_data:
+  airflow_config:
+  airflow_logs:
+  airflow_plugins:
+  airflow_sources:
 
 networks:
   development:

From 2bf0c02825daa84d46d10dee07565d4b4c7c1cc3 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 09:30:40 +0100
Subject: [PATCH 02/41] add pgvector

---
 development/docker-compose.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index f8abace..70f1573 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -25,7 +25,7 @@ x-airflow-common:
     &airflow-common-depends-on
     airflow-redis:
       condition: service_healthy
-    postgres:
+    pgvector:
       condition: service_healthy
     neo4j-dev:
       condition: service_healthy
@@ -145,6 +145,21 @@ services:
       - monitoring
     <<: *logging
 
+  pgvector:
+    image: ankane/pgvector
+    ports:
+      - 5432:5432
+    env_file:
+      ./.env.pgvector
+    volumes:
+      - pgvector_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "airflow"]
+      interval: 10s
+      retries: 5
+      start_period: 5s
+    restart: always
+
   api:
     build:
       context: ../../api
@@ -450,7 +465,7 @@ volumes:
   neo4j_import:
   neo4j_plugins:
   hivemind_vector_store:
-  postgres_data:
+  pgvector_data:
   airflow_config:
   airflow_logs:
   airflow_plugins:

From 1d9e41a2e03d5a4ac9dd4234e6e3a1439c4f12bf Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 09:43:58 +0100
Subject: [PATCH 03/41] Added airflow services

---
 development/docker-compose.yml | 134 +++++++++++++++++++++++++++++++++
 1 file changed, 134 insertions(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 70f1573..3182e1b 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -29,6 +29,21 @@ x-airflow-common:
       condition: service_healthy
     neo4j-dev:
       condition: service_healthy
+  networks:
+    - development
+    - monitoring
+
+x-redis-common:
+  &redis-common
+  image: redis:7.0.11
+  restart: unless-stopped
+  command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ]
+  healthcheck:
+    test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+    interval: 60s
+    timeout: 10s
+    retries: 2
+    start_period: 40s
 
 services:
   mongodb-dev:
@@ -450,7 +465,126 @@ services:
 
   # AIRFLOW SERVICES - START
 
+  airflow-redis:
+    <<: *redis-common
+    volumes:
+      - ./redis/airflow.conf:/usr/local/etc/redis/redis.conf:ro
+    networks:
+      - development
+      - monitoring
+    <<: *logging
+
+  airflow-webserver:
+    <<: *airflow-common
+    command: webserver
+    ports:
+      - "8080:8080"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+    depends_on:
+      <<: *airflow-common-depends-on
+      airflow-init:
+        condition: service_completed_successfully
+
+  airflow-scheduler:
+    <<: *airflow-common
+    command: scheduler
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+    depends_on:
+      <<: *airflow-common-depends-on
+      airflow-init:
+        condition: service_completed_successfully
+
+  airflow-worker:
+    <<: *airflow-common
+    command: celery worker
+    healthcheck:
+      # yamllint disable rule:line-length
+      test:
+        - "CMD-SHELL"
+        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    environment:
+      # Required to handle warm shutdown of the celery workers properly
+      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
+      DUMB_INIT_SETSID: "0"
+    restart: always
+    depends_on:
+      <<: *airflow-common-depends-on
+      airflow-init:
+        condition: service_completed_successfully
+
+  airflow-triggerer:
+    <<: *airflow-common
+    command: triggerer
+    healthcheck:
+      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+    depends_on:
+      <<: *airflow-common-depends-on
+      airflow-init:
+        condition: service_completed_successfully
+
+  airflow-init:
+    <<: *airflow-common
+    entrypoint: /bin/bash
+    command: /init.sh
+    env_file:
+      - ./.env.airflow
+      - ./.env.airflow.init
+    user: "0:0"
+    volumes:
+      - airflow_sources:/sources
+      - ./init.sh:/init.sh:ro
+
+  airflow-cli:
+    <<: *airflow-common
+    profiles:
+      - debug
+    environment:
+      CONNECTION_CHECK_MAX_COUNT: "0"
+    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
+    command: bash -c airflow
 
+  # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up
+  # or by explicitly targeted on the command line e.g. docker-compose up flower.
+  # See: https://docs.docker.com/compose/profiles/
+  flower:
+    <<: *airflow-common
+    command: celery flower
+    profiles:
+      - flower
+    ports:
+      - "5555:5555"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+    depends_on:
+      <<: *airflow-common-depends-on
+      airflow-init:
+        condition: service_completed_successfully
 
   # AIRFLOW SERVICES - END
 

From 41eab86c4058910238d8422e5b8c5c5a3b7b0094 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 09:50:08 +0100
Subject: [PATCH 04/41] added logging

---
 development/docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 3182e1b..de00d4c 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -32,6 +32,7 @@ x-airflow-common:
   networks:
     - development
     - monitoring
+  <<: *logging
 
 x-redis-common:
   &redis-common
@@ -44,6 +45,7 @@ x-redis-common:
     timeout: 10s
     retries: 2
     start_period: 40s
+  <<: *logging
 
 services:
   mongodb-dev:
@@ -472,7 +474,6 @@ services:
     networks:
       - development
       - monitoring
-    <<: *logging
 
   airflow-webserver:
     <<: *airflow-common

From bbff256e20a9f6f626dcfe2f3a5b2bf072f6d187 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:19:58 +0100
Subject: [PATCH 05/41] update pgvector healthcheck test

---
 development/docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index de00d4c..7ad6e73 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -171,7 +171,8 @@ services:
     volumes:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD", "pg_isready", "-U", "airflow"]
+      test: "pg_isready -U ${POSTGRES_USER}"
+      # test: ["CMD", "pg_isready", "-U", "airflow"]
       interval: 10s
       retries: 5
       start_period: 5s

From 72644ceab39d9c2f1ed5b09028471cad874a2ccb Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:23:09 +0100
Subject: [PATCH 06/41] remove old healthcheck for pgvector

---
 development/docker-compose.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 7ad6e73..41d3002 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -172,7 +172,6 @@ services:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
       test: "pg_isready -U ${POSTGRES_USER}"
-      # test: ["CMD", "pg_isready", "-U", "airflow"]
       interval: 10s
       retries: 5
       start_period: 5s

From 06b4e7c8ff4005ebcdd7a88c64dc522dc3feb58f Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:24:24 +0100
Subject: [PATCH 07/41] change default postgres port

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 41d3002..1458c19 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -165,7 +165,7 @@ services:
   pgvector:
     image: ankane/pgvector
     ports:
-      - 5432:5432
+      - 45432:5432
     env_file:
       ./.env.pgvector
     volumes:

From b98dae1ce0548617773b919026140e875daedb17 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:28:02 +0100
Subject: [PATCH 08/41] remove brackets around pgvector test

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 1458c19..0c6e437 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -171,7 +171,7 @@ services:
     volumes:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
-      test: "pg_isready -U ${POSTGRES_USER}"
+      test: "pg_isready -U $POSTGRES_USER"
       interval: 10s
       retries: 5
       start_period: 5s

From 5c677e2acf33a7b8ee87862d348854e6634f9dcd Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:37:47 +0100
Subject: [PATCH 09/41] remove user from pg_isready

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 0c6e437..a18a7bd 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -171,7 +171,7 @@ services:
     volumes:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
-      test: "pg_isready -U $POSTGRES_USER"
+      test: "pg_isready"
       interval: 10s
       retries: 5
       start_period: 5s

From fde96be3bb443c53719ae6eed502add4914b3eac Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:38:58 +0100
Subject: [PATCH 10/41] add POSTGRES_USER

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index a18a7bd..0c6e437 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -171,7 +171,7 @@ services:
     volumes:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
-      test: "pg_isready"
+      test: "pg_isready -U $POSTGRES_USER"
       interval: 10s
       retries: 5
       start_period: 5s

From f8e0307bc397122947afbab50f053e4c8ab51eb9 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:43:38 +0100
Subject: [PATCH 11/41] change airflow dir for init.sh

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 0c6e437..ae31be5 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -554,7 +554,7 @@ services:
     user: "0:0"
     volumes:
       - airflow_sources:/sources
-      - ./init.sh:/init.sh:ro
+      - ./airflow/init.sh:/init.sh:ro
 
   airflow-cli:
     <<: *airflow-common

From b42e22f0632276d6a03064e3c5f8d70c8fdf0a3b Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:47:28 +0100
Subject: [PATCH 12/41] comment out twitter services

---
 development/docker-compose.yml | 118 ++++++++++++++++-----------------
 1 file changed, 59 insertions(+), 59 deletions(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index ae31be5..fe8722f 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -282,65 +282,65 @@ services:
       - monitoring
     <<: *logging
 
-  redis-twitter-bot:
-    image: redis:7.0.11
-    restart: unless-stopped
-    command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ]
-    volumes:
-      - type: bind
-        source: ./redis/twitter.conf
-        target: /usr/local/etc/redis/redis.conf
-        read_only: true
-    healthcheck:
-      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
-      interval: 60s
-      timeout: 10s
-      retries: 2
-      start_period: 40s
-    networks:
-      - development
-      - monitoring
-    <<: *logging
-
-  twitter-bot-server:
-    image: ghcr.io/togethercrew/twitter-bot:main
-    command: python3 server.py
-    restart: unless-stopped
-    env_file:
-      - ./.env.twitter
-    depends_on:
-      mongodb-dev:
-        condition: service_healthy
-      neo4j-dev:
-        condition: service_healthy
-      rabbitmq-dev: 
-        condition: service_healthy
-      redis-twitter-bot:
-        condition: service_healthy
-    networks:
-      - development
-      - monitoring
-    <<: *logging
-
-  twitter-bot-worker:
-    image: ghcr.io/togethercrew/twitter-bot:main
-    command: python3 worker.py
-    restart: unless-stopped
-    env_file:
-      - ./.env.twitter
-    depends_on:
-      mongodb-dev:
-        condition: service_healthy
-      neo4j-dev:
-        condition: service_healthy
-      rabbitmq-dev: 
-        condition: service_healthy
-      redis-twitter-bot:
-        condition: service_healthy
-    networks:
-      - development
-      - monitoring
-    <<: *logging
+  # redis-twitter-bot:
+  #   image: redis:7.0.11
+  #   restart: unless-stopped
+  #   command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ]
+  #   volumes:
+  #     - type: bind
+  #       source: ./redis/twitter.conf
+  #       target: /usr/local/etc/redis/redis.conf
+  #       read_only: true
+  #   healthcheck:
+  #     test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+  #     interval: 60s
+  #     timeout: 10s
+  #     retries: 2
+  #     start_period: 40s
+  #   networks:
+  #     - development
+  #     - monitoring
+  #   <<: *logging
+
+  # twitter-bot-server:
+  #   image: ghcr.io/togethercrew/twitter-bot:main
+  #   command: python3 server.py
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ./.env.twitter
+  #   depends_on:
+  #     mongodb-dev:
+  #       condition: service_healthy
+  #     neo4j-dev:
+  #       condition: service_healthy
+  #     rabbitmq-dev: 
+  #       condition: service_healthy
+  #     redis-twitter-bot:
+  #       condition: service_healthy
+  #   networks:
+  #     - development
+  #     - monitoring
+  #   <<: *logging
+
+  # twitter-bot-worker:
+  #   image: ghcr.io/togethercrew/twitter-bot:main
+  #   command: python3 worker.py
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ./.env.twitter
+  #   depends_on:
+  #     mongodb-dev:
+  #       condition: service_healthy
+  #     neo4j-dev:
+  #       condition: service_healthy
+  #     rabbitmq-dev: 
+  #       condition: service_healthy
+  #     redis-twitter-bot:
+  #       condition: service_healthy
+  #   networks:
+  #     - development
+  #     - monitoring
+  #   <<: *logging
 
   # HIVEMIND SERVICES - START
   hivemind-vector-server:

From b95880e9041a6a339c4d81b781e14f570bdbdecd Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:47:52 +0100
Subject: [PATCH 13/41] remove all hivemind services

---
 development/docker-compose.yml | 74 ----------------------------------
 1 file changed, 74 deletions(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index fe8722f..82bdf77 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -343,80 +343,6 @@ services:
   #   <<: *logging
 
   # HIVEMIND SERVICES - START
-  hivemind-vector-server:
-    image: ghcr.io/togethercrew/qabot:main-vector_server
-    depends_on:
-      rabbitmq-dev: 
-        condition: service_healthy
-      hivemind-vector-redis:
-        condition: service_healthy
-      hivemind-vector-worker:
-        condition: service_healthy
-    env_file:
-      - .env.hivemind.vector.server
-    networks:
-      - hivemind
-      - development
-      - monitoring
-    <<: *logging
-    ports:
-      - 41234:1234
-
-  hivemind-vector-worker:
-    image: ghcr.io/togethercrew/qabot:main-vector_server
-    env_file:
-      - .env.hivemind.vector.worker
-    depends_on:
-      hivemind-vector-redis:
-        condition: service_healthy
-    command: celery -A tasks.celery worker -c 1 --hostname hivemind-vector-worker --loglevel=info
-    volumes:
-      - hivemind_vector_store:/project/vector_store
-    healthcheck:
-      test: ["CMD-SHELL", "celery -A tasks.celery inspect ping"]
-      interval: 10s
-      timeout: 10s
-      retries: 2
-      start_period: 10s
-    networks:
-      - hivemind
-      - monitoring
-    <<: *logging
-
-  hivemind-vector-redis:
-    image: redis:7.0.11
-    restart: unless-stopped
-    command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ]
-    volumes:
-      - type: bind
-        source: ./redis/hivemind.vector.conf
-        target: /usr/local/etc/redis/redis.conf
-        read_only: true
-    healthcheck:
-      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
-      interval: 60s
-      timeout: 10s
-      retries: 2
-      start_period: 40s
-    networks:
-      - hivemind
-      - monitoring
-    <<: *logging
-
-  hivemind-api:
-    image: ghcr.io/togethercrew/qabot:main-ml
-    env_file:
-      - .env.hivemind.api
-    depends_on:
-      rabbitmq-dev: 
-        condition: service_healthy
-    networks:
-      - hivemind
-      - development
-      - monitoring
-    <<: *logging
-    ports:
-      - 43333:3333
   
   # HIVEMIND SERVICES - END
 

From a19eb7d44419224293fca059a75d4f8037ce51ef Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:51:48 +0100
Subject: [PATCH 14/41] pg_isready -U $$POSTGRES_USER

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 82bdf77..00973fe 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -171,7 +171,7 @@ services:
     volumes:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
-      test: "pg_isready -U $POSTGRES_USER"
+      test: pg_isready -U $$POSTGRES_USER
       interval: 10s
       retries: 5
       start_period: 5s

From c7309a2e86baa3582ccaa2715e025949eff5e37b Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 11:01:40 +0100
Subject: [PATCH 15/41] remove init.sh from airflow-init volumes

---
 development/docker-compose.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 00973fe..d5d4cf3 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -480,7 +480,6 @@ services:
     user: "0:0"
     volumes:
       - airflow_sources:/sources
-      - ./airflow/init.sh:/init.sh:ro
 
   airflow-cli:
     <<: *airflow-common

From f677aa645d7e9639773818ee16feb01276ebd429 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 11:44:44 +0100
Subject: [PATCH 16/41] add pg to networks + logs

---
 development/docker-compose.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index d5d4cf3..8c510b0 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -176,6 +176,10 @@ services:
       retries: 5
       start_period: 5s
     restart: always
+    networks:
+      - development
+      - monitoring
+    <<: *logging
 
   api:
     build:

From 9106bbe1845155f612416b2b7219207b236716cf Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 11:45:29 +0100
Subject: [PATCH 17/41] update airflow-init command

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 8c510b0..828f678 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -477,7 +477,7 @@ services:
   airflow-init:
     <<: *airflow-common
     entrypoint: /bin/bash
-    command: /init.sh
+    command: -c "./init.sh"
     env_file:
       - ./.env.airflow
       - ./.env.airflow.init

From e1d4221b7393a2303c3968d8f7ea251893874239 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 12:49:55 +0100
Subject: [PATCH 18/41] add database to healthcheck pgvector

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 828f678..985ff71 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -171,7 +171,7 @@ services:
     volumes:
       - pgvector_data:/var/lib/postgresql/data
     healthcheck:
-      test: pg_isready -U $$POSTGRES_USER
+      test: pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB
       interval: 10s
       retries: 5
       start_period: 5s

From 5fb10c7f98de60904346a72a3cd992e89323c657 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 15:38:44 +0100
Subject: [PATCH 19/41] hivemind-bot service

---
 development/docker-compose.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 985ff71..0a043d2 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -516,6 +516,18 @@ services:
       airflow-init:
         condition: service_completed_successfully
 
+  hivemind-bot:
+    image: ghcr.io/togethercrew/hivemind-bot:main
+    env_file:
+      - ./.env.hivemind-bot
+    depends_on:
+      neo4j-dev:
+        condition: service_healthy
+      mongo-dev:
+        condition: service_healthy
+      pgvector:
+        condition: service_healthy
+
   # AIRFLOW SERVICES - END
 
 volumes:

From 588257d4edf93f21a74d4fdb364cb26056e7c0e5 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 15:39:25 +0100
Subject: [PATCH 20/41] fix mongodb name

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 0a043d2..b0d514f 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -523,7 +523,7 @@ services:
     depends_on:
       neo4j-dev:
         condition: service_healthy
-      mongo-dev:
+      mongodb-dev:
         condition: service_healthy
       pgvector:
         condition: service_healthy

From 3954a424ad540794c9ee28a7f758884c6b49a7a8 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 15:45:11 +0100
Subject: [PATCH 21/41] add networks + logging to hivemind-bot

---
 development/docker-compose.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index b0d514f..609dff0 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -527,6 +527,10 @@ services:
         condition: service_healthy
       pgvector:
         condition: service_healthy
+    networks:
+      - discourse
+      - monitoring
+    <<: *logging
 
   # AIRFLOW SERVICES - END
 

From 1301adedf4d306e4e0818a1b8462e7cd74a3587f Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 15:47:12 +0100
Subject: [PATCH 22/41] hivemind-bot restart: unless-stopped

---
 development/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 609dff0..ffaec79 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -518,6 +518,7 @@ services:
 
   hivemind-bot:
     image: ghcr.io/togethercrew/hivemind-bot:main
+    restart: unless-stopped
     env_file:
       - ./.env.hivemind-bot
     depends_on:

From 6eed83900e31065525e7ea62fc434ba01039e707 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 15:53:37 +0100
Subject: [PATCH 23/41] change hivemind-bot network to development

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index ffaec79..c2d0a70 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -529,7 +529,7 @@ services:
       pgvector:
         condition: service_healthy
     networks:
-      - discourse
+      - development
       - monitoring
     <<: *logging
 

From 0969836b34b283a02ec2337600746a9675a9f8bc Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 16:13:08 +0100
Subject: [PATCH 24/41] expose airflow dashboard

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index c2d0a70..93381c0 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -409,7 +409,7 @@ services:
     <<: *airflow-common
     command: webserver
     ports:
-      - "8080:8080"
+      - 48080:8080
     healthcheck:
       test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
       interval: 30s

From 2df8da406f8f8a4ba9f7d5c5d58fdb569459a3ed Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 16:33:27 +0100
Subject: [PATCH 25/41] remove dags from volumes

---
 development/docker-compose.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 93381c0..9c96a20 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -16,7 +16,6 @@ x-airflow-common:
     &airflow-common-env
     - ./.env.airflow
   volumes:
-    - ./dags:/opt/airflow/dags:ro
     - airflow_logs:/opt/airflow/logs
     - airflow_config:/opt/airflow/config
     - airflow_plugins:/opt/airflow/plugins

From 06ce223047d6efdb3b6466825f338f29e9adf39a Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 16:39:28 +0100
Subject: [PATCH 26/41] add hivemind server + worker

---
 development/docker-compose.yml | 40 ++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 9c96a20..351d848 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -46,6 +46,24 @@ x-redis-common:
     start_period: 40s
   <<: *logging
 
+x-hivemind-common:
+  &hivemind-common
+  image: ghcr.io/togethercrew/hivemind-bot:main
+  restart: unless-stopped
+  env_file:
+    - ./.env.hivemind-bot
+  depends_on:
+    neo4j-dev:
+      condition: service_healthy
+    mongodb-dev:
+      condition: service_healthy
+    pgvector:
+      condition: service_healthy
+  networks:
+    - development
+    - monitoring
+  <<: *logging
+
 services:
   mongodb-dev:
     image: mongo:7
@@ -515,22 +533,12 @@ services:
       airflow-init:
         condition: service_completed_successfully
 
-  hivemind-bot:
-    image: ghcr.io/togethercrew/hivemind-bot:main
-    restart: unless-stopped
-    env_file:
-      - ./.env.hivemind-bot
-    depends_on:
-      neo4j-dev:
-        condition: service_healthy
-      mongodb-dev:
-        condition: service_healthy
-      pgvector:
-        condition: service_healthy
-    networks:
-      - development
-      - monitoring
-    <<: *logging
+  hivemind-server:
+    <<: *hivemind-common
+
+  hivemind-worker:
+    <<: *hivemind-common
+    command: python3 worker.py
 
   # AIRFLOW SERVICES - END
 

From db363af6091c017dc53a99768ec40922a53a3dba Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Wed, 17 Jan 2024 17:00:57 +0100
Subject: [PATCH 27/41] rabbitmq enabled_plugins

---
 development/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 351d848..a65c50f 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -96,6 +96,7 @@ services:
       - 25672:15672
     volumes:
       - rmq_data_container:/var/lib/rabbitmq/
+      - ./rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins
     env_file:
       - ./.env.rmq
     healthcheck:

From c867cd1953814e6fda0e921406d5fc9fdfa3eeb2 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 08:20:22 +0100
Subject: [PATCH 28/41] move monitoring into docker-compose

---
 development/docker-compose.monitoring.yml | 109 ----------------------
 development/docker-compose.yml            |  97 ++++++++++++++++++-
 2 files changed, 94 insertions(+), 112 deletions(-)
 delete mode 100644 development/docker-compose.monitoring.yml

diff --git a/development/docker-compose.monitoring.yml b/development/docker-compose.monitoring.yml
deleted file mode 100644
index 85c3efd..0000000
--- a/development/docker-compose.monitoring.yml
+++ /dev/null
@@ -1,109 +0,0 @@
-version: '3.9'
-
-networks:
-  monitoring:
-    external: true
-    name: monitoring
-    ipam:
-      config:
-        - subnet: 172.22.22.0/16
-
-x-logging:
-  &logging
-  logging:
-    driver: loki
-    options:
-      loki-url: http://172.22.22.15:3100/loki/api/v1/push
-      loki-retries: 5
-      loki-batch-size: 400
-
-volumes:
-  grafana_volume:
-  prometheus_volume:
-  loki_volume:
-
-services:
-  grafana:
-    image: grafana/grafana
-    container_name: grafana
-    restart: always
-    volumes:
-      - ./grafana/provisioning/:/etc/grafana/provisioning
-      - grafana_volume:/var/lib/grafana
-    depends_on:
-      - prometheus
-      - loki
-    ports:
-      - 3000:3000
-    networks:
-      - monitoring
-    <<: *logging
-
-  prometheus:
-    image: prom/prometheus:latest
-    container_name: prometheus
-    command:
-      - --config.file=/etc/prometheus/prometheus.yml
-    volumes:
-      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - prometheus_volume:/prometheus
-    depends_on:
-      - cadvisor
-      - node-exporter
-    networks:
-      - monitoring
-    restart: unless-stopped
-    <<: *logging
-
-  cadvisor:
-    image: gcr.io/cadvisor/cadvisor:latest
-    container_name: cadvisor
-    volumes:
-      - /:/rootfs:ro
-      - /var/run:/var/run:rw
-      - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker:ro
-    depends_on:
-      - redis-cadvisor
-    networks:
-      - monitoring
-    <<: *logging
-
-  redis-cadvisor:
-    image: redis:latest
-    container_name: redis-cadvisor
-    networks:
-      - monitoring
-    <<: *logging
-
-  loki:
-    container_name: loki
-    image: grafana/loki:2.8.0
-    restart: unless-stopped
-    ports:
-      - 3100
-    volumes:
-      - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml
-      - loki_volume:/data/loki
-    command: -config.file=/etc/loki/loki-config.yaml
-    networks:
-      monitoring:
-        ipv4_address: 172.22.22.15
-    <<: *logging
-
-  node-exporter:
-    image: prom/node-exporter:latest
-    container_name: node-exporter
-    restart: unless-stopped
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    command:
-      - '--path.procfs=/host/proc'
-      - '--path.rootfs=/rootfs'
-      - '--path.sysfs=/host/sys'
-      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
-    networks:
-      - monitoring
-    <<: *logging
diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index a65c50f..19a0409 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -543,6 +543,95 @@ services:
 
   # AIRFLOW SERVICES - END
 
+  # MONITORING SERVICES - START
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: always
+    volumes:
+      - ./grafana/provisioning/:/etc/grafana/provisioning
+      - grafana_volume:/var/lib/grafana
+    depends_on:
+      - prometheus
+      - loki
+    ports:
+      - 3000:3000
+    networks:
+      - monitoring
+    <<: *logging
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_volume:/prometheus
+    depends_on:
+      - cadvisor
+      - node-exporter
+    networks:
+      - monitoring
+    restart: unless-stopped
+    <<: *logging
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    depends_on:
+      - redis-cadvisor
+    networks:
+      - monitoring
+    <<: *logging
+
+  redis-cadvisor:
+    image: redis:latest
+    container_name: redis-cadvisor
+    networks:
+      - monitoring
+    <<: *logging
+
+  loki:
+    container_name: loki
+    image: grafana/loki:2.8.0
+    restart: unless-stopped
+    ports:
+      - 3100
+    volumes:
+      - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml
+      - loki_volume:/data/loki
+    command: -config.file=/etc/loki/loki-config.yaml
+    networks:
+      monitoring:
+        ipv4_address: 172.22.22.15
+    <<: *logging
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: node-exporter
+    restart: unless-stopped
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+    networks:
+      - monitoring
+    <<: *logging
+
+  #MONITORING SERVICES - END
+
 volumes:
   mongodb_data_container:
   rmq_data_container:
@@ -563,10 +652,12 @@ volumes:
 networks:
   development:
     driver: bridge
-  monitoring:
-    driver: bridge
-    external: true
   hivemind:
     driver: bridge
   discourse:
     driver: bridge
+  monitoring:
+    name: monitoring
+    ipam:
+      config:
+        - subnet: 172.22.22.0/16
\ No newline at end of file

From aeb06cf4ce59da60ab8ac6bb61c56300b607dd39 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:21:22 +0100
Subject: [PATCH 29/41] monitoring network

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 19a0409..116def2 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -657,7 +657,7 @@ networks:
   discourse:
     driver: bridge
   monitoring:
-    name: monitoring
+    driver: bridge
     ipam:
       config:
         - subnet: 172.22.22.0/16
\ No newline at end of file

From e689fbd8a933c9e073f44c7b533d52edd901860a Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:22:26 +0100
Subject: [PATCH 30/41] netowrk monitoring

---
 development/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 116def2..78ed313 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -657,6 +657,7 @@ networks:
   discourse:
     driver: bridge
   monitoring:
+    name: monitoring
     driver: bridge
     ipam:
       config:

From a6ba9f201a767bf36acd5f59f669921da6523e00 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:23:34 +0100
Subject: [PATCH 31/41] development

---
 development/docker-compose.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 78ed313..0c250b9 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -609,7 +609,7 @@ services:
       - loki_volume:/data/loki
     command: -config.file=/etc/loki/loki-config.yaml
     networks:
-      monitoring:
+      development_monitoring:
         ipv4_address: 172.22.22.15
     <<: *logging
 
@@ -657,7 +657,6 @@ networks:
   discourse:
     driver: bridge
   monitoring:
-    name: monitoring
     driver: bridge
     ipam:
       config:

From 7b1416c08202c31d456d4520de09e45f2df83540 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 09:24:59 +0100
Subject: [PATCH 32/41] network

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 0c250b9..116def2 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -609,7 +609,7 @@ services:
       - loki_volume:/data/loki
     command: -config.file=/etc/loki/loki-config.yaml
     networks:
-      development_monitoring:
+      monitoring:
         ipv4_address: 172.22.22.15
     <<: *logging
 

From 27635cd4f00b0d288b4133485c1512caad701b4c Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 17:56:08 +0100
Subject: [PATCH 33/41] add tempo

---
 development/docker-compose.yml                | 18 ++++++
 .../provisioning/datasources/tempo.yml        | 24 ++++++++
 development/prometheus/prometheus.yml         |  7 ++-
 development/tempo/tempo.yaml                  | 57 +++++++++++++++++++
 4 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 development/grafana/provisioning/datasources/tempo.yml
 create mode 100644 development/tempo/tempo.yaml

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 116def2..32d8ccc 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -630,6 +630,24 @@ services:
       - monitoring
     <<: *logging
 
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo/tempo.yaml:/etc/tempo.yaml
+      - ./tempo-data/:/tmp/tempo
+    ports:
+      - "14268:14268"  # jaeger ingest
+      - "3200:3200"   # tempo
+      - "9095:9095" # tempo grpc
+      - "4317:4317"  # otlp grpc
+      - "4318:4318"  # otlp http
+      - "9411:9411"   # zipkin
+    networks:
+      - monitoring
+      - development
+    <<: *logging
+
   #MONITORING SERVICES - END
 
 volumes:
diff --git a/development/grafana/provisioning/datasources/tempo.yml b/development/grafana/provisioning/datasources/tempo.yml
new file mode 100644
index 0000000..dfc2b56
--- /dev/null
+++ b/development/grafana/provisioning/datasources/tempo.yml
@@ -0,0 +1,24 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Tempo
+    orgId: 1
+
+datasources:
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: false
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus
\ No newline at end of file
diff --git a/development/prometheus/prometheus.yml b/development/prometheus/prometheus.yml
index f259b88..f764121 100644
--- a/development/prometheus/prometheus.yml
+++ b/development/prometheus/prometheus.yml
@@ -15,4 +15,9 @@ scrape_configs:
 - job_name: traefik
   static_configs:
     - targets:
-      - traefik:8080
\ No newline at end of file
+      - traefik:8080
+
+- job_name: tempo
+    static_configs:
+      - targets:
+        - tempo:3200
\ No newline at end of file
diff --git a/development/tempo/tempo.yaml b/development/tempo/tempo.yaml
new file mode 100644
index 0000000..bce8a02
--- /dev/null
+++ b/development/tempo/tempo.yaml
@@ -0,0 +1,57 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator
\ No newline at end of file

From 652796b6993872eccbf5ae135f2ea970914420a1 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 18:03:56 +0100
Subject: [PATCH 34/41] fix prometheus config

---
 development/prometheus/prometheus.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/development/prometheus/prometheus.yml b/development/prometheus/prometheus.yml
index f764121..a96a168 100644
--- a/development/prometheus/prometheus.yml
+++ b/development/prometheus/prometheus.yml
@@ -18,6 +18,6 @@ scrape_configs:
       - traefik:8080
 
 - job_name: tempo
-    static_configs:
-      - targets:
-        - tempo:3200
\ No newline at end of file
+  static_configs:
+    - targets:
+      - tempo:3200
\ No newline at end of file

From e9cba116a82dcf6a2c45656f7c8e256c90849d61 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 18:53:11 +0100
Subject: [PATCH 35/41] Update tempo.yml

---
 development/grafana/provisioning/datasources/tempo.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/development/grafana/provisioning/datasources/tempo.yml b/development/grafana/provisioning/datasources/tempo.yml
index dfc2b56..bd2a079 100644
--- a/development/grafana/provisioning/datasources/tempo.yml
+++ b/development/grafana/provisioning/datasources/tempo.yml
@@ -13,7 +13,7 @@ datasources:
   orgId: 1
   url: http://tempo:3200
   basicAuth: false
-  isDefault: true
+  isDefault: false
   version: 1
   editable: false
   apiVersion: 1
@@ -21,4 +21,4 @@ datasources:
   jsonData:
     httpMethod: GET
     serviceMap:
-      datasourceUid: prometheus
\ No newline at end of file
+      datasourceUid: prometheus

From 22add630f70c62efc56e1e3e47500223b20aee18 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Thu, 18 Jan 2024 18:56:45 +0100
Subject: [PATCH 36/41] Update tempo.yml

---
 development/grafana/provisioning/datasources/tempo.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/grafana/provisioning/datasources/tempo.yml b/development/grafana/provisioning/datasources/tempo.yml
index bd2a079..e45a5bd 100644
--- a/development/grafana/provisioning/datasources/tempo.yml
+++ b/development/grafana/provisioning/datasources/tempo.yml
@@ -21,4 +21,4 @@ datasources:
   jsonData:
     httpMethod: GET
     serviceMap:
-      datasourceUid: prometheus
+      datasourceUid: Prometheus

From bc5e2af3f4b51850bc591e9a6861a0509f0d88da Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Mon, 22 Jan 2024 09:20:43 +0100
Subject: [PATCH 37/41] otel collector

---
 development/docker-compose.yml                | 37 +++++++++++++++----
 .../otel-collector/otel-collector.yaml        | 14 +++++++
 2 files changed, 44 insertions(+), 7 deletions(-)
 create mode 100644 development/otel-collector/otel-collector.yaml

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 32d8ccc..f10988f 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -630,19 +630,42 @@ services:
       - monitoring
     <<: *logging
 
+  otel-collector:
+    image: otel/opentelemetry-collector:0.92.0
+    command: [ "--config=/etc/otel-collector.yaml" ]
+    volumes:
+      - ./otel-collector/otel-collector.yaml:/etc/otel-collector.yaml
+    networks:
+      - monitoring
+      - development
+    <<: *logging
+
   tempo:
     image: grafana/tempo:latest
     command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo/tempo.yaml:/etc/tempo.yaml
       - ./tempo-data/:/tmp/tempo
-    ports:
-      - "14268:14268"  # jaeger ingest
-      - "3200:3200"   # tempo
-      - "9095:9095" # tempo grpc
-      - "4317:4317"  # otlp grpc
-      - "4318:4318"  # otlp http
-      - "9411:9411"   # zipkin
+    # ports:
+      # - "14268:14268"  # jaeger ingest
+      # - "3200:3200"   # tempo
+      # - "9095:9095" # tempo grpc
+      # - "4317:4317"  # otlp grpc
+      # - "4318:4318"  # otlp http
+      # - "9411:9411"   # zipkin
+    networks:
+      - monitoring
+      - development
+    <<: *logging
+
+  # Generate fake traces...
+  k6-tracing:
+    image: ghcr.io/grafana/xk6-client-tracing:v0.0.2
+    environment:
+      - ENDPOINT=otel-collector:4317
+    restart: always
+    depends_on:
+      - otel-collector
     networks:
       - monitoring
       - development
diff --git a/development/otel-collector/otel-collector.yaml b/development/otel-collector/otel-collector.yaml
new file mode 100644
index 0000000..870609d
--- /dev/null
+++ b/development/otel-collector/otel-collector.yaml
@@ -0,0 +1,14 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+exporters:
+  otlp:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]
\ No newline at end of file

From bd4d043dcdad1c19cf9be895a1fe04825218dad1 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Mon, 22 Jan 2024 09:26:40 +0100
Subject: [PATCH 38/41] open otel port

---
 development/docker-compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index f10988f..30624d6 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -638,6 +638,8 @@ services:
     networks:
       - monitoring
       - development
+    ports:
+      - 44317:4317
     <<: *logging
 
   tempo:

From c86f71715c5b2d1b3d4781921ffaf64b2b680c77 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Mon, 22 Jan 2024 10:33:49 +0100
Subject: [PATCH 39/41] otel config

---
 development/docker-compose.yml                 | 13 ++++---------
 development/otel-collector/otel-collector.yaml |  1 +
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 30624d6..34f4803 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -53,6 +53,7 @@ x-hivemind-common:
   env_file:
     - ./.env.hivemind-bot
   depends_on:
+    &hivemind-common-depends-on
     neo4j-dev:
       condition: service_healthy
     mongodb-dev:
@@ -536,6 +537,9 @@ services:
 
   hivemind-server:
     <<: *hivemind-common
+    depends_on:
+      <<: *hivemind-common-depends-on
+      - otel-collector
 
   hivemind-worker:
     <<: *hivemind-common
@@ -638,8 +642,6 @@ services:
     networks:
       - monitoring
       - development
-    ports:
-      - 44317:4317
     <<: *logging
 
   tempo:
@@ -648,13 +650,6 @@ services:
     volumes:
       - ./tempo/tempo.yaml:/etc/tempo.yaml
       - ./tempo-data/:/tmp/tempo
-    # ports:
-      # - "14268:14268"  # jaeger ingest
-      # - "3200:3200"   # tempo
-      # - "9095:9095" # tempo grpc
-      # - "4317:4317"  # otlp grpc
-      # - "4318:4318"  # otlp http
-      # - "9411:9411"   # zipkin
     networks:
       - monitoring
       - development
diff --git a/development/otel-collector/otel-collector.yaml b/development/otel-collector/otel-collector.yaml
index 870609d..edc1cdf 100644
--- a/development/otel-collector/otel-collector.yaml
+++ b/development/otel-collector/otel-collector.yaml
@@ -2,6 +2,7 @@ receivers:
   otlp:
     protocols:
       grpc:
+      http:
 exporters:
   otlp:
     endpoint: tempo:4317

From 6a9032ca424e42e6baf8e48ab79c67c5b246be82 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Mon, 22 Jan 2024 10:35:08 +0100
Subject: [PATCH 40/41] fix

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 34f4803..99eb3d4 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -539,7 +539,7 @@ services:
     <<: *hivemind-common
     depends_on:
       <<: *hivemind-common-depends-on
-      - otel-collector
+      otel-collector:
 
   hivemind-worker:
     <<: *hivemind-common

From 0ebab57d3d013d94611d285bf52eb0eeee1be858 Mon Sep 17 00:00:00 2001
From: Cyrille <1618640+cyri113@users.noreply.github.com>
Date: Mon, 22 Jan 2024 10:35:38 +0100
Subject: [PATCH 41/41] fix

---
 development/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/development/docker-compose.yml b/development/docker-compose.yml
index 99eb3d4..f8e3cdc 100644
--- a/development/docker-compose.yml
+++ b/development/docker-compose.yml
@@ -539,7 +539,7 @@ services:
     <<: *hivemind-common
     depends_on:
       <<: *hivemind-common-depends-on
-      otel-collector:
+      # otel-collector:
 
   hivemind-worker:
     <<: *hivemind-common