From b07465532a6cbdbc5a43069fbfab2635a7b2e9f0 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:02:12 +0100 Subject: [PATCH 01/16] Add hivemind --- development/.env.hivemind.api.sample | 12 ++++ .../.env.hivemind.vector.server.sample | 11 +++ .../.env.hivemind.vector.worker.sample | 11 +++ development/docker-compose.yml | 72 ++++++++++++++++++- development/redis/hivemind.vector.conf.sample | 0 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 development/.env.hivemind.api.sample create mode 100644 development/.env.hivemind.vector.server.sample create mode 100644 development/.env.hivemind.vector.worker.sample create mode 100644 development/redis/hivemind.vector.conf.sample diff --git a/development/.env.hivemind.api.sample b/development/.env.hivemind.api.sample new file mode 100644 index 0000000..d34d4b5 --- /dev/null +++ b/development/.env.hivemind.api.sample @@ -0,0 +1,12 @@ +#OpenAI CONFIG +OPENAI_API_KEY= +OPENAI_API_MODEL= + +# AGENT CONFIG +AGENT_NAME= +AGENT_ROLE= + +# BROKER EVENTS CONFIG +RABBITMQ_HOST= +RABBITMQ_USER= +RABBITMQ_PASS= \ No newline at end of file diff --git a/development/.env.hivemind.vector.server.sample b/development/.env.hivemind.vector.server.sample new file mode 100644 index 0000000..d32c32d --- /dev/null +++ b/development/.env.hivemind.vector.server.sample @@ -0,0 +1,11 @@ +OPENAI_API_KEY= +OPENAI_API_MODEL= + +DB_CONNECTION_STR= +DB_GUILD= + +REDIS_HOST= + +RABBITMQ_HOST= +RABBITMQ_USER= +RABBITMQ_PASS= \ No newline at end of file diff --git a/development/.env.hivemind.vector.worker.sample b/development/.env.hivemind.vector.worker.sample new file mode 100644 index 0000000..d32c32d --- /dev/null +++ b/development/.env.hivemind.vector.worker.sample @@ -0,0 +1,11 @@ +OPENAI_API_KEY= +OPENAI_API_MODEL= + +DB_CONNECTION_STR= +DB_GUILD= + +REDIS_HOST= + +RABBITMQ_HOST= +RABBITMQ_USER= +RABBITMQ_PASS= \ No newline at end of file diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 1a517e8..39658cc 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -268,7 +268,77 @@ services: - development - monitoring <<: *logging - + + # HIVEMIND SERVICES - START + + hivemind-vector-server: + image: ghcr.io/kamikazebr/qabot:main-vector_server + depends_on: + rabbitmq-dev: + condition: service_healthy + hivemind-vector-redis: + condition: service_healthy + hivemind-vector-worker: + condition: service_healthy + env_file: + - .env.hivemind.vector.server + networks: + - hivemind + - development + - monitoring + <<: *logging + + hivemind-vector-worker: + image: ghcr.io/kamikazebr/qabot:main-vector_server + env_file: + - .env.hivemind.vector.worker + depends_on: + hivemind-vector-redis: + condition: service_healthy + command: celery -A tasks.celery worker -c 1 --hostname celery --loglevel=info + healthcheck: + test: ["CMD", "celery", "inspect", "ping", "-A", "tasks.add", "-d" "celery@celery"] + interval: 60s + timeout: 10s + retries: 2 + start_period: 10s + networks: + - hivemind + - monitoring + <<: *logging + + hivemind-vector-redis: + image: redis:7.0.11 + restart: unless-stopped + command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ] + volumes: + - type: bind + source: ./redis/hivemind.vector.conf + target: /usr/local/etc/redis/redis.conf + read_only: true + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 60s + timeout: 10s + retries: 2 + start_period: 40s + networks: + - hivemind + - monitoring + <<: *logging + + hivemind-api: + image: ghcr.io/kamikazebr/qabot:main-ml + env_file: + - .env.hivemind.api + networks: + - hivemind + - development + - monitoring + <<: *logging + + # HIVEMIND SERVICES - END + volumes: mongodb_data_container: diff --git a/development/redis/hivemind.vector.conf.sample b/development/redis/hivemind.vector.conf.sample new file mode 100644 index 0000000..e69de29 From c746b5eb899759798b5b2dec3c23d460c6ed42ba Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:07:39 +0100 Subject: [PATCH 02/16] Fix comma --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 39658cc..c5278d2 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -297,7 +297,7 @@ services: condition: service_healthy command: celery -A tasks.celery worker -c 1 --hostname celery --loglevel=info healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "tasks.add", "-d" "celery@celery"] + test: ["CMD", "celery", "inspect", "ping", "-A", "tasks.add", "-d", "celery@celery"] interval: 60s timeout: 10s retries: 2 From fe5971fe35bf5252fd1207d64dcaecfdd3a00c6f Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:08:30 +0100 Subject: [PATCH 03/16] Add hivemind network --- development/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index c5278d2..75be0d9 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -357,3 +357,5 @@ networks: monitoring: driver: bridge external: true + hivemind: + driver: bridge From c15b2fc9d4196415f87380c131a0af10ed63645c Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:16:54 +0100 Subject: [PATCH 04/16] add redis conf --- development/redis/hivemind.vector.conf.sample | 1 + 1 file changed, 1 insertion(+) diff --git a/development/redis/hivemind.vector.conf.sample b/development/redis/hivemind.vector.conf.sample index e69de29..6cac345 100644 --- a/development/redis/hivemind.vector.conf.sample +++ b/development/redis/hivemind.vector.conf.sample @@ -0,0 +1 @@ +requirepass PASSWORD \ No newline at end of file From d5f7e40e5e8292a2e62e08786e8ef3565d1369b0 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:21:57 +0100 Subject: [PATCH 05/16] add hivemind volume --- development/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 75be0d9..926fd67 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -296,6 +296,8 @@ services: hivemind-vector-redis: condition: service_healthy command: celery -A tasks.celery worker -c 1 --hostname celery --loglevel=info + volumes: + - hivemind_vector_store::/project/vector_store healthcheck: test: ["CMD", "celery", "inspect", "ping", "-A", "tasks.add", "-d", "celery@celery"] interval: 60s @@ -350,6 +352,7 @@ volumes: neo4j_data: neo4j_import: neo4j_plugins: + hivemind_vector_store: networks: development: From 8b89da117212b3d2f7ef566af407eddf998803c6 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:27:27 +0100 Subject: [PATCH 06/16] fix colon --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 926fd67..eeac5f9 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -297,7 +297,7 @@ services: condition: service_healthy command: celery -A tasks.celery worker -c 1 --hostname celery --loglevel=info volumes: - - hivemind_vector_store::/project/vector_store + - hivemind_vector_store:/project/vector_store healthcheck: test: ["CMD", "celery", "inspect", "ping", "-A", "tasks.add", "-d", "celery@celery"] interval: 60s From 8ac53fa276be04c463845c0f8617c131084a6062 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:38:36 +0100 Subject: [PATCH 07/16] hindmind worker healthcheck --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index eeac5f9..5dbb976 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -299,7 +299,7 @@ services: volumes: - hivemind_vector_store:/project/vector_store healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "tasks.add", "-d", "celery@celery"] + test: ["CMD", "celery", "inspect", "ping", "-d", "celery@$$HOSTNAME"] interval: 60s timeout: 10s retries: 2 From 2195c2c0996fabd48814e3f37eee374ec9905b23 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:44:02 +0100 Subject: [PATCH 08/16] celery@celery --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 5dbb976..18e7a57 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -299,7 +299,7 @@ services: volumes: - hivemind_vector_store:/project/vector_store healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-d", "celery@$$HOSTNAME"] + test: ["CMD", "celery", "inspect", "ping", "-d", "celery@celery"] interval: 60s timeout: 10s retries: 2 From a838424a4c9af2bd4c44966728f811a160f05d16 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:45:17 +0100 Subject: [PATCH 09/16] 10s interval --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 18e7a57..c52c9a1 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -300,7 +300,7 @@ services: - hivemind_vector_store:/project/vector_store healthcheck: test: ["CMD", "celery", "inspect", "ping", "-d", "celery@celery"] - interval: 60s + interval: 10s timeout: 10s retries: 2 start_period: 10s From 991306e706f786070176a1171b0ef1cc59f6031e Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:54:46 +0100 Subject: [PATCH 10/16] update healthcheck --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index c52c9a1..daefd76 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -299,7 +299,7 @@ services: volumes: - hivemind_vector_store:/project/vector_store healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-d", "celery@celery"] + test: ["CMD-SHELL", "celery inspect ping --destination celery@$$HOSTNAME"] interval: 10s timeout: 10s retries: 2 From 11263d6b318b8ec2330d6da117317b9e8bb43982 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:56:30 +0100 Subject: [PATCH 11/16] update hostname --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index daefd76..97593c6 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -295,7 +295,7 @@ services: depends_on: hivemind-vector-redis: condition: service_healthy - command: celery -A tasks.celery worker -c 1 --hostname celery --loglevel=info + command: celery -A tasks.celery worker -c 1 --hostname hivemind-vector-worker --loglevel=info volumes: - hivemind_vector_store:/project/vector_store healthcheck: From 73dad9e11b1e492ce21e198f4c2922ec2aeadfc3 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:02:47 +0100 Subject: [PATCH 12/16] fix for healthcheck --- development/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 97593c6..7c66d1c 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -299,7 +299,7 @@ services: volumes: - hivemind_vector_store:/project/vector_store healthcheck: - test: ["CMD-SHELL", "celery inspect ping --destination celery@$$HOSTNAME"] + test: ["CMD-SHELL", "celery -A tasks.celery inspect ping"] interval: 10s timeout: 10s retries: 2 From b39f15c373f2926dff3812575cfd49e3ec71e362 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:21:16 +0100 Subject: [PATCH 13/16] Port mapping 41234 and 43333 --- development/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 7c66d1c..da047a7 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -287,6 +287,8 @@ services: - development - monitoring <<: *logging + ports: + - 41234:1234 hivemind-vector-worker: image: ghcr.io/kamikazebr/qabot:main-vector_server @@ -338,6 +340,8 @@ services: - development - monitoring <<: *logging + ports: + - 43333:3333 # HIVEMIND SERVICES - END From 153c599552210cd3d2220902edcdefa89b13dd74 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:50:07 +0100 Subject: [PATCH 14/16] discord analyzer development --- development/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index da047a7..494070b 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -174,7 +174,7 @@ services: <<: *logging discord-analyzer-server: - image: ghcr.io/togethercrew/discord-analyzer:pr-18 + image: ghcr.io/togethercrew/discord-analyzer:development command: python3 server.py restart: unless-stopped env_file: @@ -190,7 +190,7 @@ services: <<: *logging discord-analyzer-worker: - image: ghcr.io/togethercrew/discord-analyzer:pr-18 + image: ghcr.io/togethercrew/discord-analyzer:development command: python3 worker.py restart: unless-stopped env_file: From 1e2afcb42d3361d7237940a5ac50dee9384ea498 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:52:59 +0100 Subject: [PATCH 15/16] pr-39 --- development/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 494070b..d82c2bd 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -174,7 +174,7 @@ services: <<: *logging discord-analyzer-server: - image: ghcr.io/togethercrew/discord-analyzer:development + image: ghcr.io/togethercrew/discord-analyzer:pr-39 command: python3 server.py restart: unless-stopped env_file: @@ -190,7 +190,7 @@ services: <<: *logging discord-analyzer-worker: - image: ghcr.io/togethercrew/discord-analyzer:development + image: ghcr.io/togethercrew/discord-analyzer:pr-39 command: python3 worker.py restart: unless-stopped env_file: From c6febd48a73bdf4e1d3b4c49e080449334a66e60 Mon Sep 17 00:00:00 2001 From: Cyrille <1618640+cyri113@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:41:44 +0100 Subject: [PATCH 16/16] Update docker-compose.yml --- development/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/development/docker-compose.yml b/development/docker-compose.yml index d82c2bd..72ff462 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -174,7 +174,7 @@ services: <<: *logging discord-analyzer-server: - image: ghcr.io/togethercrew/discord-analyzer:pr-39 + image: ghcr.io/togethercrew/discord-analyzer:pr-43 command: python3 server.py restart: unless-stopped env_file: @@ -190,7 +190,7 @@ services: <<: *logging discord-analyzer-worker: - image: ghcr.io/togethercrew/discord-analyzer:pr-39 + image: ghcr.io/togethercrew/discord-analyzer:pr-43 command: python3 worker.py restart: unless-stopped env_file: