From ade6d7288415052c9ae5270a38b3dca69706d3e0 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Mon, 25 Nov 2024 09:52:24 +0100 Subject: [PATCH] misc(worker): Introduce SolidQueue --- api | 2 +- docker-compose.dev.yml | 18 ++++++++++++------ docker-compose.yml | 23 ++++++++++++----------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/api b/api index cdeac9a4..f3c2b79e 160000 --- a/api +++ b/api @@ -1 +1 @@ -Subproject commit cdeac9a468fd5b874e417f843aed067a6091c395 +Subproject commit f3c2b79ee47124e5a822402d189c6211387ed748 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f84f16b8..1fc0b257 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -94,6 +94,7 @@ services: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} - DATABASE_TEST_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_test} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_FRONT_URL=${LAGO_FRONT_URL:-https://app.lago.dev} @@ -110,8 +111,9 @@ services: - LAGO_LICENSE=${LAGO_LICENSE:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - LAGO_SIDEKIQ_WEB=true - - SIDEKIQ_EVENTS=true - - SIDEKIQ_PDFS=true + - LAGO_WORKER_EVENTS=true + - LAGO_WORKER_PDFS=true + - LAGO_JOB_DASHBOARD=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -148,6 +150,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} @@ -162,8 +165,8 @@ services: - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - - SIDEKIQ_EVENTS=true - - SIDEKIQ_PDFS=true + - LAGO_WORKER_EVENTS=true + - LAGO_WORKER_PDFS=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -188,6 +191,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} @@ -202,7 +206,7 @@ services: - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - - SIDEKIQ_EVENTS=true + - LAGO_WORKER_EVENTS=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -227,6 +231,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} @@ -241,7 +246,7 @@ services: - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - - SIDEKIQ_PDFS=true + - LAGO_WORKER_PDFS=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -266,6 +271,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} diff --git a/docker-compose.yml b/docker-compose.yml index 2150e714..32cfaa2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: depends_on: - db - redis - command: ['./scripts/start.sh'] + command: ["./scripts/start.sh"] healthcheck: test: curl -f http://localhost:3000/health || exit 1 interval: 10s @@ -58,6 +58,7 @@ services: - RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded - LAGO_RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded - LAGO_SIDEKIQ_WEB=${LAGO_SIDEKIQ_WEB} + - LAGO_JOB_DASHBOARD=${LAGO_JOB_DASHBOARD} - LAGO_ENCRYPTION_PRIMARY_KEY=${LAGO_ENCRYPTION_PRIMARY_KEY:-your-encryption-primary-key} - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${LAGO_ENCRYPTION_DETERMINISTIC_KEY:-your-encryption-deterministic-key} - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${LAGO_ENCRYPTION_KEY_DERIVATION_SALT:-your-encryption-derivation-salt} @@ -80,8 +81,8 @@ services: - GOOGLE_AUTH_CLIENT_ID=${GOOGLE_AUTH_CLIENT_ID:-} - GOOGLE_AUTH_CLIENT_SECRET=${GOOGLE_AUTH_CLIENT_SECRET:-} - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - # - SIDEKIQ_EVENTS=true - # - SIDEKIQ_PDFS=true + # - LAGO_WORKER_EVENTS=true + # - LAGO_WORKER_PDS=true volumes: - lago_storage_data:/app/storage # If using GCS, you need to put the credentials keyfile here @@ -135,9 +136,9 @@ services: depends_on: api: condition: service_healthy - command: ['./scripts/start.worker.sh'] + command: ["./scripts/start.worker.sh"] healthcheck: - test: ['CMD-SHELL', 'bundle exec sidekiqmon | grep $(hostname) || exit 1'] + test: ["CMD-SHELL", "bundle exec sidekiqmon | grep $(hostname) || exit 1"] environment: - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000} - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public} @@ -169,8 +170,8 @@ services: - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - LAGO_LICENSE=${LAGO_LICENSE:-} - # - SIDEKIQ_EVENTS=true - # - SIDEKIQ_PDFS=true + # - LAGO_WORKER_EVENTS=true + # - LAGO_WORKER_PDFS=true volumes: - lago_storage_data:/app/storage @@ -214,7 +215,7 @@ services: # - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT} # - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - # - SIDEKIQ_EVENTS=true + # - LAGO_WORKER_EVENTS=true # - LAGO_LICENSE=${LAGO_LICENSE:-} # You can uncomment this if you want to use a dedicated Sidekiq worker for the invoices pdf creation. @@ -257,7 +258,7 @@ services: # - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT} # - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - # - SIDEKIQ_PDFS=true + # - LAGO_WORKER_PDS=true # - LAGO_LICENSE=${LAGO_LICENSE:-} api-clock: @@ -267,7 +268,7 @@ services: depends_on: api: condition: service_healthy - command: ['./scripts/start.clock.sh'] + command: ["./scripts/start.clock.sh"] environment: - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000} - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public} @@ -296,7 +297,7 @@ services: depends_on: - db - redis - command: ['./scripts/start.migrate.sh'] + command: ["./scripts/start.migrate.sh"] volumes: - lago_storage_data:/app/storage environment: