From ade6d7288415052c9ae5270a38b3dca69706d3e0 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Mon, 25 Nov 2024 09:52:24 +0100 Subject: [PATCH 1/2] misc(worker): Introduce SolidQueue --- api | 2 +- docker-compose.dev.yml | 18 ++++++++++++------ docker-compose.yml | 23 ++++++++++++----------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/api b/api index cdeac9a..f3c2b79 160000 --- a/api +++ b/api @@ -1 +1 @@ -Subproject commit cdeac9a468fd5b874e417f843aed067a6091c395 +Subproject commit f3c2b79ee47124e5a822402d189c6211387ed748 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f84f16b..1fc0b25 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -94,6 +94,7 @@ services: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} - DATABASE_TEST_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_test} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_FRONT_URL=${LAGO_FRONT_URL:-https://app.lago.dev} @@ -110,8 +111,9 @@ services: - LAGO_LICENSE=${LAGO_LICENSE:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - LAGO_SIDEKIQ_WEB=true - - SIDEKIQ_EVENTS=true - - SIDEKIQ_PDFS=true + - LAGO_WORKER_EVENTS=true + - LAGO_WORKER_PDFS=true + - LAGO_JOB_DASHBOARD=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -148,6 +150,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} @@ -162,8 +165,8 @@ services: - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - - SIDEKIQ_EVENTS=true - - SIDEKIQ_PDFS=true + - LAGO_WORKER_EVENTS=true + - LAGO_WORKER_PDFS=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -188,6 +191,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} @@ -202,7 +206,7 @@ services: - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - - SIDEKIQ_EVENTS=true + - LAGO_WORKER_EVENTS=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -227,6 +231,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} @@ -241,7 +246,7 @@ services: - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:-noreply@getlago.com} - - SIDEKIQ_PDFS=true + - LAGO_WORKER_PDFS=true - LAGO_CLICKHOUSE_ENABLED=true - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 @@ -266,6 +271,7 @@ services: environment: - LAGO_API_URL=https://api.lago.dev - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} + - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - REDIS_URL=redis://redis:6379 - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} diff --git a/docker-compose.yml b/docker-compose.yml index 2150e71..32cfaa2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: depends_on: - db - redis - command: ['./scripts/start.sh'] + command: ["./scripts/start.sh"] healthcheck: test: curl -f http://localhost:3000/health || exit 1 interval: 10s @@ -58,6 +58,7 @@ services: - RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded - LAGO_RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded - LAGO_SIDEKIQ_WEB=${LAGO_SIDEKIQ_WEB} + - LAGO_JOB_DASHBOARD=${LAGO_JOB_DASHBOARD} - LAGO_ENCRYPTION_PRIMARY_KEY=${LAGO_ENCRYPTION_PRIMARY_KEY:-your-encryption-primary-key} - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${LAGO_ENCRYPTION_DETERMINISTIC_KEY:-your-encryption-deterministic-key} - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${LAGO_ENCRYPTION_KEY_DERIVATION_SALT:-your-encryption-derivation-salt} @@ -80,8 +81,8 @@ services: - GOOGLE_AUTH_CLIENT_ID=${GOOGLE_AUTH_CLIENT_ID:-} - GOOGLE_AUTH_CLIENT_SECRET=${GOOGLE_AUTH_CLIENT_SECRET:-} - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - # - SIDEKIQ_EVENTS=true - # - SIDEKIQ_PDFS=true + # - LAGO_WORKER_EVENTS=true + # - LAGO_WORKER_PDS=true volumes: - lago_storage_data:/app/storage # If using GCS, you need to put the credentials keyfile here @@ -135,9 +136,9 @@ services: depends_on: api: condition: service_healthy - command: ['./scripts/start.worker.sh'] + command: ["./scripts/start.worker.sh"] healthcheck: - test: ['CMD-SHELL', 'bundle exec sidekiqmon | grep $(hostname) || exit 1'] + test: ["CMD-SHELL", "bundle exec sidekiqmon | grep $(hostname) || exit 1"] environment: - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000} - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public} @@ -169,8 +170,8 @@ services: - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - LAGO_LICENSE=${LAGO_LICENSE:-} - # - SIDEKIQ_EVENTS=true - # - SIDEKIQ_PDFS=true + # - LAGO_WORKER_EVENTS=true + # - LAGO_WORKER_PDFS=true volumes: - lago_storage_data:/app/storage @@ -214,7 +215,7 @@ services: # - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT} # - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - # - SIDEKIQ_EVENTS=true + # - LAGO_WORKER_EVENTS=true # - LAGO_LICENSE=${LAGO_LICENSE:-} # You can uncomment this if you want to use a dedicated Sidekiq worker for the invoices pdf creation. @@ -257,7 +258,7 @@ services: # - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT} # - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} # - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - # - SIDEKIQ_PDFS=true + # - LAGO_WORKER_PDS=true # - LAGO_LICENSE=${LAGO_LICENSE:-} api-clock: @@ -267,7 +268,7 @@ services: depends_on: api: condition: service_healthy - command: ['./scripts/start.clock.sh'] + command: ["./scripts/start.clock.sh"] environment: - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000} - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public} @@ -296,7 +297,7 @@ services: depends_on: - db - redis - command: ['./scripts/start.migrate.sh'] + command: ["./scripts/start.migrate.sh"] volumes: - lago_storage_data:/app/storage environment: From 7062aede99ec8b4e3edca51bbd7f4c8212da69f5 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Tue, 26 Nov 2024 18:20:23 +0100 Subject: [PATCH 2/2] Remove clock container --- docker-compose.dev.yml | 33 --------------------------------- docker-compose.yml | 27 --------------------------- 2 files changed, 60 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1fc0b25..93b62b4 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -256,39 +256,6 @@ services: - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - LAGO_LICENSE=${LAGO_LICENSE:-} - api-clock: - image: api_dev - container_name: lago_api_clock_dev - restart: unless-stopped - command: ["./scripts/start.clock.dev.sh"] - depends_on: - - api - build: - context: ./api - dockerfile: $LAGO_PATH/api/Dockerfile.dev - volumes: - - $LAGO_PATH/api:/app:delegated - environment: - - LAGO_API_URL=https://api.lago.dev - - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago} - - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue} - - REDIS_URL=redis://redis:6379 - - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key} - - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${ENCRYPTION_DETERMINISTIC_KEY:-your-encrpytion-deterministic-key} - - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${ENCRYPTION_KEY_DERIVATION_SALT:-your-encrpytion-derivation-salt} - - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY:-} - - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-} - - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-} - - LAGO_CLICKHOUSE_ENABLED=true - - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true - - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 - - LAGO_KAFKA_RAW_EVENTS_TOPIC=events-raw - - LAGO_KAFKA_ENRICHED_EVENTS_TOPIC=events_enriched - - LAGO_KAFKA_CLICKHOUSE_CONSUMER_GROUP=clickhouse - - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - - LAGO_LICENSE=${LAGO_LICENSE:-} - pdf: image: getlago/lago-gotenberg:7 container_name: lago_pdf_dev diff --git a/docker-compose.yml b/docker-compose.yml index 32cfaa2..8a20d46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -261,33 +261,6 @@ services: # - LAGO_WORKER_PDS=true # - LAGO_LICENSE=${LAGO_LICENSE:-} - api-clock: - container_name: lago-clock - image: getlago/api:v1.16.0 - restart: unless-stopped - depends_on: - api: - condition: service_healthy - command: ["./scripts/start.clock.sh"] - environment: - - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000} - - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public} - - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379} - - REDIS_PASSWORD=${REDIS_PASSWORD} - - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64} - - RAILS_ENV=production - - RAILS_LOG_TO_STDOUT=${LAGO_RAILS_STDOUT:-true} - - SENTRY_DSN=${SENTRY_DSN} - - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH} - - LAGO_REDIS_CACHE_URL=redis://${LAGO_REDIS_CACHE_HOST:-redis}:${LAGO_REDIS_CACHE_PORT:-6379} - - LAGO_RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded - - RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded - - LAGO_ENCRYPTION_PRIMARY_KEY=${LAGO_ENCRYPTION_PRIMARY_KEY:-your-encryption-primary-key} - - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${LAGO_ENCRYPTION_DETERMINISTIC_KEY:-your-encryption-deterministic-key} - - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${LAGO_ENCRYPTION_KEY_DERIVATION_SALT:-your-encryption-derivation-salt} - - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-} - - LAGO_LICENSE=${LAGO_LICENSE:-} - pdf: image: getlago/lago-gotenberg:7.8.2