misc(worker): Introduce SolidQueue

getlago · Nov 26, 2024 · ade6d72 · ade6d72
1 parent 152790a
commit ade6d72
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 18 deletions.
diff --git a/api b/api
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -94,6 +94,7 @@ services:
       - LAGO_API_URL=https://api.lago.dev
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago}
       - DATABASE_TEST_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_test}
+      - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue}
       - REDIS_URL=redis://redis:6379
       - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64}
       - LAGO_FRONT_URL=${LAGO_FRONT_URL:-https://app.lago.dev}
@@ -110,8 +111,9 @@ services:
       - LAGO_LICENSE=${LAGO_LICENSE:-}
       - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:[email protected]}
       - LAGO_SIDEKIQ_WEB=true
-      - SIDEKIQ_EVENTS=true
-      - SIDEKIQ_PDFS=true
+      - LAGO_WORKER_EVENTS=true
+      - LAGO_WORKER_PDFS=true
+      - LAGO_JOB_DASHBOARD=true
       - LAGO_CLICKHOUSE_ENABLED=true
       - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true
       - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092
@@ -148,6 +150,7 @@ services:
     environment:
       - LAGO_API_URL=https://api.lago.dev
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago}
+      - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue}
       - REDIS_URL=redis://redis:6379
       - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64}
       - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key}
@@ -162,8 +165,8 @@ services:
       - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-}
       - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-}
       - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:[email protected]}
-      - SIDEKIQ_EVENTS=true
-      - SIDEKIQ_PDFS=true
+      - LAGO_WORKER_EVENTS=true
+      - LAGO_WORKER_PDFS=true
       - LAGO_CLICKHOUSE_ENABLED=true
       - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true
       - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092
@@ -188,6 +191,7 @@ services:
     environment:
       - LAGO_API_URL=https://api.lago.dev
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago}
+      - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue}
       - REDIS_URL=redis://redis:6379
       - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64}
       - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key}
@@ -202,7 +206,7 @@ services:
       - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-}
       - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-}
       - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:[email protected]}
-      - SIDEKIQ_EVENTS=true
+      - LAGO_WORKER_EVENTS=true
       - LAGO_CLICKHOUSE_ENABLED=true
       - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true
       - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092
@@ -227,6 +231,7 @@ services:
     environment:
       - LAGO_API_URL=https://api.lago.dev
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago}
+      - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue}
       - REDIS_URL=redis://redis:6379
       - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64}
       - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key}
@@ -241,7 +246,7 @@ services:
       - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT:-}
       - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH:-}
       - LAGO_FROM_EMAIL=${LAGO_FROM_EMAIL:[email protected]}
-      - SIDEKIQ_PDFS=true
+      - LAGO_WORKER_PDFS=true
       - LAGO_CLICKHOUSE_ENABLED=true
       - LAGO_CLICKHOUSE_MIGRATIONS_ENABLED=true
       - LAGO_KAFKA_BOOTSTRAP_SERVERS=redpanda:9092
@@ -266,6 +271,7 @@ services:
     environment:
       - LAGO_API_URL=https://api.lago.dev
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago}
+      - DATABASE_QUEUE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@db:5432/${POSTGRES_DB:-lago_queue}
       - REDIS_URL=redis://redis:6379
       - SECRET_KEY_BASE=${SECRET_KEY_BASE:-your-secret-key-base-hex-64}
       - LAGO_ENCRYPTION_PRIMARY_KEY=${ENCRYPTION_PRIMARY_KEY:-your-encrpytion-primary-key}

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -37,7 +37,7 @@ services:
     depends_on:
       - db
       - redis
-    command: ['./scripts/start.sh']
+    command: ["./scripts/start.sh"]
     healthcheck:
       test: curl -f http://localhost:3000/health || exit 1
       interval: 10s
@@ -58,6 +58,7 @@ services:
       - RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded
       - LAGO_RSA_PRIVATE_KEY=${LAGO_RSA_PRIVATE_KEY} # Should be base64 encoded
       - LAGO_SIDEKIQ_WEB=${LAGO_SIDEKIQ_WEB}
+      - LAGO_JOB_DASHBOARD=${LAGO_JOB_DASHBOARD}
       - LAGO_ENCRYPTION_PRIMARY_KEY=${LAGO_ENCRYPTION_PRIMARY_KEY:-your-encryption-primary-key}
       - LAGO_ENCRYPTION_DETERMINISTIC_KEY=${LAGO_ENCRYPTION_DETERMINISTIC_KEY:-your-encryption-deterministic-key}
       - LAGO_ENCRYPTION_KEY_DERIVATION_SALT=${LAGO_ENCRYPTION_KEY_DERIVATION_SALT:-your-encryption-derivation-salt}
@@ -80,8 +81,8 @@ services:
       - GOOGLE_AUTH_CLIENT_ID=${GOOGLE_AUTH_CLIENT_ID:-}
       - GOOGLE_AUTH_CLIENT_SECRET=${GOOGLE_AUTH_CLIENT_SECRET:-}
       - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-}
-      # - SIDEKIQ_EVENTS=true
-      # - SIDEKIQ_PDFS=true
+      # - LAGO_WORKER_EVENTS=true
+      # - LAGO_WORKER_PDS=true
     volumes:
       - lago_storage_data:/app/storage
       # If using GCS, you need to put the credentials keyfile here
@@ -135,9 +136,9 @@ services:
     depends_on:
       api:
         condition: service_healthy
-    command: ['./scripts/start.worker.sh']
+    command: ["./scripts/start.worker.sh"]
     healthcheck:
-      test: ['CMD-SHELL', 'bundle exec sidekiqmon | grep $(hostname) || exit 1']
+      test: ["CMD-SHELL", "bundle exec sidekiqmon | grep $(hostname) || exit 1"]
     environment:
       - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000}
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public}
@@ -169,8 +170,8 @@ services:
       - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH}
       - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-}
       - LAGO_LICENSE=${LAGO_LICENSE:-}
-      # - SIDEKIQ_EVENTS=true
-      # - SIDEKIQ_PDFS=true
+      # - LAGO_WORKER_EVENTS=true
+      # - LAGO_WORKER_PDFS=true
     volumes:
       - lago_storage_data:/app/storage
 
@@ -214,7 +215,7 @@ services:
   #    - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT}
   #    - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH}
   #    - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-}
-  #    - SIDEKIQ_EVENTS=true
+  #    - LAGO_WORKER_EVENTS=true
   #    - LAGO_LICENSE=${LAGO_LICENSE:-}
 
   # You can uncomment this if you want to use a dedicated Sidekiq worker for the invoices pdf creation.
@@ -257,7 +258,7 @@ services:
   #    - LAGO_DISABLE_SEGMENT=${LAGO_DISABLE_SEGMENT}
   #    - LAGO_DISABLE_WALLET_REFRESH=${LAGO_DISABLE_WALLET_REFRESH}
   #    - NANGO_SECRET_KEY=${NANGO_SECRET_KEY:-}
-  #    - SIDEKIQ_PDFS=true
+  #    - LAGO_WORKER_PDS=true
   #    - LAGO_LICENSE=${LAGO_LICENSE:-}
 
   api-clock:
@@ -267,7 +268,7 @@ services:
     depends_on:
       api:
         condition: service_healthy
-    command: ['./scripts/start.clock.sh']
+    command: ["./scripts/start.clock.sh"]
     environment:
       - LAGO_API_URL=${LAGO_API_URL:-http://localhost:3000}
       - DATABASE_URL=postgresql://${POSTGRES_USER:-lago}:${POSTGRES_PASSWORD:-changeme}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-lago}?search_path=${POSTGRES_SCHEMA:-public}
@@ -296,7 +297,7 @@ services:
     depends_on:
       - db
       - redis
-    command: ['./scripts/start.migrate.sh']
+    command: ["./scripts/start.migrate.sh"]
     volumes:
       - lago_storage_data:/app/storage
     environment: