docker-compose.yml

version: '3'
services:
  postgres-airflow:
    image: postgres
    environment:
      POSTGRES_PASSWORD: pssd
      POSTGRES_USER : airflowuser
      POSTGRES_DB : airflowdb
      AIRFLOW_SCHEMA: airflow
    expose:
      - 5432
    restart: always
    volumes:
      - ./postgres/scripts:/docker-entrypoint-initdb.d
      # - ./sample_data:/sample_data
    networks:
      - common_network

  postgres-dbt:
    image: postgres
    environment:
      POSTGRES_PASSWORD: pssd
      POSTGRES_USER : dbtuser
      POSTGRES_DB : dbtdb
      DBT_SCHEMA: dbt
      DBT_RAW_DATA_SCHEMA: dbt_raw_data
    expose:
      - 5432
    restart: always
    volumes:
      - ./sample_data:/sample_data
    networks:
      - common_network

  airflow:
    build: ./airflow/
    restart: always
    environment: 
      DBT_PROFILES_DIR: /dbt
      AIRFLOW_HOME: /airflow
      PYTHON_DEPS: flask-session==0.3.2
      AIRFLOW__CORE__DAGS_FOLDER: /airflow/dags
      AIRFLOW__CORE__PARALLELISM: 4
      AIRFLOW__CORE__DAG_CONCURRENCY: 4
      AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG: 4
      # AIRFLOW__ADMIN__HIDE_SENSITIVE_VARIABLE_FIELDS: False
      # Postgres details need to match with the values defined in the postgres-airflow service
      POSTGRES_USER: airflowuser
      POSTGRES_PASSWORD: pssd
      POSTGRES_HOST: postgres-airflow
      POSTGRES_PORT: 5432
      POSTGRES_DB: airflowdb
      # postgres-dbt connection details. Required for the inital loading of seed data
      # Credentials need to match with service postgres-dbt
      DBT_POSTGRES_PASSWORD: pssd
      DBT_POSTGRES_USER : dbtuser
      DBT_POSTGRES_DB : dbtdb
      DBT_DBT_SCHEMA: dbt
      DBT_DBT_RAW_DATA_SCHEMA: dbt_raw_data
      DBT_POSTGRES_HOST: postgres-dbt
    depends_on:
      - postgres-airflow
      - postgres-dbt
    ports:
      - 8000:8080
    volumes:
      - ./dbt:/dbt
      - ./airflow:/airflow
    networks:
      - common_network

  adminer:
    image: adminer
    restart: always
    ports:
      - 8080:8080
    depends_on:
      - postgres-airflow
      - postgres-dbt
    networks:
      - common_network

  audit-service:
    build: ./dapr/node
    command: node audit-service.js
    ports:
      - "3000:3000"
      - "9091:9091"  # Add this line for app metrics
    depends_on:
      - redis
      - placement
    networks:
      - common_network

  audit-service-dapr:
    image: "daprio/daprd:edge"
    command: ["./daprd",
      "-app-id", "audit-service",
      "-app-port", "3000",
      "-components-path", "/components",
      "-config", "/config/tracing.yaml",
      "-metrics-port", "9093"]  # Add this line for Dapr sidecar metrics
    volumes:
      - "./dapr/components/:/components"
      - "./dapr/config/:/config"
    depends_on:
      - audit-service
    network_mode: "service:audit-service"

  airflow-trigger-service:
    build: ./dapr/node
    command: node airflow-trigger-service.js
    ports:
      - "3001:3001"
      - "9095:9095"  # Add this line for app metrics
    depends_on:
      - redis
      - placement
    networks:
      - common_network

  airflow-trigger-service-dapr:
    image: "daprio/daprd:edge"
    command: ["./daprd",
      "-app-id", "airflow-trigger-service",
      "-app-port", "3001",
      "-components-path", "/components",
      "-config", "/config/tracing.yaml",
      "-metrics-port", "9097"]  # Add this line for Dapr sidecar metrics
    volumes:
      - "./dapr/components/:/components"
      - "./dapr/config/:/config"
    depends_on:
      - airflow-trigger-service
    network_mode: "service:airflow-trigger-service"

  airflow-config-service:
    build: ./dapr/node
    command: node airflow-config-service.js
    ports:
      - "3002:3002"
      - "9099:9099"  # Add this line for app metrics
    depends_on:
      - redis
      - placement
    networks:
      - common_network

  airflow-config-service-dapr:
    image: "daprio/daprd:edge"
    command: ["./daprd",
      "-app-id", "airflow-config-service",
      "-app-port", "3002",
      "-components-path", "/components",
      "-config", "/config/tracing.yaml",
      "-metrics-port", "9101"]  # Add this line for Dapr sidecar metrics
    volumes:
      - "./dapr/components/:/components"
      - "./dapr/config/:/config"
    depends_on:
      - airflow-config-service
    network_mode: "service:airflow-config-service"
  
  management-service:
    build: ./dapr/node
    command: node management-service.js
    ports:
      - "3003:3003"
      - "9103:9103"  # Add this line for app metrics
    depends_on:
      - redis
      - placement
    networks:
      - common_network

  management-service-dapr:
    image: "daprio/daprd:edge"
    command: ["./daprd",
      "-app-id", "management-service",
      "-app-port", "3003",
      "-components-path", "/components",
      "-config", "/config/tracing.yaml",
      "-metrics-port", "9105"]  # Add this line for Dapr sidecar metrics
    volumes:
      - "./dapr/components/:/components"
      - "./dapr/config/:/config"
    depends_on:
      - management-service
    network_mode: "service:management-service"

  lineage-service:
    build: ./dapr/node
    command: node lineage-service.js
    ports:
      - "3004:3004"  # Change this line
      - "9107:9107"
    depends_on:
      - redis
      - placement
    networks:
      - common_network

  lineage-service-dapr:
    image: "daprio/daprd:edge"
    command: ["./daprd",
      "-app-id", "lineage-service",
      "-app-port", "3004",
      "-components-path", "/components",
      "-config", "/config/tracing.yaml",
      "-metrics-port", "9109"]  # Add this line for Dapr sidecar metrics
    volumes:
      - "./dapr/components/:/components"
      - "./dapr/config/:/config"
    depends_on:
      - lineage-service
    network_mode: "service:lineage-service"

  ############################
  # Python app + Dapr sidecar
  ############################
  pythonapp:
    environment:
      PYTHONUNBUFFERED: 1
      OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:4317"
      OTEL_RESOURCE_ATTRIBUTES: "service.name=pythonapp"
    build: ./dapr/python
    networks:
      - common_network
    ports:
      - "8501:8501"
      - "9092:9092"
      - "9094:9094"

  pythonapp-dapr:
    image: "daprio/daprd:edge"
    command: ["./daprd",
      "-app-id", "pythonapp",
      "-placement-host-address", "placement:50006",
      "-components-path", "/components",
      "-config", "/config/tracing.yaml",
      "-metrics-port", "9094"]
    volumes:
      - "./dapr/components/:/components"
      - "./dapr/config/:/config"
    depends_on:
      - pythonapp
      - otel-collector
    network_mode: "service:pythonapp"
  ############################
  # Dapr placement service
  ############################
  placement:
    image: "daprio/dapr"
    command: ["./placement", "-port", "50006"]
    ports:
      - "50006:50006"
    networks:
      - common_network
  ############################
  # Redis state store
  ############################
  redis:
    image: "redis:alpine"
    ports:
      - "6380:6379"
    networks:
      - common_network
  ############################
  # Prometheus
  ############################
  prometheus:
    image: prom/prometheus
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
    networks:
      - common_network

  ############################
  # Loki
  ############################
  # loki:
  #   image: grafana/loki:2.2.1
  #   ports:
  #     - "3100:3100"
  #   networks:
  #     - common_network
  #   command: -config.file=/etc/loki/local-config.yaml
  #   volumes:
  #     - ./loki/local-config.yaml:/etc/loki/local-config.yaml

  ############################
  # OpenTelemetry Collector
  ############################
  otel-collector:
    image: otel/opentelemetry-collector:latest
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"   # OTLP gRPC receiver
      - "4318:4318"   # OTLP HTTP receiver
      - "8888:8888"   # Prometheus metrics exposed by the collector
      - "8889:8889"   # Prometheus exporter metrics
      - "13133:13133" # Health check extension
    networks:
      - common_network
    healthcheck:  # Add this healthcheck block
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133/"]
      interval: 5s
      timeout: 5s
      retries: 5

  ############################
  # Grafana
  ############################
  grafana:
    image: grafana/grafana
    ports:
      - "4000:3000"
    networks:
      - common_network
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning
      - grafana-storage:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
      - GF_USERS_ALLOW_SIGN_UP=false
    restart: always

networks:
  common_network:

volumes:
  grafana-storage: