Increased observability by having metrics, logs and traces in one place

.env

@@ -0,0 +1,2 @@
+
+HOST_WORKING_DIR=${PWD}

customer-service/build.gradle

@@ -10,6 +10,9 @@ dependencies {
 
     compile "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion"
     compile "org.springframework.boot:spring-boot-starter-web:$springBootVersion"
+
+    compile 'io.micrometer:micrometer-registry-jmx:1.3.0'
+    compile 'io.micrometer:micrometer-registry-prometheus:1.3.0'
 
     compile "io.eventuate.tram.core:eventuate-tram-spring-events:$eventuateTramVersion"
 

customer-service/src/main/java/io/eventuate/examples/tram/ordersandcustomers/customers/web/CustomerController.java

@@ -4,6 +4,8 @@
 import io.eventuate.examples.tram.ordersandcustomers.customers.service.CustomerService;
 import io.eventuate.examples.tram.ordersandcustomers.customers.webapi.CreateCustomerRequest;
 import io.eventuate.examples.tram.ordersandcustomers.customers.webapi.CreateCustomerResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -12,6 +14,8 @@
 
 @RestController
 public class CustomerController {
+
+  private Logger logger = LoggerFactory.getLogger(getClass());
 
   private CustomerService customerService;
 
@@ -22,6 +26,7 @@ public CustomerController(CustomerService customerService) {
 
   @RequestMapping(value = "/customers", method = RequestMethod.POST)
   public CreateCustomerResponse createCustomer(@RequestBody CreateCustomerRequest createCustomerRequest) {
+    logger.info("Create customer {} with credit limit of {}", createCustomerRequest.getName(), createCustomerRequest.getCreditLimit());
     Customer customer = customerService.createCustomer(createCustomerRequest.getName(), createCustomerRequest.getCreditLimit());
     return new CreateCustomerResponse(customer.getId());
   }

customer-service/src/main/resources/application.properties

@@ -18,3 +18,7 @@ eventuate.redis.partitions=2
 spring.sleuth.enabled=true
 spring.sleuth.sampler.probability=1
 spring.zipkin.base.url=http://${DOCKER_HOST_IP:localhost}:9411/
+
+
+management.endpoint.health.show-details=ALWAYS
+management.endpoints.web.exposure.include=prometheus,health

docker-compose-observable.yml

@@ -0,0 +1,161 @@
+version: '3.4'
+
+x-logging:
+  &loki-logging
+    driver: loki
+    options:
+      loki-url: "http://localhost:3100/loki/api/v1/push"
+
+
+services:
+
+  customer-service:
+    logging: *loki-logging
+    build: ./customer-service/
+    ports:
+      - "8081:8080"
+    depends_on:
+      - mysql
+      - redis
+      - cdc-service
+      - grafana
+    environment:
+      SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate
+      SPRING_DATASOURCE_USERNAME: mysqluser
+      SPRING_DATASOURCE_PASSWORD: mysqlpw
+      SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver
+      EVENTUATE_REDIS_SERVERS: redis:6379
+      EVENTUATE_REDIS_PARTITIONS: 2
+      SPRING_PROFILES_ACTIVE: Redis
+      SPRING_SLEUTH_ENABLED: "true"
+      SPRING_SLEUTH_SAMPLER_PROBABILITY: 1
+      SPRING_ZIPKIN_BASE_URL: http://zipkin:9411/
+      JAVA_OPTS: -Xmx128m
+
+
+  order-service:
+    logging: *loki-logging
+    build: ./order-service/
+    ports:
+      - "8082:8080"
+    depends_on:
+      - mysql
+      - redis
+      - cdc-service
+      - grafana
+    environment:
+      SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate
+      SPRING_DATASOURCE_USERNAME: mysqluser
+      SPRING_DATASOURCE_PASSWORD: mysqlpw
+      SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver
+      EVENTUATE_REDIS_SERVERS: redis:6379
+      EVENTUATE_REDIS_PARTITIONS: 2
+      SPRING_PROFILES_ACTIVE: Redis
+      SPRING_SLEUTH_ENABLED: "true"
+      SPRING_SLEUTH_SAMPLER_PROBABILITY: 1
+      SPRING_ZIPKIN_BASE_URL: http://zipkin:9411/
+      JAVA_OPTS: -Xmx128m
+
+  order-history-service:
+    logging: *loki-logging
+    build: ./order-history-service/
+    ports:
+      - "8083:8080"
+    depends_on:
+      - redis
+      - grafana
+    environment:
+      EVENTUATE_REDIS_SERVERS: redis:6379
+      EVENTUATE_REDIS_PARTITIONS: 2
+      SPRING_PROFILES_ACTIVE: Redis
+      SPRING_SLEUTH_ENABLED: "true"
+      SPRING_SLEUTH_SAMPLER_PROBABILITY: 1
+      SPRING_ZIPKIN_BASE_URL: http://zipkin:9411/
+      JAVA_OPTS: -Xmx128m
+
+## Infrastructure
+
+  zipkin:
+    logging: *loki-logging
+    image: openzipkin/zipkin:2.21
+    ports:
+      - "9411:9411"
+    environment:
+      JAVA_OPTS: -Xmx128m
+
+  mysql:
+    logging: *loki-logging
+    image: eventuateio/eventuate-mysql:0.9.0.RELEASE
+    ports:
+      - 3306:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=rootpassword
+      - MYSQL_USER=mysqluser
+      - MYSQL_PASSWORD=mysqlpw
+
+  redis:
+    logging: *loki-logging
+    image: redis:5.0.7
+    ports:
+      - 6379:6379
+
+  cdc-service:
+    logging: *loki-logging
+    image: eventuateio/eventuate-cdc-service:0.6.1.RELEASE
+    ports:
+      - "8099:8080"
+    depends_on:
+      - mysql
+      - redis
+    environment:
+      SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate
+      SPRING_DATASOURCE_USERNAME: mysqluser
+      SPRING_DATASOURCE_PASSWORD: mysqlpw
+      SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver
+      EVENTUATELOCAL_CDC_DB_USER_NAME: root
+      EVENTUATELOCAL_CDC_DB_PASSWORD: rootpassword
+      EVENTUATELOCAL_CDC_MYSQL_BINLOG_CLIENT_UNIQUE_ID: 1234567890
+      EVENTUATELOCAL_CDC_READER_NAME: MySqlReader
+      EVENTUATELOCAL_CDC_OFFSET_STORE_KEY: MySqlBinlog
+      EVENTUATELOCAL_CDC_READ_OLD_DEBEZIUM_DB_OFFSET_STORAGE_TOPIC: "false"
+      SPRING_PROFILES_ACTIVE: Redis
+      EVENTUATE_REDIS_SERVERS: redis:6379
+      EVENTUATE_REDIS_PARTITIONS: 2
+      JAVA_OPTS: -Xmx128m
+
+
+## Observability
+
+  loki:
+    logging: *loki-logging
+    image: grafana/loki:1.5.0
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
+  prometheus:
+    logging: *loki-logging
+    image: prom/prometheus:v2.1.0
+    ports:
+      - 9090:9090
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    volumes:
+      - ${HOST_WORKING_DIR}/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    environment:
+      HOST_WORKING_DIR: "${PWD}"
+
+  grafana:
+    logging: *loki-logging
+    depends_on:
+      - loki
+      - prometheus
+      - zipkin
+    image: grafana/grafana:master
+    ports:
+      - "3000:3000"
+    volumes:
+      - ${HOST_WORKING_DIR}/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+      - ${HOST_WORKING_DIR}/grafana/grafana.ini:/etc/grafana/grafana.ini
+    environment:
+      HOST_WORKING_DIR: "${PWD}"

docker-compose.yml

@@ -130,3 +130,4 @@ services:
       SPRING_SLEUTH_ENABLED: "true"
       SPRING_SLEUTH_SAMPLER_PROBABILITY: 1
       SPRING_ZIPKIN_BASE_URL: http://zipkin:9411/
+      HOST_WORKING_DIR: "${PWD}"

grafana/datasources.yaml

@@ -0,0 +1,29 @@
+apiVersion: 1
+
+datasources:
+
+  - name: Zipkin
+    type: zipkin
+    access: proxy
+    url: http://zipkin:9411
+    editable: true
+    uid: zipkin9411
+
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    editable: true
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: true
+    isDefault: true
+    jsonData:
+      derivedFields:
+        - datasourceUid: zipkin9411
+          matcherRegex: "\\[.*,(\\w+),.*,.*\\]"
+          name: TraceID
+          url: "$${__value.raw}"

grafana/grafana.ini

@@ -0,0 +1,3 @@
+[users]
+# Default UI theme ("dark" or "light")
+default_theme = light

order-history-service/build.gradle

@@ -19,5 +19,8 @@ dependencies {
 
     testCompile "junit:junit:4.12"
     testCompile "org.springframework.boot:spring-boot-starter-test:$springBootVersion"
+
+    compile 'io.micrometer:micrometer-registry-jmx:1.3.0'
+    compile 'io.micrometer:micrometer-registry-prometheus:1.3.0'
 
 }

order-history-service/src/main/resources/application.properties

@@ -8,4 +8,7 @@ eventuate.redis.partitions=2
 
 spring.sleuth.enabled=true
 spring.sleuth.sampler.probability=1
-spring.zipkin.base.url=http://${DOCKER_HOST_IP:localhost}:9411/
+spring.zipkin.base.url=http://${DOCKER_HOST_IP:localhost}:9411/
+
+management.endpoint.health.show-details=ALWAYS
+management.endpoints.web.exposure.include=prometheus,health

order-service/build.gradle

@@ -19,6 +19,9 @@ dependencies {
 
     compile "org.springframework.boot:spring-boot-starter-web:$springBootVersion"
     compile "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion"
+
+    compile 'io.micrometer:micrometer-registry-jmx:1.3.0'
+    compile 'io.micrometer:micrometer-registry-prometheus:1.3.0'
 
 }
 

order-service/src/main/java/io/eventuate/examples/tram/ordersandcustomers/orders/web/OrderController.java

@@ -7,13 +7,17 @@
 import io.eventuate.examples.tram.ordersandcustomers.orders.webapi.CreateOrderResponse;
 import io.eventuate.examples.tram.ordersandcustomers.orders.webapi.GetOrderResponse;
 import io.eventuate.examples.tram.ordersandcustomers.orderservice.domain.events.OrderDetails;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
 @RestController
 public class OrderController {
+
+  private Logger logger = LoggerFactory.getLogger(getClass());
 
   private OrderService orderService;
   private OrderRepository orderRepository;
@@ -26,6 +30,7 @@ public OrderController(OrderService orderService, OrderRepository orderRepositor
 
   @RequestMapping(value = "/orders", method = RequestMethod.POST)
   public CreateOrderResponse createOrder(@RequestBody CreateOrderRequest createOrderRequest) {
+    logger.info("Create order for customer {} with total of {}", createOrderRequest.getCustomerId(), createOrderRequest.getOrderTotal());
     Order order = orderService.createOrder(new OrderDetails(createOrderRequest.getCustomerId(), createOrderRequest.getOrderTotal()));
     return new CreateOrderResponse(order.getId());
   }

order-service/src/main/resources/application.properties

@@ -14,3 +14,7 @@ eventuate.redis.partitions=2
 spring.sleuth.enabled=true
 spring.sleuth.sampler.probability=1
 spring.zipkin.base.url=http://${DOCKER_HOST_IP:localhost}:9411/
+
+
+management.endpoint.health.show-details=ALWAYS
+management.endpoints.web.exposure.include=prometheus,health

prometheus/prometheus.yml

@@ -0,0 +1,53 @@
+# my global config
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+    - targets: ['localhost:9090']
+
+
+  - job_name: 'cdc-service'
+    metrics_path: '/actuator/prometheus'
+    static_configs:
+    - targets: ['cdc-service:8080']
+
+
+  - job_name: 'customer-service'
+    metrics_path: '/actuator/prometheus'
+    static_configs:
+    - targets: ['customer-service:8080']
+
+
+  - job_name: 'order-service'
+    metrics_path: '/actuator/prometheus'
+    static_configs:
+    - targets: ['order-service:8080']
+
+
+  - job_name: 'order-history-service'
+    metrics_path: '/actuator/prometheus'
+    static_configs:
+    - targets: ['order-history-service:8080']