Add fluentbit setup (#77)

### Description Add fluentbit setup to send the logs to firehose
LiskHQ · Oct 18, 2024 · 62f1ce8 · 62f1ce8
1 parent fa79cc8
commit 62f1ce8
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 43 deletions.
diff --git a/.prettierignore b/.prettierignore
@@ -38,6 +38,7 @@ dist/
 *.lock
 *.keep
 *.txt
+*.conf
 
 # env example files
 *.env.example

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,16 +21,14 @@ services:
  depends_on:
  redis:
  condition: service_healthy
+ fluent-bit:
+ condition: service_healthy
  networks:
  - across_relayer_network
  restart: always
  command: ["/bin/sh", "/home/lisk/across-relayer/scripts/${NETWORK:-mainnet}/containerStart_relayer_1.sh"]
  logging:
- driver: awslogs
- options:
- awslogs-region: eu-west-3
- awslogs-group: /ec2/lisk-across-relayer
- awslogs-stream: ec2/lisk.across.relayer.docker.log
+ driver: fluentd
  environment:
  - REDIS_URL=redis://redis:6379
  ports:
@@ -45,19 +43,33 @@ services:
  depends_on:
  redis:
  condition: service_healthy
+ fluent-bit:
+ condition: service_healthy
  networks:
  - across_relayer_network
  restart: always
  command: ["/bin/sh", "/home/lisk/across-relayer/scripts/${NETWORK:-mainnet}/containerStart_rebalancer.sh"]
  logging:
- driver: awslogs
- options:
- awslogs-region: eu-west-3
- awslogs-group: /ec2/lisk-across-relayer
- awslogs-stream: ec2/lisk.across.rebalancer.docker.log
+ driver: fluentd
  environment:
  - REDIS_URL=redis://redis:6379
 
+ fluent-bit:
+ image: amazon/aws-for-fluent-bit:2.32.2
+ container_name: fluent-bit
+ networks:
+ - across_relayer_network
+ volumes:
+ - ./fluentbit_config:/fluent-bit/etc
+ ports:
+ - "24224:24224"
+ - "24224:24224/udp"
+ healthcheck:
+ test: ["CMD-SHELL", "curl -f http://localhost:2020/api/v1/health || exit 1"]
+ interval: 10s
+ timeout: 5s
+ retries: 3
+
 networks:
  across_relayer_network:
 

diff --git a/fluentbit_config/fluent-bit.conf b/fluentbit_config/fluent-bit.conf
@@ -0,0 +1,30 @@
+[SERVICE]
+ HTTP_Server On
+ HTTP_Listen 0.0.0.0
+ HTTP_Port 2020
+ Flush 1
+ Log_Level info
+
+[INPUT]
+ Name forward
+ Listen 0.0.0.0
+ Port 24224
+
+[FILTER]
+ Name aws
+ Match *
+
+[FILTER]
+ Name modify
+ Match *
+ Set container_name @liskhq/across-relayer
+
+[OUTPUT]
+ Name kinesis_firehose
+ Match *
+ delivery_stream Lisk-ELK-Firehose
+ region eu-west-3
+
+[OUTPUT]
+ Name stdout
+ Match *
diff --git a/scripts/lisk/docker/dev/docker-compose-dev.patch b/scripts/lisk/docker/dev/docker-compose-dev.patch
@@ -1,8 +1,8 @@
 diff --git a/docker-compose.yml b/docker-compose.yml
-index 871674f2..303cb03c 100644
+index c92b5456..3253a2cb 100644
 --- a/docker-compose.yml
 +++ b/docker-compose.yml
-@@ -16,21 +16,24 @@ services:
+@@ -16,7 +16,8 @@ services:
 
  relayer_1:
  container_name: relayer_1
@@ -12,30 +12,20 @@ index 871674f2..303cb03c 100644
  platform: linux/amd64
  depends_on:
  redis:
- condition: service_healthy
-+ volumes:
-+ - ./.aws/credentials:/home/lisk/.aws/credentials
+@@ -26,9 +27,11 @@ services:
  networks:
  - across_relayer_network
  restart: always
 - command: ["/bin/sh", "/home/lisk/across-relayer/scripts/${NETWORK:-mainnet}/containerStart_relayer_1.sh"]
-- logging:
-- driver: awslogs
-- options:
-- awslogs-region: eu-west-3
-- awslogs-group: /ec2/lisk-across-relayer
-- awslogs-stream: ec2/lisk.across.relayer.docker.log
 + command: ["/bin/sh", "/home/lisk/across-relayer/scripts/${NETWORK:-sepolia}/containerStart_relayer_1.sh"]
-+ # logging:
-+ # driver: awslogs
-+ # options:
-+ # awslogs-region: eu-west-3
-+ # awslogs-group: /ec2/lisk-across-relayer
-+ # awslogs-stream: ec2/lisk.across.relayer.docker.log
+ logging:
+ driver: fluentd
++ volumes:
++ - ./.aws/credentials:/home/lisk/.aws/credentials
  environment:
  - REDIS_URL=redis://redis:6379
  ports:
-@@ -40,21 +43,24 @@ services:
+@@ -38,7 +41,8 @@ services:
 
  rebalancer:
  container_name: rebalancer
@@ -45,26 +35,24 @@ index 871674f2..303cb03c 100644
  platform: linux/amd64
  depends_on:
  redis:
- condition: service_healthy
-+ volumes:
-+ - ./.aws/credentials:/home/lisk/.aws/credentials
+@@ -48,9 +52,11 @@ services:
  networks:
  - across_relayer_network
  restart: always
 - command: ["/bin/sh", "/home/lisk/across-relayer/scripts/${NETWORK:-mainnet}/containerStart_rebalancer.sh"]
-- logging:
-- driver: awslogs
-- options:
-- awslogs-region: eu-west-3
-- awslogs-group: /ec2/lisk-across-relayer
-- awslogs-stream: ec2/lisk.across.rebalancer.docker.log
 + command: ["/bin/sh", "/home/lisk/across-relayer/scripts/${NETWORK:-sepolia}/containerStart_rebalancer.sh"]
-+ # logging:
-+ # driver: awslogs
-+ # options:
-+ # awslogs-region: eu-west-3
-+ # awslogs-group: /ec2/lisk-across-relayer
-+ # awslogs-stream: ec2/lisk.across.rebalancer.docker.log
+ logging:
+ driver: fluentd
++ volumes:
++ - ./.aws/credentials:/home/lisk/.aws/credentials
  environment:
  - REDIS_URL=redis://redis:6379
 
+@@ -61,6 +67,7 @@ services:
+ - across_relayer_network
+ volumes:
+ - ./fluentbit_config:/fluent-bit/etc
++ - ./.aws/credentials:/home/lisk/.aws/credentials
+ ports:
+ - "24224:24224"
+ - "24224:24224/udp"