Merge pull request #27 from SEKOIA-IO/forwarder_metrics

Forwarder metrics

.github/workflows/build-docker-image.yaml

@@ -31,4 +31,4 @@ jobs:
       with:
         push: true
         tags: |
-          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.6.0
+          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.7.0

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes with sekoiaio concentrator will be documented in this file.
 
+## [2.7.0]
+
+- Enable forwarder monitoring
+
 ## [2.6.0]
 
 - Add the support of TLS

Dockerfile

@@ -20,11 +20,13 @@ ENV REGION=FRA1
 RUN rm -rf /etc/rsyslog.d/50-default.conf
 
 COPY generate_config.py generate_config.py
+COPY rsyslog-imstats /etc/logrotate.d/rsyslog-imstats
 COPY rsyslog.conf rsyslog.conf
 COPY entrypoint.sh entrypoint.sh
 COPY intakes.yaml intakes.yaml
 COPY template.j2 template.j2
 COPY template_tls.j2 template_tls.j2
+COPY stats_template.j2 stats_template.j2
 
 RUN chmod +x entrypoint.sh
 

docker-compose/docker-compose.yml

@@ -2,7 +2,7 @@
 # version: "3.9"
 services:
   rsyslog:
-    image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.6.0
+    image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.7.0
     environment:
       - MEMORY_MESSAGES=2000000
       - DISK_SPACE=180g

docker-compose/intakes.yaml

@@ -11,4 +11,7 @@ intakes:
 - name: Techno3
   protocol: tcp
   port: 20518
-  intake_key: INTAKE_KEY_FOR_TECHNO_3
+  intake_key: INTAKE_KEY_FOR_TECHNO_3
+- name: Forwarder
+  stats: true 
+  intake_key: INTAKE_KEY_FOR_FORWARDER_LOGS

generate_config.py

@@ -12,13 +12,27 @@ def is_intake_key(intake_key: str) -> re.Match[str] | None:
     return re.search(pattern, intake_key)
 
 
+def activate_monitoring(item: dict[str, str]) -> None:
+    to_print.append("Forwarder monitoring is active")
+    to_print.append("Intake key: " + str(item["intake_key"]))
+    to_print.append("")
+    config = template_stats.render(item)
+    filename = f"/etc/rsyslog.d/stats_{item['name']}.conf"
+    # Écrire le contenu généré dans le fichier
+    with open(filename, "w") as f:
+        f.write(config)
+
+
 # Open input config file
 with open("intakes.yaml", "r") as fyaml:
     data = yaml.safe_load(fyaml)
 
 # Load jinja template
 template = Environment(loader=FileSystemLoader(".")).get_template("template.j2")
 template_tls = Environment(loader=FileSystemLoader(".")).get_template("template_tls.j2")
+template_stats = Environment(loader=FileSystemLoader(".")).get_template(
+    "stats_template.j2"
+)
 
 # Identify the region
 region = os.getenv("REGION")
@@ -42,22 +56,30 @@ def is_intake_key(intake_key: str) -> re.Match[str] | None:
 for item in data.get("intakes", []):
     if not is_intake_key(item["intake_key"]):
         print(
-            f"ERROR: The Intake Key provided for Intake Name {item['name'].lower()} is incorrect. Exiting..."
+            f"ERROR: The Intake Key provided for Intake Name {item['name']} is incorrect. Exiting..."
         )
         exit(0)
 
-    to_print.append("Intake name: " + str(item["name"].lower()))
+    item["endpoint"] = endpoint
+
+    name_origin = item["name"]
+    item["name"] = item["name"].replace(" ", "_").lower()
+
+    if item.get("stats") is not None and item.get("stats") is not False:
+        activate_monitoring(item)
+        continue
+
+    to_print.append("Intake name: " + str(name_origin))
     to_print.append("Protocol: " + str(item["protocol"]))
     to_print.append("Port: " + str(item["port"]))
     to_print.append("Intake key: " + str(item["intake_key"]))
     to_print.append("")
-    item["endpoint"] = endpoint
 
     if item["protocol"].lower() == "tls":
         config = template_tls.render(item)
     else:
         config = template.render(item)
-    filename = f"/etc/rsyslog.d/{i}_{item['name'].lower()}.conf"
+    filename = f"/etc/rsyslog.d/{i}_{item['name']}.conf"
     # Écrire le contenu généré dans le fichier
     with open(filename, "w") as f:
         f.write(config)

intakes.yaml

@@ -11,4 +11,7 @@ intakes:
 - name: Techno3
   protocol: tcp
   port: 20518
-  intake_key: INTAKE_KEY_FOR_TECHNO_3
+  intake_key: INTAKE_KEY_FOR_TECHNO_3
+- name: Forwarder
+  stats: true 
+  intake_key: INTAKE_KEY_FOR_FORWARDER_LOGS

rsyslog-imstats

@@ -0,0 +1,8 @@
+/var/log/rsyslog-stats.log {
+    size 10M
+    rotate 3
+    compress
+    missingok
+    notifempty
+    copytruncate
+}

rsyslog.conf

@@ -17,7 +17,7 @@ global(
 #################
 #### MODULES ####
 #################
-
+module(load="impstats" log.file="/var/log/rsyslog-stats.log" ruleset="stats" format="json")
 module(load="imuxsock") # Provides support for local system logging
 module(load="imtcp") # Provides support for tcp connections
 module(load="imudp") # Provides support for udp connections

stats_template.j2

@@ -0,0 +1,16 @@
+template(name="SEKOIAIO_stats_Template" type="string" string="<%pri%>1 %timegenerated:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg%\n")
+ruleset(name="stats"){
+action(
+    name="output-{{ name |lower }}"
+    type="omfwd"
+    protocol="tcp"
+    target="{{ endpoint }}"
+    port="10514"
+    TCP_Framing="octet-counted"
+    StreamDriver="gtls"
+    StreamDriverMode="1"
+    StreamDriverAuthMode="x509/name"
+    StreamDriverPermittedPeers="{{ endpoint }}"
+    Template="SEKOIAIO_stats_Template"
+    )
+}

template.j2

@@ -1,13 +1,13 @@
-input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}")
+input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="ruleset-{{ name | lower}}" name="input-{{ name | lower}}")
 
 {% if debug %}
 template(name="SEKOIAIO_{{ name |lower }}_Input_Template" type="string" string="[Input \"{{ intake_key }}\"] %rawmsg%\n")
 template(name="SEKOIAIO_{{ name |lower }}_Output_Template" type="string" string="[Output \"{{ intake_key }}\"] <%pri%>1 %timegenerated:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
 {% endif %}
 template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timegenerated:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
-ruleset(name="remote{{ port }}" queue.type="LinkedList" queue.filename="sekoia_{{ name |lower }}_queue" queue.saveOnShutdown="on"){
+ruleset(name="ruleset-{{ name | lower}}" queue.type="LinkedList" queue.filename="sekoia_{{ name |lower }}_queue" queue.saveOnShutdown="on"){
 action(
-    name="action{{ name |lower }}"
+    name="output-{{ name |lower }}"
     type="omfwd"
     protocol="tcp"
     target="{{ endpoint }}"
@@ -24,11 +24,13 @@ action(
 action(
     type="omfile"
     file="/dev/stdout"
+    name="debugin-{{ name | lower}}"
     Template="SEKOIAIO_{{ name |lower }}_Input_Template"
     )
 action(
     type="omfile"
     file="/dev/stdout"
+    name="debugout-{{ name | lower}}"
     Template="SEKOIAIO_{{ name |lower }}_Output_Template"
     )
 {% endif %}

template_tls.j2

@@ -1,6 +1,7 @@
 input(type="imtcp"
       port="{{ port }}"
-      ruleset="remote{{ port }}"
+      ruleset="ruleset-{{ name | lower}}"
+      name="input-{{ name | lower}}"
       StreamDriver.Name="gtls"
       StreamDriver.Mode="1"
       StreamDriver.AuthMode="anon"
@@ -15,9 +16,9 @@ template(name="SEKOIAIO_{{ name |lower }}_Output_Template" type="string" string=
 
 
 template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timegenerated:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
-ruleset(name="remote{{ port }}" queue.type="LinkedList" queue.filename="sekoia_{{ name |lower }}_queue" queue.saveOnShutdown="on"){
+ruleset(name="ruleset-{{ name | lower}}" queue.type="LinkedList" queue.filename="sekoia_{{ name |lower }}_queue" queue.saveOnShutdown="on"){
 action(
-    name="action{{ name |lower }}"
+    name="output-{{ name |lower }}"
     type="omfwd"
     protocol="tcp"
     target="{{ endpoint }}"
@@ -34,11 +35,13 @@ action(
 action(
     type="omfile"
     file="/dev/stdout"
+    name="debugin-{{ name | lower}}"
     Template="SEKOIAIO_{{ name |lower }}_Input_Template"
     )
 action(
     type="omfile"
     file="/dev/stdout"
+    name="debugout-{{ name | lower}}"
     Template="SEKOIAIO_{{ name |lower }}_Output_Template"
     )
 {% endif %}