Merge pull request #14 from SEKOIA-IO/debug-improvements

Debug improvements

.github/workflows/build-docker-image.yaml

@@ -31,5 +31,4 @@ jobs:
       with:
         push: true
         tags: |
-          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:1.0
-          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:latest
+          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0

CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+
+All notable changes with sekoiaio concentrator will be documented in this file.
+
+## [2.0]
+
+- Manage syslog RFC 3164 (only 5424 in 1.0 version)
+- Add advanced debug options
+- Update implementation from bash to jinja
+
+## [1.0] 
+
+- Initial version

Dockerfile

@@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y \
     gettext-base \
     python3 \
     python3-yaml \
+    python3-jinja2 \
     wget
 
 RUN wget -O /SEKOIA-IO-intake.pem https://app.sekoia.io/assets/files/SEKOIA-IO-intake.pem
@@ -17,11 +18,11 @@ ENV MEMORY_MESSAGES=100000
 # Setting up Rsyslog
 RUN rm -rf /etc/rsyslog.d/50-default.conf
 
-COPY parse_yaml.py parse_yaml.py
+COPY generate_config.py generate_config.py
 COPY rsyslog.conf rsyslog.conf
 COPY entrypoint.sh entrypoint.sh
 COPY intakes.yaml intakes.yaml
-COPY template.conf template.conf
+COPY template.j2 template.j2
 
 RUN chmod +x entrypoint.sh
 

README.md

@@ -52,6 +52,28 @@ intakes:
   intake_key: INTAKE_KEY_FOR_TECHNO_3
 ```
 
+#### Debug 
+A debug variable is available in order to debug a specific intake, for example 
+```yaml
+---
+intakes:
+- name: Techno1
+  protocol: tcp
+  port: 20516
+  intake_key: INTAKE_KEY_FOR_TECHNO_1
+- name: Techno2
+  protocol: tcp
+  port: 20517
+  intake_key: INTAKE_KEY_FOR_TECHNO_2
+  debug: True
+- name: Techno3
+  protocol: tcp
+  port: 20518
+  intake_key: INTAKE_KEY_FOR_TECHNO_3
+```
+
+By using this key, the raw received message and the output message will be printed in the console. Each one will be respectively identified using tags: : [Input $INTAKE_KEY] & [Output $INTAKE_KEY]
+
 ### Docker-compose file
 To ease the deployment, a `docker-compose.yml` file is suggested and a template is given.
 
@@ -128,6 +150,11 @@ To view container logs:
 sudo docker compose logs
 ```
 
+To view container logs for a specific intake:
+```bash
+sudo docker compose logs | grep "YOUR_INTAKE_KEY"
+```
+
 To stop the container:
 ```bash
 sudo docker compose stop

docker-compose/docker-compose.yml

@@ -5,7 +5,7 @@ services:
       options:
         max-size: "1000m"
         max-file: "2"
-    image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:1.0
+    image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0
     environment:
       - MEMORY_MESSAGES=100000
       - DISK_SPACE=32g

entrypoint.sh

@@ -9,29 +9,12 @@ echo "-----------------------------"
 envsubst '${DISK_SPACE} ${MEMORY_MESSAGES}' <rsyslog.conf >/etc/rsyslog.conf
 
 # Parse yaml intake file
-python3 parse_yaml.py
+python3 generate_config.py
 ret=$?
 if [ $ret -ne 0 ]; then
     # If the the YAML is not as expected
     echo -e "\n\nError in the file intakes.yaml. Verify is as expected and rerun the container."
     exit 1
 fi
 
-i=1
-while IFS=";" read -r rec_column1 rec_column2 rec_column3 rec_column4 || [ -n "$rec_column4" ]
-do
-    intake_name=$(echo "${rec_column1// /-}" | awk '{print tolower($0)}')
-    protocol=$(echo "$rec_column2" | awk '{print tolower($0)}')
-    port=$rec_column3
-    intake_key=$rec_column4
-
-    intake_name=$intake_name protocol=$protocol port=$port intake_key=${intake_key//[$'\t\r\n']} envsubst <template.conf >/etc/rsyslog.d/$i-$intake_name.conf
-    i=$(($i+1))
-    echo "Intake name: $intake_name"
-    echo "Protocol: $protocol"
-    echo "Port: $port"
-    echo "Intake key: $intake_key"
-    echo ""
-done < intakes.csv
-
 exec "$@"

generate_config.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import yaml
+from jinja2 import Environment, FileSystemLoader
+
+# Open input config file
+with open("intakes.yaml", "r") as fyaml:
+    data = yaml.safe_load(fyaml)
+
+# Load jinja template
+template = Environment(loader=FileSystemLoader(".")).get_template("template.j2")
+
+i=1
+# Generate one file per intake
+for item in data.get("intakes", []):
+    print("Intake name: " + str(item["name"].lower()))
+    print("Protocol: " + str(item["protocol"]))
+    print("Port: " + str(item["port"]))
+    print("Intake key: " + str(item["intake_key"]))
+    print("")
+    config = template.render(item)
+    filename = f"/etc/rsyslog.d/{i}_{item['name'].lower()}.conf"
+    # Écrire le contenu généré dans le fichier
+    with open(filename, "w") as f:
+        f.write(config)
+    i=i+1
+

rsyslog.conf

@@ -9,7 +9,7 @@
 
 global(
     defaultNetstreamDriverCAFile="/SEKOIA-IO-intake.pem"
-    maxMessageSize="64k"
+    maxMessageSize="250k"
     umask="0022"
     workDirectory="/var/spool/rsyslog"
 )

template.j2

@@ -0,0 +1,35 @@
+input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}")
+
+{% if debug %}
+template(name="SEKOIAIO_{{ name |lower }}_Input_Template" type="string" string="[Input \"{{ intake_key }}\"] %rawmsg%\n")
+template(name="SEKOIAIO_{{ name |lower }}_Output_Template" type="string" string="[Output \"{{ intake_key }}\"] <%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
+{% endif %}
+template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
+ruleset(name="remote{{ port }}"){
+action(
+    type="omfwd"
+    protocol="tcp"
+    target="intake.sekoia.io"
+    port="10514"
+    TCP_Framing="octet-counted"
+    StreamDriver="gtls"
+    StreamDriverMode="1"
+    StreamDriverAuthMode="x509/name"
+    StreamDriverPermittedPeers="intake.sekoia.io"
+    Template="SEKOIAIO_{{ name | lower }}_Template"
+    )
+
+{% if ( debug | lower ) == "true"  %}
+action(
+    type="omfile"
+    file="/dev/stdout"
+    Template="SEKOIAIO_{{ name |lower }}_Input_Template"
+    )
+action(
+    type="omfile"
+    file="/dev/stdout"
+    Template="SEKOIAIO_{{ name |lower }}_Output_Template"
+    )
+{% endif %}
+
+}