From 52d9e82fe1fd3c18e792adf5a7e239ae4ec4edac Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 13:50:57 +0200
Subject: [PATCH 01/12] Use jinja to simplify impl

---
 Dockerfile         |  5 +++--
 entrypoint.sh      | 19 +------------------
 generate_config.py | 22 ++++++++++++++++++++++
 parse_yaml.py      | 11 -----------
 template.conf      | 23 -----------------------
 template.j2        | 32 ++++++++++++++++++++++++++++++++
 6 files changed, 58 insertions(+), 54 deletions(-)
 create mode 100644 generate_config.py
 delete mode 100644 parse_yaml.py
 delete mode 100644 template.conf
 create mode 100644 template.j2

diff --git a/Dockerfile b/Dockerfile
index d6a82f1..ca6eab1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y \
     gettext-base \
     python3 \
     python3-yaml \
+    python3-jinja2 \
     wget
 
 RUN wget -O /SEKOIA-IO-intake.pem https://app.sekoia.io/assets/files/SEKOIA-IO-intake.pem
@@ -17,11 +18,11 @@ ENV MEMORY_MESSAGES=100000
 # Setting up Rsyslog
 RUN rm -rf /etc/rsyslog.d/50-default.conf
 
-COPY parse_yaml.py parse_yaml.py
+COPY generate_config.py generate_config.py
 COPY rsyslog.conf rsyslog.conf
 COPY entrypoint.sh entrypoint.sh
 COPY intakes.yaml intakes.yaml
-COPY template.conf template.conf
+COPY template.j2 template.j2
 
 RUN chmod +x entrypoint.sh
 
diff --git a/entrypoint.sh b/entrypoint.sh
index 60945d2..a9d91ac 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -9,7 +9,7 @@ echo "-----------------------------"
 envsubst '${DISK_SPACE} ${MEMORY_MESSAGES}' <rsyslog.conf >/etc/rsyslog.conf
 
 # Parse yaml intake file
-python3 parse_yaml.py
+python3 generate_config.py
 ret=$?
 if [ $ret -ne 0 ]; then
     # If the the YAML is not as expected
@@ -17,21 +17,4 @@ if [ $ret -ne 0 ]; then
     exit 1
 fi
 
-i=1
-while IFS=";" read -r rec_column1 rec_column2 rec_column3 rec_column4 || [ -n "$rec_column4" ]
-do
-    intake_name=$(echo "${rec_column1// /-}" | awk '{print tolower($0)}')
-    protocol=$(echo "$rec_column2" | awk '{print tolower($0)}')
-    port=$rec_column3
-    intake_key=$rec_column4
-
-    intake_name=$intake_name protocol=$protocol port=$port intake_key=${intake_key//[$'\t\r\n']} envsubst <template.conf >/etc/rsyslog.d/$i-$intake_name.conf
-    i=$(($i+1))
-    echo "Intake name: $intake_name"
-    echo "Protocol: $protocol"
-    echo "Port: $port"
-    echo "Intake key: $intake_key"
-    echo ""
-done < intakes.csv
-
 exec "$@"
diff --git a/generate_config.py b/generate_config.py
new file mode 100644
index 0000000..9a8363a
--- /dev/null
+++ b/generate_config.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+import yaml
+from jinja2 import Environment, FileSystemLoader
+
+# Open input config file
+with open("intakes.yaml", "r") as fyaml:
+    data = yaml.safe_load(fyaml)
+
+# Load jinja template
+template = Environment(loader=FileSystemLoader(".")).get_template("template.j2")
+
+i=1
+# Generate one file per intake
+for item in data.get("intakes", []):
+    config = template.render(item)
+    filename = f"/etc/rsyslog.d/{i}_{item['name'].lower()}.conf"
+    # Écrire le contenu généré dans le fichier
+    with open(filename, "w") as f:
+        f.write(filename)
+    i=i+1
+
diff --git a/parse_yaml.py b/parse_yaml.py
deleted file mode 100644
index cb0b294..0000000
--- a/parse_yaml.py
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-
-import yaml
-import csv
-
-with open("intakes.yaml", "r") as fyaml:
-    data = yaml.safe_load(fyaml)
-    with open('intakes.csv', 'w') as fcsv:
-        csvwriter = csv.writer(fcsv, delimiter=';')
-        for intake in data['intakes']:
-            csvwriter.writerow([intake['name'], intake['protocol'], intake['port'], intake['intake_key']])
diff --git a/template.conf b/template.conf
deleted file mode 100644
index df5739a..0000000
--- a/template.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-input(type="im${protocol}" port="${port}" ruleset="remote${port}")
-
-template(name="SEKOIAIO_${intake_name}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"${intake_key}\"] %msg%\n")
-ruleset(name="remote${port}"){
-action(
-    type="omfwd"
-    protocol="tcp"
-    target="intake.sekoia.io"
-    port="10514"
-    TCP_Framing="octet-counted"
-    StreamDriver="gtls"
-    StreamDriverMode="1"
-    StreamDriverAuthMode="x509/name"
-    StreamDriverPermittedPeers="intake.sekoia.io"
-    Template="SEKOIAIO_${intake_name}_Template"
-    )
-
-action(
-    type="omfile"
-    file="/dev/stdout"
-    Template="SEKOIAIO_${intake_name}_Template"
-    )
-}
diff --git a/template.j2 b/template.j2
new file mode 100644
index 0000000..a7fd2a5
--- /dev/null
+++ b/template.j2
@@ -0,0 +1,32 @@
+input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}")
+
+template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg%\n")
+ruleset(name="remote{{ port }}"){
+action(
+    type="omfwd"
+    protocol="tcp"
+    target="intake.sekoia.io"
+    port="10514"
+    TCP_Framing="octet-counted"
+    StreamDriver="gtls"
+    StreamDriverMode="1"
+    StreamDriverAuthMode="x509/name"
+    StreamDriverPermittedPeers="intake.sekoia.io"
+    Template="SEKOIAIO_{{ name | lower }}_Template"
+    )
+
+{% if debug %}
+template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="%rawmsg%\n")
+action(
+    type="omfile"
+    file="/dev/stdout"
+    Template="SEKOIAIO_{{ name |lower }}_Debug_Template"
+    )
+action(
+    type="omfile"
+    file="/dev/stdout"
+    Template="SEKOIAIO_{{ name |lower }}_Template"
+    )
+{% endif %}
+    
+}

From a4ca746d433c5d24c307aaa93d1c0e4e41e1781c Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 13:59:43 +0200
Subject: [PATCH 02/12] Fix file generation

---
 generate_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generate_config.py b/generate_config.py
index 9a8363a..052540f 100644
--- a/generate_config.py
+++ b/generate_config.py
@@ -17,6 +17,6 @@
     filename = f"/etc/rsyslog.d/{i}_{item['name'].lower()}.conf"
     # Écrire le contenu généré dans le fichier
     with open(filename, "w") as f:
-        f.write(filename)
+        f.write(config)
     i=i+1
 

From 8fe56d90aa427045217864d1ebd698f5524148b0 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 14:02:44 +0200
Subject: [PATCH 03/12] Fix template for debugging

---
 template.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/template.j2 b/template.j2
index a7fd2a5..4e0569b 100644
--- a/template.j2
+++ b/template.j2
@@ -16,7 +16,7 @@ action(
     )
 
 {% if debug %}
-template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="%rawmsg%\n")
+template(name="SEKOIAIO_{{ name |lower }}_Debug_Template" type="string" string="%rawmsg%\n")
 action(
     type="omfile"
     file="/dev/stdout"

From 3ca7df63cba29d2b3aff2495084a51d47031dfc0 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 14:05:08 +0200
Subject: [PATCH 04/12] Fix template for debugging

---
 template.j2 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/template.j2 b/template.j2
index 4e0569b..a57efae 100644
--- a/template.j2
+++ b/template.j2
@@ -1,5 +1,8 @@
 input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}")
 
+{% if debug %}
+template(name="SEKOIAIO_{{ name |lower }}_Debug_Template" type="string" string="%rawmsg%\n")
+{% endif %}
 template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg%\n")
 ruleset(name="remote{{ port }}"){
 action(
@@ -16,7 +19,6 @@ action(
     )
 
 {% if debug %}
-template(name="SEKOIAIO_{{ name |lower }}_Debug_Template" type="string" string="%rawmsg%\n")
 action(
     type="omfile"
     file="/dev/stdout"

From cbfaed8006012b3b680dd589974111162c7712f1 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 14:17:17 +0200
Subject: [PATCH 05/12] Update readme

---
 README.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/README.md b/README.md
index 90e4115..56926a9 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,28 @@ intakes:
   intake_key: INTAKE_KEY_FOR_TECHNO_3
 ```
 
+#### Debug 
+A debug variable is available in order to debug a specific intake, for example 
+```yaml
+---
+intakes:
+- name: Techno1
+  protocol: tcp
+  port: 20516
+  intake_key: INTAKE_KEY_FOR_TECHNO_1
+- name: Techno2
+  protocol: tcp
+  port: 20517
+  intake_key: INTAKE_KEY_FOR_TECHNO_2
+  debug: True
+- name: Techno3
+  protocol: tcp
+  port: 20518
+  intake_key: INTAKE_KEY_FOR_TECHNO_3
+```
+
+By using this key, the raw received message and the output message will be printed in the console
+
 ### Docker-compose file
 To ease the deployment, a `docker-compose.yml` file is suggested and a template is given.
 

From 49ec594dcd6cc95f2d9ca6c2bc28be94d86ed484 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 14:37:58 +0200
Subject: [PATCH 06/12] Increase max message size

---
 rsyslog.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rsyslog.conf b/rsyslog.conf
index 6990af7..2a9c74f 100644
--- a/rsyslog.conf
+++ b/rsyslog.conf
@@ -9,7 +9,7 @@
 
 global(
     defaultNetstreamDriverCAFile="/SEKOIA-IO-intake.pem"
-    maxMessageSize="64k"
+    maxMessageSize="250k"
     umask="0022"
     workDirectory="/var/spool/rsyslog"
 )

From f21112b790d79ba2e6e788db8454fa310ac76c63 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 14:38:19 +0200
Subject: [PATCH 07/12] Add prefix when debug is on

---
 template.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/template.j2 b/template.j2
index a57efae..cb0c497 100644
--- a/template.j2
+++ b/template.j2
@@ -1,7 +1,7 @@
 input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}")
 
 {% if debug %}
-template(name="SEKOIAIO_{{ name |lower }}_Debug_Template" type="string" string="%rawmsg%\n")
+template(name="SEKOIAIO_{{ name |lower }}_Debug_Template" type="string" string="[DEBUG \"{{ intake_key }}\"] %rawmsg%\n")
 {% endif %}
 template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg%\n")
 ruleset(name="remote{{ port }}"){

From ac63e5ed2d89a041ba42eefc0428d0ccc0d10b0f Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Mon, 16 Oct 2023 15:42:30 +0200
Subject: [PATCH 08/12] Manage rfc 3164 & improve debug messages

---
 README.md   |  2 +-
 template.j2 | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 56926a9..a9c2682 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ intakes:
   intake_key: INTAKE_KEY_FOR_TECHNO_3
 ```
 
-By using this key, the raw received message and the output message will be printed in the console
+By using this key, the raw received message and the output message will be printed in the console. Each one will be respectively identified using tags: : [Input $INTAKE_KEY] & [Output $INTAKE_KEY]
 
 ### Docker-compose file
 To ease the deployment, a `docker-compose.yml` file is suggested and a template is given.
diff --git a/template.j2 b/template.j2
index cb0c497..01b0d24 100644
--- a/template.j2
+++ b/template.j2
@@ -1,9 +1,10 @@
 input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}")
 
 {% if debug %}
-template(name="SEKOIAIO_{{ name |lower }}_Debug_Template" type="string" string="[DEBUG \"{{ intake_key }}\"] %rawmsg%\n")
+template(name="SEKOIAIO_{{ name |lower }}_Input_Template" type="string" string="[Input \"{{ intake_key }}\"] %rawmsg%\n")
+template(name="SEKOIAIO_{{ name |lower }}_Output_Template" type="string" string="[Output \"{{ intake_key }}\"] <%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
 {% endif %}
-template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg%\n")
+template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n")
 ruleset(name="remote{{ port }}"){
 action(
     type="omfwd"
@@ -22,13 +23,13 @@ action(
 action(
     type="omfile"
     file="/dev/stdout"
-    Template="SEKOIAIO_{{ name |lower }}_Debug_Template"
+    Template="SEKOIAIO_{{ name |lower }}_Input_Template"
     )
 action(
     type="omfile"
     file="/dev/stdout"
-    Template="SEKOIAIO_{{ name |lower }}_Template"
+    Template="SEKOIAIO_{{ name |lower }}_Output_Template"
     )
 {% endif %}
     
-}
+}
\ No newline at end of file

From 1d583ab476d84b86ccdb536f66a8c55fc1d5a33e Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Wed, 18 Oct 2023 17:20:33 +0200
Subject: [PATCH 09/12] Update version, add changelog

---
 .github/workflows/build-docker-image.yaml |  2 +-
 CHANGELOG.md                              | 13 +++++++++++++
 README.md                                 |  5 +++++
 docker-compose/docker-compose.yml         |  2 +-
 4 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 CHANGELOG.md

diff --git a/.github/workflows/build-docker-image.yaml b/.github/workflows/build-docker-image.yaml
index 4932dee..2d8a07e 100644
--- a/.github/workflows/build-docker-image.yaml
+++ b/.github/workflows/build-docker-image.yaml
@@ -31,5 +31,5 @@ jobs:
       with:
         push: true
         tags: |
-          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:1.0
+          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0
           ghcr.io/sekoia-io/sekoiaio-docker-concentrator:latest
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..f89f678
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,13 @@
+# Changelog
+
+All notable changes with sekoiaio concentrator will be documented in this file.
+
+## [2.0]
+
+- Manage syslog RFC 3164 (only 5424 in 1.0 version)
+- Add advanced debug options
+- Update implemention from bash to jinja
+
+## [1.0] 
+
+- Initial version
\ No newline at end of file
diff --git a/README.md b/README.md
index a9c2682..556bd6c 100644
--- a/README.md
+++ b/README.md
@@ -150,6 +150,11 @@ To view container logs:
 sudo docker compose logs
 ```
 
+To view container logs for a specific intake:
+```bash
+sudo docker compose logs | grep "YOUR_INTAKE_KEY"
+```
+
 To stop the container:
 ```bash
 sudo docker compose stop
diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml
index 2b53fc1..de7a67b 100644
--- a/docker-compose/docker-compose.yml
+++ b/docker-compose/docker-compose.yml
@@ -5,7 +5,7 @@ services:
       options:
         max-size: "1000m"
         max-file: "2"
-    image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:1.0
+    image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0
     environment:
       - MEMORY_MESSAGES=100000
       - DISK_SPACE=32g

From 3f1589f56aa513f35844faa7304d389b6c6a80d2 Mon Sep 17 00:00:00 2001
From: Jean GOUDY <jean.goudy@sekoia.io>
Date: Fri, 20 Oct 2023 11:13:32 +0200
Subject: [PATCH 10/12] feat(debug-improvements): modify debug variable

---
 template.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/template.j2 b/template.j2
index 01b0d24..2d44226 100644
--- a/template.j2
+++ b/template.j2
@@ -19,7 +19,7 @@ action(
     Template="SEKOIAIO_{{ name | lower }}_Template"
     )
 
-{% if debug %}
+{% if ( debug | lower ) == "true"  %}
 action(
     type="omfile"
     file="/dev/stdout"

From 131d23fabbb6c9eab822200fc4bbe397d043f639 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Fri, 20 Oct 2023 11:34:38 +0200
Subject: [PATCH 11/12] Removing latest tag maangement

---
 .github/workflows/build-docker-image.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build-docker-image.yaml b/.github/workflows/build-docker-image.yaml
index 2d8a07e..5183726 100644
--- a/.github/workflows/build-docker-image.yaml
+++ b/.github/workflows/build-docker-image.yaml
@@ -31,5 +31,4 @@ jobs:
       with:
         push: true
         tags: |
-          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0
-          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:latest
\ No newline at end of file
+          ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0
\ No newline at end of file

From 8bb9e79dc1e3291c73a8b2ed2ac46ba3cbd0f246 Mon Sep 17 00:00:00 2001
From: Pierre Penhouet <pierre.penhouet@sekoia.io>
Date: Fri, 20 Oct 2023 11:50:32 +0200
Subject: [PATCH 12/12] Print intake list

---
 CHANGELOG.md       | 2 +-
 generate_config.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f89f678..1fc27ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@ All notable changes with sekoiaio concentrator will be documented in this file.
 
 - Manage syslog RFC 3164 (only 5424 in 1.0 version)
 - Add advanced debug options
-- Update implemention from bash to jinja
+- Update implementation from bash to jinja
 
 ## [1.0] 
 
diff --git a/generate_config.py b/generate_config.py
index 052540f..6ed573e 100644
--- a/generate_config.py
+++ b/generate_config.py
@@ -13,6 +13,11 @@
 i=1
 # Generate one file per intake
 for item in data.get("intakes", []):
+    print("Intake name: " + str(item["name"].lower()))
+    print("Protocol: " + str(item["protocol"]))
+    print("Port: " + str(item["port"]))
+    print("Intake key: " + str(item["intake_key"]))
+    print("")
     config = template.render(item)
     filename = f"/etc/rsyslog.d/{i}_{item['name'].lower()}.conf"
     # Écrire le contenu généré dans le fichier