diff --git a/.github/workflows/build-docker-image.yaml b/.github/workflows/build-docker-image.yaml index 4932dee..5183726 100644 --- a/.github/workflows/build-docker-image.yaml +++ b/.github/workflows/build-docker-image.yaml @@ -31,5 +31,4 @@ jobs: with: push: true tags: | - ghcr.io/sekoia-io/sekoiaio-docker-concentrator:1.0 - ghcr.io/sekoia-io/sekoiaio-docker-concentrator:latest \ No newline at end of file + ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1fc27ba --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +All notable changes with sekoiaio concentrator will be documented in this file. + +## [2.0] + +- Manage syslog RFC 3164 (only 5424 in 1.0 version) +- Add advanced debug options +- Update implementation from bash to jinja + +## [1.0] + +- Initial version \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d6a82f1..ca6eab1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y \ gettext-base \ python3 \ python3-yaml \ + python3-jinja2 \ wget RUN wget -O /SEKOIA-IO-intake.pem https://app.sekoia.io/assets/files/SEKOIA-IO-intake.pem @@ -17,11 +18,11 @@ ENV MEMORY_MESSAGES=100000 # Setting up Rsyslog RUN rm -rf /etc/rsyslog.d/50-default.conf -COPY parse_yaml.py parse_yaml.py +COPY generate_config.py generate_config.py COPY rsyslog.conf rsyslog.conf COPY entrypoint.sh entrypoint.sh COPY intakes.yaml intakes.yaml -COPY template.conf template.conf +COPY template.j2 template.j2 RUN chmod +x entrypoint.sh diff --git a/README.md b/README.md index 90e4115..556bd6c 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,28 @@ intakes: intake_key: INTAKE_KEY_FOR_TECHNO_3 ``` +#### Debug +A debug variable is available in order to debug a specific intake, for example +```yaml +--- +intakes: +- name: Techno1 + protocol: tcp + port: 20516 + intake_key: INTAKE_KEY_FOR_TECHNO_1 +- name: Techno2 + protocol: tcp + port: 20517 + intake_key: INTAKE_KEY_FOR_TECHNO_2 + debug: True +- name: Techno3 + protocol: tcp + port: 20518 + intake_key: INTAKE_KEY_FOR_TECHNO_3 +``` + +By using this key, the raw received message and the output message will be printed in the console. Each one will be respectively identified using tags: : [Input $INTAKE_KEY] & [Output $INTAKE_KEY] + ### Docker-compose file To ease the deployment, a `docker-compose.yml` file is suggested and a template is given. @@ -128,6 +150,11 @@ To view container logs: sudo docker compose logs ``` +To view container logs for a specific intake: +```bash +sudo docker compose logs | grep "YOUR_INTAKE_KEY" +``` + To stop the container: ```bash sudo docker compose stop diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 2b53fc1..de7a67b 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -5,7 +5,7 @@ services: options: max-size: "1000m" max-file: "2" - image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:1.0 + image: ghcr.io/sekoia-io/sekoiaio-docker-concentrator:2.0 environment: - MEMORY_MESSAGES=100000 - DISK_SPACE=32g diff --git a/entrypoint.sh b/entrypoint.sh index 60945d2..a9d91ac 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,7 +9,7 @@ echo "-----------------------------" envsubst '${DISK_SPACE} ${MEMORY_MESSAGES}' /etc/rsyslog.conf # Parse yaml intake file -python3 parse_yaml.py +python3 generate_config.py ret=$? if [ $ret -ne 0 ]; then # If the the YAML is not as expected @@ -17,21 +17,4 @@ if [ $ret -ne 0 ]; then exit 1 fi -i=1 -while IFS=";" read -r rec_column1 rec_column2 rec_column3 rec_column4 || [ -n "$rec_column4" ] -do - intake_name=$(echo "${rec_column1// /-}" | awk '{print tolower($0)}') - protocol=$(echo "$rec_column2" | awk '{print tolower($0)}') - port=$rec_column3 - intake_key=$rec_column4 - - intake_name=$intake_name protocol=$protocol port=$port intake_key=${intake_key//[$'\t\r\n']} envsubst /etc/rsyslog.d/$i-$intake_name.conf - i=$(($i+1)) - echo "Intake name: $intake_name" - echo "Protocol: $protocol" - echo "Port: $port" - echo "Intake key: $intake_key" - echo "" -done < intakes.csv - exec "$@" diff --git a/generate_config.py b/generate_config.py new file mode 100644 index 0000000..6ed573e --- /dev/null +++ b/generate_config.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +import yaml +from jinja2 import Environment, FileSystemLoader + +# Open input config file +with open("intakes.yaml", "r") as fyaml: + data = yaml.safe_load(fyaml) + +# Load jinja template +template = Environment(loader=FileSystemLoader(".")).get_template("template.j2") + +i=1 +# Generate one file per intake +for item in data.get("intakes", []): + print("Intake name: " + str(item["name"].lower())) + print("Protocol: " + str(item["protocol"])) + print("Port: " + str(item["port"])) + print("Intake key: " + str(item["intake_key"])) + print("") + config = template.render(item) + filename = f"/etc/rsyslog.d/{i}_{item['name'].lower()}.conf" + # Écrire le contenu généré dans le fichier + with open(filename, "w") as f: + f.write(config) + i=i+1 + diff --git a/parse_yaml.py b/parse_yaml.py deleted file mode 100644 index cb0b294..0000000 --- a/parse_yaml.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import yaml -import csv - -with open("intakes.yaml", "r") as fyaml: - data = yaml.safe_load(fyaml) - with open('intakes.csv', 'w') as fcsv: - csvwriter = csv.writer(fcsv, delimiter=';') - for intake in data['intakes']: - csvwriter.writerow([intake['name'], intake['protocol'], intake['port'], intake['intake_key']]) diff --git a/rsyslog.conf b/rsyslog.conf index 6990af7..2a9c74f 100644 --- a/rsyslog.conf +++ b/rsyslog.conf @@ -9,7 +9,7 @@ global( defaultNetstreamDriverCAFile="/SEKOIA-IO-intake.pem" - maxMessageSize="64k" + maxMessageSize="250k" umask="0022" workDirectory="/var/spool/rsyslog" ) diff --git a/template.conf b/template.conf deleted file mode 100644 index df5739a..0000000 --- a/template.conf +++ /dev/null @@ -1,23 +0,0 @@ -input(type="im${protocol}" port="${port}" ruleset="remote${port}") - -template(name="SEKOIAIO_${intake_name}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"${intake_key}\"] %msg%\n") -ruleset(name="remote${port}"){ -action( - type="omfwd" - protocol="tcp" - target="intake.sekoia.io" - port="10514" - TCP_Framing="octet-counted" - StreamDriver="gtls" - StreamDriverMode="1" - StreamDriverAuthMode="x509/name" - StreamDriverPermittedPeers="intake.sekoia.io" - Template="SEKOIAIO_${intake_name}_Template" - ) - -action( - type="omfile" - file="/dev/stdout" - Template="SEKOIAIO_${intake_name}_Template" - ) -} diff --git a/template.j2 b/template.j2 new file mode 100644 index 0000000..2d44226 --- /dev/null +++ b/template.j2 @@ -0,0 +1,35 @@ +input(type="im{{ protocol | lower }}" port="{{ port }}" ruleset="remote{{ port }}") + +{% if debug %} +template(name="SEKOIAIO_{{ name |lower }}_Input_Template" type="string" string="[Input \"{{ intake_key }}\"] %rawmsg%\n") +template(name="SEKOIAIO_{{ name |lower }}_Output_Template" type="string" string="[Output \"{{ intake_key }}\"] <%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n") +{% endif %} +template(name="SEKOIAIO_{{ name |lower }}_Template" type="string" string="<%pri%>1 %timestamp:::date-rfc3339% %hostname% %app-name% %procid% LOG [SEKOIA@53288 intake_key=\"{{ intake_key }}\"] %msg:R,ERE,1,FIELD:^[ \t]*(.*)$--end%\n") +ruleset(name="remote{{ port }}"){ +action( + type="omfwd" + protocol="tcp" + target="intake.sekoia.io" + port="10514" + TCP_Framing="octet-counted" + StreamDriver="gtls" + StreamDriverMode="1" + StreamDriverAuthMode="x509/name" + StreamDriverPermittedPeers="intake.sekoia.io" + Template="SEKOIAIO_{{ name | lower }}_Template" + ) + +{% if ( debug | lower ) == "true" %} +action( + type="omfile" + file="/dev/stdout" + Template="SEKOIAIO_{{ name |lower }}_Input_Template" + ) +action( + type="omfile" + file="/dev/stdout" + Template="SEKOIAIO_{{ name |lower }}_Output_Template" + ) +{% endif %} + +} \ No newline at end of file