diff --git a/.github/workflows/contrib.yml b/.github/workflows/contrib.yml index 1cb3b773be..395675b433 100644 --- a/.github/workflows/contrib.yml +++ b/.github/workflows/contrib.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "Contributor Check" if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: cla-assistant/github-action@v2.1.3-beta + uses: cla-assistant/github-action@v2.3.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} diff --git a/pillar/top.sls b/pillar/top.sls index 53ec8a3306..ed3e492543 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -61,8 +61,6 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - curator.soc_curator - - curator.adv_curator - soctopus.soc_soctopus - soctopus.adv_soctopus - minions.{{ grains.id }} @@ -113,8 +111,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -172,8 +168,6 @@ base: - kibana.adv_kibana - strelka.soc_strelka - strelka.adv_strelka - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - zeek.soc_zeek @@ -194,8 +188,6 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - curator.soc_curator - - curator.adv_curator - redis.soc_redis - redis.adv_redis - zeek.soc_zeek @@ -268,8 +260,6 @@ base: - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - - curator.soc_curator - - curator.adv_curator - backup.soc_backup - backup.adv_backup - kratos.soc_kratos diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 1775e7aa10..f4fb94a2dc 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -220,10 +220,6 @@ {% do allowed_states.append('kibana.secrets') %} {% endif %} - {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} - {% do allowed_states.append('curator') %} - {% endif %} - {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %} {% do allowed_states.append('elastalert') %} {% endif %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 7e510e3ad6..7900b3c529 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -42,7 +42,6 @@ container_list() { ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( - "so-curator" "so-elastalert" "so-elastic-agent" "so-elastic-agent-builder" diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index d2582ff94b..282411ecca 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -144,6 +144,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # false positive (InfluxDB flux query, import nodes) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then diff --git a/salt/common/tools/sbin/so-nsm-clear b/salt/common/tools/sbin/so-nsm-clear index 3d9596238c..c9e5e86d2b 100755 --- a/salt/common/tools/sbin/so-nsm-clear +++ b/salt/common/tools/sbin/so-nsm-clear @@ -41,8 +41,13 @@ done if [ $SKIP -ne 1 ]; then # Inform user we are about to delete all data echo - echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)" - echo "If you would like to proceed, please type "AGREE" and hit ENTER." + echo "This script will delete all NSM data from /nsm." + echo + echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)." + echo + echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch." + echo + echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT @@ -54,8 +59,8 @@ delete_pcap() { [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start } delete_suricata() { - SURI_LOG="/opt/so/log/suricata/eve.json" - [ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start + SURI_LOG="/nsm/suricata/" + [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start } delete_zeek() { ZEEK_LOG="/nsm/zeek/logs/" diff --git a/salt/curator/config.sls b/salt/curator/config.sls deleted file mode 100644 index 89ff53b2a5..0000000000 --- a/salt/curator/config.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from "curator/map.jinja" import CURATORMERGED %} - -# Create the group -curatorgroup: - group.present: - - name: curator - - gid: 934 - -# Add user -curator: - user.present: - - uid: 934 - - gid: 934 - - home: /opt/so/conf/curator - - createhome: False - -# Create the log directory -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - -curactiondir: - file.directory: - - name: /opt/so/conf/curator/action - - user: 934 - - group: 939 - - makedirs: True - -actionconfs: - file.recurse: - - name: /opt/so/conf/curator/action - - source: salt://curator/files/action - - user: 934 - - group: 939 - - template: jinja - - defaults: - CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }} - -curconf: - file.managed: - - name: /opt/so/conf/curator/curator.yml - - source: salt://curator/files/curator.yml - - user: 934 - - group: 939 - - mode: 660 - - template: jinja - - show_changes: False - -curator_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin - - user: 934 - - group: 939 - - file_mode: 755 - -curator_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://curator/tools/sbin_jinja - - user: 934 - - group: 939 - - file_mode: 755 - - template: jinja - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml deleted file mode 100644 index b2be8d5edf..0000000000 --- a/salt/curator/defaults.yaml +++ /dev/null @@ -1,34 +0,0 @@ -curator: - enabled: False - elasticsearch: - index_settings: - so-beats: - delete: 365 - so-elasticsearch: - delete: 365 - so-firewall: - delete: 365 - so-ids: - delete: 365 - so-import: - delete: 73001 - so-kratos: - delete: 365 - so-kibana: - delete: 365 - so-logstash: - delete: 365 - so-netflow: - delete: 365 - so-osquery: - delete: 365 - so-ossec: - delete: 365 - so-redis: - delete: 365 - so-strelka: - delete: 365 - so-syslog: - delete: 365 - so-zeek: - delete: 365 diff --git a/salt/curator/disabled.sls b/salt/curator/disabled.sls deleted file mode 100644 index acf9e37011..0000000000 --- a/salt/curator/disabled.sls +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - curator.sostatus - -so-curator: - docker_container.absent: - - force: True - -so-curator_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.absent: - - identifier: so-curator-cluster-delete - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/enabled.sls b/salt/curator/enabled.sls deleted file mode 100644 index 916aa920d5..0000000000 --- a/salt/curator/enabled.sls +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'docker/docker.map.jinja' import DOCKER %} - -include: - - curator.config - - curator.sostatus - -so-curator: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }} - - start: True - - hostname: curator - - name: so-curator - - user: curator - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-curator'].ip }} - - interactive: True - - tty: True - - binds: - - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - - /opt/so/conf/curator/action/:/etc/curator/action:ro - - /opt/so/log/curator:/var/log/curator:rw - {% if DOCKER.containers['so-curator'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_hosts %} - - extra_hosts: - {% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - {% if DOCKER.containers['so-curator'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-curator'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - require: - - file: actionconfs - - file: curconf - - file: curlogdir - - watch: - - file: curconf - -delete_so-curator_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-curator$ - -so-curator-cluster-close: - cron.absent: - - identifier: so-curator-cluster-close - -so-curator-cluster-delete: - cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1 - - identifier: so-curator-cluster-delete - - user: root - - minute: '*/5' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml deleted file mode 100644 index 253c6fd674..0000000000 --- a/salt/curator/files/action/delete.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %} -{% set ELASTICMERGED = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) %} - -{{ ELASTICMERGED.retention_pct }} - -{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit') %} -actions: - 1: - action: delete_indices - description: >- - Delete indices when {{log_size_limit}}(GB) is exceeded. - options: - allow_ilm_indices: True - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-.*|so-.*|.ds-logs-.*-so.*)$' - - filtertype: pattern - kind: regex - value: '^(so-case.*)$' - exclude: True - - filtertype: space - source: creation_date - use_age: True - disk_space: {{log_size_limit}} diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml deleted file mode 100644 index c4e1f8b4e9..0000000000 --- a/salt/curator/files/action/so-beats-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-beats'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete beats indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-beats.*|so-beats.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml deleted file mode 100644 index 3c6bf4aacf..0000000000 --- a/salt/curator/files/action/so-elasticsearch-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-elasticsearch'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml deleted file mode 100644 index 5143e2fe95..0000000000 --- a/salt/curator/files/action/so-firewall-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-firewall'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete firewall indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-firewall.*|so-firewall.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml deleted file mode 100644 index 6cf120fef3..0000000000 --- a/salt/curator/files/action/so-ids-delete.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -{%- set DELETE_DAYS = CURATORMERGED['so-ids'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete IDS indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ids.*|so-ids.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml deleted file mode 100644 index 36e213b26f..0000000000 --- a/salt/curator/files/action/so-import-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-import'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-import.*|so-import.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml deleted file mode 100644 index 971a178fef..0000000000 --- a/salt/curator/files/action/so-kibana-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kibana'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kibana indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kibana.*|so-kibana.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-kratos-delete.yml b/salt/curator/files/action/so-kratos-delete.yml deleted file mode 100644 index d7cb2c4adf..0000000000 --- a/salt/curator/files/action/so-kratos-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-kratos'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete kratos indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-kratos.*|so-kratos.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml deleted file mode 100644 index 1ca1a6f6c6..0000000000 --- a/salt/curator/files/action/so-logstash-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-logstash'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete logstash indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-logstash.*|so-logstash.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml deleted file mode 100644 index 63adaa393e..0000000000 --- a/salt/curator/files/action/so-netflow-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-netflow'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete netflow indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-netflow.*|so-netflow.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml deleted file mode 100644 index b6263b0e84..0000000000 --- a/salt/curator/files/action/so-osquery-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-osquery'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-osquery.*|so-osquery.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml deleted file mode 100644 index e24fe38191..0000000000 --- a/salt/curator/files/action/so-ossec-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-ossec'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete ossec indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-ossec.*|so-ossec.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml deleted file mode 100644 index 1c7f95ded4..0000000000 --- a/salt/curator/files/action/so-redis-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-redis'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete redis indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-redis.*|so-redis.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml deleted file mode 100644 index 90cf88e46f..0000000000 --- a/salt/curator/files/action/so-strelka-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-strelka'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Strelka indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-strelka.*|so-strelka.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml deleted file mode 100644 index c11d2ef5aa..0000000000 --- a/salt/curator/files/action/so-syslog-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-syslog'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete syslog indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-syslog.*|so-syslog.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml deleted file mode 100644 index 1f85226961..0000000000 --- a/salt/curator/files/action/so-zeek-delete.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set DELETE_DAYS = CURATORMERGED['so-zeek'].delete %} -actions: - 1: - action: delete_indices - description: >- - Delete Zeek indices when older than {{ DELETE_DAYS }} days. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(logstash-zeek.*|so-zeek.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{ DELETE_DAYS }} - exclude: - - \ No newline at end of file diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml deleted file mode 100644 index 4ea1dddf70..0000000000 --- a/salt/curator/files/curator.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %} - {%- set elasticsearch = GLOBALS.node_ip -%} -{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} - {%- set elasticsearch = GLOBALS.manager_ip -%} -{%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} - ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -elasticsearch: - client: - hosts: - - https://{{elasticsearch}}:9200 - cloud_id: - ca_certs: - client_cert: - client_key: - verify_certs: False - request_timeout: 30 - other_settings: - api_key: - id: - api_key: - master_only: False - username: "{{ ES_USER }}" - password: "{{ ES_PASS }}" - -logging: - loglevel: INFO - logfile: '/var/log/curator/curator.log' - logformat: default - blacklist: ['elasticsearch', 'urllib3'] diff --git a/salt/curator/init.sls b/salt/curator/init.sls deleted file mode 100644 index 201195b609..0000000000 --- a/salt/curator/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'curator/map.jinja' import CURATORMERGED %} - -include: -{% if CURATORMERGED.enabled %} - - curator.enabled -{% else %} - - curator.disabled -{% endif %} diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja deleted file mode 100644 index 517209635b..0000000000 --- a/salt/curator/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %} -{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %} diff --git a/salt/curator/soc_curator.yaml b/salt/curator/soc_curator.yaml deleted file mode 100644 index acfba7f85d..0000000000 --- a/salt/curator/soc_curator.yaml +++ /dev/null @@ -1,39 +0,0 @@ -curator: - enabled: - description: You can enable or disable Curator. - helpLink: curator.html - elasticsearch: - index_settings: - so-beats: - delete: &delete - description: Age, in days, when Curator deletes the index. - helpLink: curator.html - forcedType: int - so-elasticsearch: - delete: *delete - so-firewall: - delete: *delete - so-ids: - delete: *delete - so-import: - delete: *delete - so-kratos: - delete: *delete - so-kibana: - delete: *delete - so-logstash: - delete: *delete - so-netflow: - delete: *delete - so-osquery: - delete: *delete - so-ossec: - delete: *delete - so-redis: - delete: *delete - so-strelka: - delete: *delete - so-syslog: - delete: *delete - so-zeek: - delete: *delete diff --git a/salt/curator/sostatus.sls b/salt/curator/sostatus.sls deleted file mode 100644 index de6459a6d7..0000000000 --- a/salt/curator/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-curator_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-curator - - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/curator/tools/sbin/so-curator-delete b/salt/curator/tools/sbin/so-curator-delete deleted file mode 100644 index 2d128bfdfb..0000000000 --- a/salt/curator/tools/sbin/so-curator-delete +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -APP=delete -lf=/tmp/$APP-pidLockFile -# create empty lock file if none exists -cat /dev/null >> $lf -read lastPID < $lf -# if lastPID is not null and a process with that pid exists , exit -[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit -echo $$ > $lf - -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1 diff --git a/salt/curator/tools/sbin/so-curator-restart b/salt/curator/tools/sbin/so-curator-restart deleted file mode 100644 index f57e7b22ee..0000000000 --- a/salt/curator/tools/sbin/so-curator-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart curator $1 diff --git a/salt/curator/tools/sbin/so-curator-start b/salt/curator/tools/sbin/so-curator-start deleted file mode 100644 index c5f0fc4d1f..0000000000 --- a/salt/curator/tools/sbin/so-curator-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start curator $1 diff --git a/salt/curator/tools/sbin/so-curator-stop b/salt/curator/tools/sbin/so-curator-stop deleted file mode 100644 index 30fb07e4bd..0000000000 --- a/salt/curator/tools/sbin/so-curator-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop curator $1 diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index a5d6c5d6d5..9a27843ae1 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -159,11 +159,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-curator': - final_octet: 43 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-elastic-fleet-package-registry': final_octet: 44 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index d227a3e85b..850324a9e8 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -60,7 +60,6 @@ docker: so-strelka-gatekeeper: *dockerOptions so-strelka-coordinator: *dockerOptions so-elastalert: *dockerOptions - so-curator: *dockerOptions so-elastic-fleet-package-registry: *dockerOptions so-idh: *dockerOptions so-elastic-agent: *dockerOptions diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 7d38838958..2c0fcb816f 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -76,6 +76,7 @@ elasticfleet: - pulse_connect_secure - redis - sentinel_one + - snort - snyk - sonicwall_firewall - sophos @@ -85,9 +86,12 @@ elasticfleet: - tcp - tenable_sc - ti_abusech + - ti_anomali + - ti_cybersixgill - ti_misp - ti_otx - ti_recordedfuture + - ti_threatq - udp - vsphere - windows diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 0979f98b63..492db03dc0 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -27,6 +27,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 32bff857b0..5462dc8611 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -28,6 +28,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json index 8feedc879e..0116ed9367 100644 --- a/salt/elasticfleet/files/integrations-optional/sublime_platform.json +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -39,6 +39,7 @@ } } } - } + }, + "force": true } {%- endraw -%} diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json index f1774b905f..b3d6c4609c 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/osquery.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json index b6a21249a5..699c07ec37 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/system-endpoints.json @@ -73,5 +73,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json index b17986a533..8e98137418 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-endpoints.json @@ -63,5 +63,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json index 4c22f92ee6..a2aaf5e0aa 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json @@ -102,5 +102,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 29cc1a8798..db4b1a8f6b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index baa8683aed..3b1a50560e 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index 3b8cffcc1e..c9b036e36d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json index b1fb710778..6a67c9c1c4 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json index 0349c9fc3f..5527607bd1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json index 6b9cbffaf0..b822421eb0 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json @@ -72,5 +72,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index 3aa7408814..aa39c177b7 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index 840f36f6be..cc92092e9b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 60ee95f454..61ad057f4f 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index b789adc1d5..a875e4bfca 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 089b5d4f81..89e9bbe8e1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index a9d857b24f..c3b04fd862 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -25,5 +25,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json index 80baa45ca8..4088f5a877 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json @@ -28,5 +28,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json index 653c788b59..ad32a69643 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json @@ -29,5 +29,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json index a5c4c3e819..8e6bf7958e 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index b1454d4bde..b1d69a44fd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -16,5 +16,6 @@ } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 3df514f0bc..6c42086bcd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -36,5 +36,6 @@ "system-system/metrics": { "enabled": false } - } + }, + "force": true } diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 02c2529a60..66916acd1d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -6737,6 +6737,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-snort_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-snort.log-*" + template: + settings: + index: + lifecycle: + name: so-logs-snort.log-logs + number_of_replicas: 0 + composed_of: + - "logs-snort.log@package" + - "logs-snort.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-snyk_x_audit: index_sorting: false index_template: @@ -7622,6 +7666,94 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-ti_anomali_x_threatstream: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_anomali.threatstream-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_anomali.threatstream-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_anomali.threatstream@package" + - "logs-ti_anomali.threatstream@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-ti_cybersixgill_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_cybersixgill.threat-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_cybersixgill.threat-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_cybersixgill.threat@package" + - "logs-ti_cybersixgill.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-ti_misp_x_threat: index_sorting: false index_template: @@ -7842,6 +7974,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-ti_threatq_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_threatq.threat-*" + template: + settings: + index: + lifecycle: + name: so-logs-ti_threatq.threat-logs + number_of_replicas: 0 + composed_of: + - "logs-ti_threatq.threat@package" + - "logs-ti_threatq.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-vsphere_x_log: index_sorting: False index_template: diff --git a/salt/elasticsearch/disabled.sls b/salt/elasticsearch/disabled.sls index 210ad59a4e..2453f9f773 100644 --- a/salt/elasticsearch/disabled.sls +++ b/salt/elasticsearch/disabled.sls @@ -18,6 +18,10 @@ so-elasticsearch_so-status.disabled: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-elasticsearch$ +so-elasticsearch-indices-delete: + cron.absent: + - identifier: so-elasticsearch-indices-delete + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index f7ab7749f0..818b0666c5 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -26,7 +26,12 @@ so-elasticsearch: - networks: - sobridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - - extra_hosts: {{ LOGSTASH_NODES }} + - extra_hosts: + {% for node in LOGSTASH_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-elasticsearch'].extra_hosts %} {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %} - {{ XTRAHOST }} @@ -195,6 +200,26 @@ so-elasticsearch-roles-load: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +so-curator-cluster-close: + cron.absent: + - identifier: so-curator-cluster-close + +so-curator-cluster-delete: + cron.absent: + - identifier: so-curator-cluster-delete + +so-elasticsearch-indices-delete: + cron.present: + - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 + - identifier: so-elasticsearch-indices-delete + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' +{% endif %} {% endif %} {% else %} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 9d9b3d294e..0b93a6c1a6 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -407,16 +407,20 @@ elasticsearch: so-logs-sentinel_one_x_group: *indexSettings so-logs-sentinel_one_x_threat: *indexSettings so-logs-sonicwall_firewall_x_log: *indexSettings + so-logs-snort_x_log: *indexSettings so-logs-symantec_endpoint_x_log: *indexSettings so-logs-ti_abusech_x_malware: *indexSettings so-logs-ti_abusech_x_malwarebazaar: *indexSettings so-logs-ti_abusech_x_threatfox: *indexSettings so-logs-ti_abusech_x_url: *indexSettings + so-logs-ti_anomali_x_threatstream: *indexSettings + so-logs-ti_cybersixgill_x_threat: *indexSettings so-logs-ti_misp_x_threat: *indexSettings so-logs-ti_misp_x_threat_attributes: *indexSettings so-logs-ti_otx_x_threat: *indexSettings so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings so-logs-ti_recordedfuture_x_threat: *indexSettings + so-logs-ti_threatq_x_threat: *indexSettings so-logs-zscaler_zia_x_alerts: *indexSettings so-logs-zscaler_zia_x_dns: *indexSettings so-logs-zscaler_zia_x_firewall: *indexSettings diff --git a/salt/elasticsearch/tools/sbin/so-elastic-clear b/salt/elasticsearch/tools/sbin/so-elastic-clear index 085327fc47..af7b91ce2c 100755 --- a/salt/elasticsearch/tools/sbin/so-elastic-clear +++ b/salt/elasticsearch/tools/sbin/so-elastic-clear @@ -67,7 +67,7 @@ if [ $SKIP -ne 1 ]; then echo echo "This script will delete all data (documents, indices, etc.) in the Elasticsearch database." echo - echo "If you would like to proceed, please type "AGREE" and hit ENTER." + echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT diff --git a/salt/curator/tools/sbin/so-curator-cluster-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete similarity index 92% rename from salt/curator/tools/sbin/so-curator-cluster-delete rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete index 0f7945b78a..036ff844fc 100755 --- a/salt/curator/tools/sbin/so-curator-cluster-delete +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-delete @@ -14,4 +14,4 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -/usr/sbin/so-curator-cluster-delete-delete +/usr/sbin/so-elasticsearch-indices-delete-delete diff --git a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete similarity index 88% rename from salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete index e0c5144bca..07feb36bdf 100755 --- a/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-indices-delete-delete @@ -9,8 +9,8 @@ {%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%} {%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%} -LOG="/opt/so/log/curator/so-curator-cluster-delete.log" -ALERT_LOG="/opt/so/log/curator/alert.log" +LOG="/opt/so/log/elasticsearch/so-elasticsearch-indices-delete.log" +ALERT_LOG="/opt/so/log/elasticsearch/indices-delete-alert.log" LOG_SIZE_LIMIT_GB=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}}) LOG_SIZE_LIMIT=$(( "$LOG_SIZE_LIMIT_GB" * 1000 * 1000 * 1000 )) ITERATION=0 @@ -51,8 +51,11 @@ while overlimit && [[ $ITERATION -lt $MAX_ITERATIONS ]]; do if [ "${INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then # This should not be a write index, so we should be allowed to delete it printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - Deleting ${INDEX} index...\n" >> ${LOG} - /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 + /usr/sbin/so-elasticsearch-query ${INDEX} -XDELETE >> ${LOG} 2>&1 fi + else + printf "\n$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT_GB} GB) - There is only one backing index (${INDEX}). Deleting ${DATASTREAM} data stream...\n" >> ${LOG} + /usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM -XDELETE >> ${LOG} 2>&1 fi if ! overlimit ; then exit diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 02e8a46446..0ba2389e91 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -2,7 +2,6 @@ {% if GLOBALS.role == 'so-eval' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -25,7 +24,6 @@ {% elif GLOBALS.role == 'so-manager' or GLOBALS.role == 'so-standalone' or GLOBALS.role == 'so-managersearch' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-dockerregistry', 'so-elasticsearch', 'so-elastic-fleet', @@ -56,7 +54,6 @@ {% elif GLOBALS.role == 'so-heavynode' %} {% set NODE_CONTAINERS = [ - 'so-curator', 'so-elasticsearch', 'so-elastic-agent', 'so-logstash', diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 4d6a688e45..7333c78e98 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -80,7 +80,7 @@ logrotate: - extension .log - dateext - dateyesterday - /opt/so/log/curator/*_x_log: + /opt/so/log/elasticsearch/*indices-delete*_x_log: - daily - rotate 14 - missingok diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 5e6c78fcc3..62aa935c96 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -56,23 +56,23 @@ logrotate: multiline: True global: True forcedType: "[]string" - "/opt/so/log/curator/*_x_log": + "/opt/so/log/elasticfleet/*_x_log": description: List of logrotate options for this file. - title: /opt/so/log/curator/*.log + title: /opt/so/log/elasticfleet/*.log advanced: True multiline: True global: True forcedType: "[]string" - "/opt/so/log/elasticfleet/*_x_log": + "/opt/so/log/elasticfleet/*_x_ndjson": description: List of logrotate options for this file. - title: /opt/so/log/elasticfleet/*.log + title: /opt/so/log/elasticfleet/*.ndjson advanced: True multiline: True global: True forcedType: "[]string" - "/opt/so/log/elasticfleet/*_x_ndjson": + "/opt/so/log/elasticsearch/*indices-delete*_x_log": description: List of logrotate options for this file. - title: /opt/so/log/elasticfleet/*.ndjson + title: /opt/so/log/elasticsearch/*indices-delete.log advanced: True multiline: True global: True diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index c76f81d214..5337351fce 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -33,11 +33,16 @@ so-logstash: - sobridge: - ipv4_address: {{ DOCKER.containers['so-logstash'].ip }} - user: logstash - - extra_hosts: {{ REDIS_NODES }} + - extra_hosts: + {% for node in REDIS_NODES %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} {% if DOCKER.containers['so-logstash'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-logstash'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} + {% endfor %} {% endif %} - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} diff --git a/salt/manager/tools/sbin/so-elastic-auth-password-reset b/salt/manager/tools/sbin/so-elastic-auth-password-reset index 5bc5230c3e..b6bfd87213 100755 --- a/salt/manager/tools/sbin/so-elastic-auth-password-reset +++ b/salt/manager/tools/sbin/so-elastic-auth-password-reset @@ -97,8 +97,6 @@ function soUserSync() { salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True printf "\nApplying kibana state to the appropriate nodes.\n\n" salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True - printf "\nApplying curator state to the appropriate nodes.\n\n" - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply curator queue=True } function highstateManager() { diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 1baf88cad9..d5225cc827 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -9,23 +9,23 @@ if [ -f /usr/sbin/so-common ]; then . /usr/sbin/so-common fi -if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" +function usage() { + echo "Usage: $0 -o= -m=[id]" + echo "" + echo " where is one of the following:" + echo "" + echo " add: Accepts a new key and adds the minion files" + echo " delete: Removes the key and deletes the minion files" + echo " list: Lists all keys with hashes" + echo " reject: Rejects a key" + echo " restart: Restart a minion (reboot)" + echo " test: Perform minion test" + echo "" exit 1 -fi +} if [[ $# -lt 1 ]]; then - echo "Usage: $0 -o= -m=[id]" - echo "" - echo " where is one of the following:" - echo "" - echo " list: Lists all keys with hashes" - echo " accept: Accepts a new key and adds the minion files" - echo " delete: Removes the key and deletes the minion files" - echo " reject: Rejects a key" - echo " test: Perform minion test" - echo "" - exit 1 + usage fi for i in "$@"; do @@ -38,6 +38,8 @@ for i in "$@"; do MINION_ID="${i#*=}" shift ;; + + # The following args are used internally during setup, not to be specified manually. -e=*|--esheap=*) ES_HEAP_SIZE="${i#*=}" shift @@ -63,6 +65,7 @@ for i in "$@"; do exit 1 ;; *) + usage ;; esac done @@ -76,7 +79,7 @@ function getinstallinfo() { source <(echo $INSTALLVARS) } -function testminion() { +function testMinion() { # Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone. # Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image. so-test @@ -92,12 +95,19 @@ function testminion() { exit $result } -function listminions() { +function restartMinion() { + salt "$MINION_ID" system.reboot + result=$? + + exit $result +} + +function listMinions() { salt-key list -F --out=json exit $? } -function rejectminion() { +function rejectMinion() { salt-key -y -r $MINION_ID exit $? } @@ -106,11 +116,11 @@ function acceptminion() { salt-key -y -a $MINION_ID } -function deleteminion() { +function deleteMinion() { salt-key -y -d $MINION_ID } -function deleteminionfiles () { +function deleteMinionFiles () { rm -f $PILLARFILE rm -f $ADVPILLARFILE } @@ -292,13 +302,6 @@ function add_strelka_to_minion() { " " >> $PILLARFILE } -function add_curator_to_minion() { - printf '%s\n'\ - "curator:"\ - " enabled: True"\ - " " >> $PILLARFILE -} - function add_telegraf_to_minion() { printf '%s\n'\ "telegraf:"\ @@ -425,7 +428,6 @@ function createEVAL() { add_playbook_to_minion add_elastalert_to_minion add_kibana_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -448,7 +450,6 @@ function createSTANDALONE() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -468,7 +469,6 @@ function createMANAGER() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -488,7 +488,6 @@ function createMANAGERSEARCH() { add_elastalert_to_minion add_kibana_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion add_influxdb_to_minion add_nginx_to_minion @@ -538,7 +537,6 @@ function createHEAVYNODE() { add_sensor_to_minion add_strelka_to_minion add_redis_to_minion - add_curator_to_minion add_telegraf_to_minion } @@ -589,25 +587,33 @@ function testConnection() { fi } -if [[ "$OPERATION" = 'list' ]]; then - listminions -fi - -if [[ "$OPERATION" = 'delete' ]]; then - deleteminionfiles - deleteminion -fi - -if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then - # Skip this if its setup - if [[ $OPERATION == 'add' ]]; then - # Accept the salt key - acceptminion - # Test to see if the minion was accepted - testConnection - # Pull the info from the file to build what is needed - getinstallinfo +function addMinion() { + # Accept the salt key + acceptminion + # Test to see if the minion was accepted + testConnection + # Pull the info from the file to build what is needed + getinstallinfo +} + +function updateMineAndApplyStates() { + # tell the minion to populate the mine with data from mine_functions which is populated during setup + # this only needs to happen on non managers since they handle this during setup + # and they need to wait for ca creation to update the mine + updateMine + checkMine "network.ip_addrs" + # apply the elasticsearch state to the manager if a new searchnode was added + if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then + # calls so-common and set_minionid sets MINIONID to local minion id + set_minionid + salt $MINIONID state.apply elasticsearch queue=True --async + salt $MINIONID state.apply soc queue=True --async fi + # run this async so the cli doesn't wait for a return + salt "$MINION_ID" state.highstate --async queue=True +} + +function setupMinionFiles() { # Check to see if nodetype is set if [ -z $NODETYPE ]; then echo "No node type specified" @@ -624,25 +630,41 @@ if [[ "$OPERATION" == 'add' || "$OPERATION" == 'setup' ]]; then create$NODETYPE echo "Minion file created for $MINION_ID" +} - if [[ "$OPERATION" == 'add' ]]; then - # tell the minion to populate the mine with data from mine_functions which is populated during setup - # this only needs to happen on non managers since they handle this during setup - # and they need to wait for ca creation to update the mine - updateMine - checkMine "network.ip_addrs" - # apply the elasticsearch state to the manager if a new searchnode was added - if [[ "$NODETYPE" == "SEARCHNODE" || "$NODETYPE" == "HEAVYNODE" ]]; then - # calls so-common and set_minionid sets MINIONID to local minion id - set_minionid - salt $MINIONID state.apply elasticsearch queue=True --async - salt $MINIONID state.apply soc queue=True --async - fi - # run this async so the cli doesn't wait for a return - salt "$MINION_ID" state.highstate --async queue=True - fi -fi +case "$OPERATION" in + "add") + addMinion + setupMinionFiles + updateMineAndApplyStates + ;; -if [[ "$OPERATION" = 'test' ]]; then - testminion -fi + "delete") + deleteMinionFiles + deleteMinion + ;; + + "list") + listMinions + ;; + + "reject") + rejectMinion + ;; + + "restart") + restartMinion + ;; + + "setup") + # only should be invoked directly during setup, never manually + setupMinionFiles + ;; + + "test") + testMinion + ;; + *) + usage + ;; +esac diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 35c9347729..4a0adbc989 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -465,9 +465,12 @@ post_to_2.4.30() { } post_to_2.4.40() { - echo "Removing Curator close and delete files" - rm -f /opt/so/conf/curator/action/*-close.y*ml - rm -f /opt/so/conf/curator/action/logs*-delete.y*ml + echo "Stopping Curator" + so-curator-stop + echo "Removing Curator configuration" + rm -rf /opt/so/conf/curator/ + rm -f /usr/sbin/so-curator-* + sed -i '/so-curator/d' /opt/so/conf/so-status/so-status.conf } repo_sync() { diff --git a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset index 564156af91..f1112c7233 100644 --- a/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset +++ b/salt/manager/tools/sbin_jinja/so-elastic-fleet-reset @@ -18,7 +18,7 @@ echo "This includes data previously ingested with Fleet such as Zeek and Suricat echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled." echo "This script should only be used as a last resort to reinstall Elastic Fleet." echo -echo "If you would like to proceed, type AGREE and hit ENTER." +echo "If you would like to proceed, then type AGREE and press ENTER." echo # Read user input read INPUT @@ -93,4 +93,4 @@ so-elastic-fleet-setup status "Re-installing Elastic Agent on all Grid Nodes..." salt \* state.apply elasticfleet.install_agent_grid queue=True -status "Elastic Fleet Reset complete...." \ No newline at end of file +status "Elastic Fleet Reset complete...." diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 3655019e49..f440bd57b6 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -470,6 +470,18 @@ soc: - rule.action - rule.reason - network.community_id + ':pfsense:': + - soc_timestamp + - source.ip + - source.port + - destination.ip + - destination.port + - network.transport + - network.direction + - observer.ingress.interface.name + - event.action + - event.reason + - network.community_id ':osquery:': - soc_timestamp - source.ip @@ -1348,7 +1360,7 @@ soc: showSubtitle: true - name: Firewall description: Firewall events grouped by action - query: 'tags:firewall | groupby rule.action' + query: 'observer.type:firewall | groupby event.action' showSubtitle: true dashboards: advanced: true @@ -1551,7 +1563,7 @@ soc: query: 'tags:s7* | groupby -sankey event.dataset source.ip destination.ip | groupby event.dataset | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Firewall description: Firewall logs - query: 'tags:firewall | groupby -sankey rule.action interface.name | groupby rule.action | groupby interface.name | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' + query: 'observer.type:firewall | groupby -sankey event.action observer.ingress.interface.name | groupby event.action | groupby observer.ingress.interface.name | groupby network.type | groupby network.transport | groupby source.ip | groupby destination.ip | groupby destination.port' - name: VLAN description: VLAN (Virtual Local Area Network) tagged logs query: '* AND _exists_:network.vlan.id | groupby network.vlan.id | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby event.dataset | groupby event.module | groupby observer.name | groupby source.geo.country_name | groupby destination.geo.country_name' diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 9e588e5b1e..2661587f48 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -33,21 +33,26 @@ so-soc: - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw - /opt/so/saltstack:/opt/so/saltstack:rw - - extra_hosts: {{ DOCKER_EXTRA_HOSTS }} - {% if DOCKER.containers['so-soc'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} + - extra_hosts: + {% for node in DOCKER_EXTRA_HOSTS %} + {% for hostname, ip in node.items() %} + - {{hostname}}:{{ip}} + {% endfor %} + {% endfor %} + {% if DOCKER.containers['so-soc'].extra_hosts %} + {% for XTRAHOST in DOCKER.containers['so-soc'].extra_hosts %} - {{ XTRAHOST }} - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} - port_bindings: - {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} + {% for BINDING in DOCKER.containers['so-soc'].port_bindings %} - {{ BINDING }} - {% endfor %} + {% endfor %} {% if DOCKER.containers['so-soc'].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-soc'].extra_env %} - {{ XTRAENV }} - {% endfor %} + {% endfor %} {% endif %} - watch: - file: /opt/so/conf/soc/* diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 0b7d532b1b..fa9d9b2b92 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -14,6 +14,7 @@ telegraf: - checkfiles.sh - influxdbsize.sh - oldpcap.sh + - os.sh - raid.sh - sostatus.sh - stenoloss.sh @@ -25,6 +26,7 @@ telegraf: - eps.sh - influxdbsize.sh - oldpcap.sh + - os.sh - raid.sh - redis.sh - sostatus.sh @@ -34,20 +36,25 @@ telegraf: - zeekloss.sh manager: - influxdbsize.sh + - os.sh - raid.sh - redis.sh - sostatus.sh managersearch: - eps.sh - influxdbsize.sh + - os.sh - raid.sh - redis.sh - sostatus.sh import: + - influxdbsize.sh + - os.sh - sostatus.sh sensor: - checkfiles.sh - oldpcap.sh + - os.sh - raid.sh - sostatus.sh - stenoloss.sh @@ -58,6 +65,7 @@ telegraf: - checkfiles.sh - eps.sh - oldpcap.sh + - os.sh - raid.sh - redis.sh - sostatus.sh @@ -66,17 +74,22 @@ telegraf: - zeekcaptureloss.sh - zeekloss.sh idh: + - os.sh - sostatus.sh searchnode: - eps.sh + - os.sh - raid.sh - sostatus.sh receiver: - eps.sh + - os.sh - raid.sh - redis.sh - sostatus.sh fleet: + - os.sh - sostatus.sh desktop: + - os.sh - sostatus.sh diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 45b1283e01..1c58016451 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -186,7 +186,7 @@ # # # Read stats from one or more Elasticsearch servers or clusters -{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %} +{%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-import'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = true diff --git a/salt/telegraf/scripts/os.sh b/salt/telegraf/scripts/os.sh new file mode 100644 index 0000000000..a067806a13 --- /dev/null +++ b/salt/telegraf/scripts/os.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# if this script isn't already running +if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then + + NEEDS_RESTART=0 + + if which needs-restarting &> /dev/null; then + # DNF/RPM family + if ! needs-restarting -r &> /dev/null; then + NEEDS_RESTART=1 + fi + else + # APT family + if [ -f /var/run/reboot-required ]; then + NEEDS_RESTART=1 + fi + fi + + echo "os restart=$NEEDS_RESTART" + +fi + +exit 0 diff --git a/salt/top.sls b/salt/top.sls index 4f84e17ac7..5e7d13532b 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -104,7 +103,6 @@ base: - redis - elastic-fleet-package-registry - kibana - - curator - elastalert - utility - soctopus @@ -140,7 +138,6 @@ base: - suricata - zeek - strelka - - curator - elastalert - utility - soctopus @@ -179,7 +176,6 @@ base: - elasticsearch - logstash - redis - - curator - elastic-fleet-package-registry - kibana - elastalert @@ -199,7 +195,6 @@ base: - elasticsearch - logstash - redis - - curator - strelka - pcap - suricata diff --git a/setup/so-functions b/setup/so-functions index 6ad0947d16..e1faa275e6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1412,7 +1412,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert global;do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls @@ -1555,7 +1555,6 @@ reserve_group_ids() { logCmd "groupadd -g 931 logstash" logCmd "groupadd -g 932 kibana" logCmd "groupadd -g 933 elastalert" - logCmd "groupadd -g 934 curator" logCmd "groupadd -g 937 zeek" logCmd "groupadd -g 940 suricata" logCmd "groupadd -g 941 stenographer" diff --git a/setup/so-variables b/setup/so-variables index 7f65224877..511dfc43bd 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -112,12 +112,6 @@ export sensoroni_pillar_file adv_sensoroni_pillar_file="$local_salt_dir/pillar/sensoroni/adv_sensoroni.sls" export adv_sensoroni_pillar_file -curator_pillar_file="$local_salt_dir/pillar/curator/soc_curator.sls" -export curator_pillar_file - -adv_curator_pillar_file="$local_salt_dir/pillar/curator/adv_curator.sls" -export adv_curator_pillar_file - soctopus_pillar_file="$local_salt_dir/pillar/soctopus/soc_soctopus.sls" export soctopus_pillar_file diff --git a/setup/so-verify b/setup/so-verify index 61f5f15825..92b6c0e815 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -33,13 +33,17 @@ log_has_errors() { # Ignore Failed: 0 since that is the salt state output, and we detect state failures # via Result: False already. - # This is ignored for Ubuntu + # This is ignored for Ubuntu: # Failed to restart snapd.mounts-pre.target: Operation refused, unit snapd.mounts-pre.target # may be requested by dependency only (it is configured to refuse manual start/stop). # Command failed with exit code is output during retry loops. # "remove failed" is caused by a warning generated by upgrade of libwbclient + + # Exit code 100 failure is likely apt-get running in the background, we wait for it to unlock. + + # Failed to deduce dest mapping appears to occur when a shard isn't yet ready. Temporary. grep -E "FAILED|Failed|failed|ERROR|Result: False|Error is not recoverable" "$setup_log" | \ grep -vE "The Salt Master has cached the public key for this node" | \ @@ -58,6 +62,7 @@ log_has_errors() { grep -vE "remove failed" | \ grep -vE "Failed to restart snapd" | \ grep -vE "Login Failed Details" | \ + grep -vE "Failed to deduce dest mappings" | \ grep -vE "response from daemon: unauthorized" | \ grep -vE "Reading first line of patchfile" | \ grep -vE "Command failed with exit code" | \