Merge pull request #139 from KPMP/develop

log aggregator release

ara/.env.example

@@ -4,3 +4,9 @@ ENV_FLUENTD_CONF_DIR=./fluentd/conf
 ENV_FLUENTD_LOG_DIR=./logs
 ENV_KIBANA_PASSWORD_FILE=./passwords
 
+# ElastAlert configuration
+ENV_ELASTALERT_CONF=./elastalert/conf/elastalert.yaml
+ENV_ELASTALERT_TEST_CONF=./elastalert/conf/elastalert-test.yaml
+ENV_ELASTALERT_SERVER_CONF=./elastalert/conf/config.json
+ENV_ELASTALERT_RULES_DIR=./elastalert/rules
+ENV_ELASTALERT_RULES_TEMPLATE_DIR=./elastalert/rule_templates

ara/.gitignore

@@ -0,0 +1 @@
+.passwords

ara/README.md

@@ -3,3 +3,5 @@
 As a stopgap, Ara is secured with a simple Apache basic_auth proxy.  The secure htpasswd file is stored in the KPMP secure files repository.
 
 Be sure to deploy the file to your environment and update your `.env` mapping accordingly.
+
+When deploying to PROD or QA, be sure to grab the config files from kpmp-secure for elastalert and put them in the 'elastalert/rules' directory. Only grab the config file appropriate for the environment you are deploying to.

ara/docker-compose.yml

@@ -103,7 +103,7 @@ services:
 
   fluentd:
     container_name: fluentd
-    image: kingstonduo/fluentd:1.0
+    image: kingstonduo/fluentd:latest
     volumes:
       - ${ENV_FLUENTD_CONF_DIR}:/fluentd/etc
       - ${ENV_FLUENTD_LOG_DIR}:/logs
@@ -114,6 +114,24 @@ services:
         aliases:
           - fluentd
 
+  elastalert:
+    container_name: elastalert
+    image: kingstonduo/elastalert
+    depends_on:
+      - es01
+      - es02
+      - es03
+    volumes:
+      - ${ENV_ELASTALERT_CONF}:/opt/elastalert/config.yaml
+      - ${ENV_ELASTALERT_TEST_CONF}:/opt/elastalert/config-test.yaml
+      - ${ENV_ELASTALERT_SERVER_CONF}:/opt/elastalert-server/config/config.json
+      - ${ENV_ELASTALERT_RULES_DIR}:/opt/elastalert/rules
+      - ${ENV_ELASTALERT_RULES_TEMPLATE_DIR}:/opt/elastalert/rule_templates
+    networks:
+      local:
+        aliases:
+          - elastalert
+
 volumes:
   esdata01:
     driver: local

ara/elastalert/conf/config.json

@@ -0,0 +1,20 @@
+{
+  "appName": "elastalert-server",
+  "port": 3030,
+  "wsport": 3333,
+  "elastalertPath": "/opt/elastalert",
+  "verbose": false,
+  "es_debug": false,
+  "debug": false,
+  "rulesPath": {
+    "relative": true,
+    "path": "/rules"
+  },
+  "templatesPath": {
+    "relative": true,
+    "path": "/rule_templates"
+  },
+  "es_host": "elasticsearch",
+  "es_port": 9200,
+  "writeback_index": "elastalert_status"
+}

ara/elastalert/conf/elastalert-test.yaml

@@ -0,0 +1,51 @@
+# NOTE: This config is used when testing a rule
+
+# The elasticsearch hostname for metadata writeback
+# Note that every rule can have its own elasticsearch host
+es_host: elasticsearch 
+
+# The elasticsearch port
+es_port: 9200
+
+# This is the folder that contains the rule yaml files
+# Any .yaml file will be loaded as a rule
+rules_folder: rules
+
+# How often ElastAlert will query elasticsearch
+# The unit can be anything from weeks to seconds
+run_every:
+  seconds: 5
+
+# ElastAlert will buffer results from the most recent
+# period of time, in case some log sources are not in real time
+buffer_time:
+  minutes: 1
+
+# Optional URL prefix for elasticsearch
+#es_url_prefix: elasticsearch
+
+# Connect with TLS to elasticsearch
+#use_ssl: True
+
+# Verify TLS certificates
+#verify_certs: True
+
+# GET request with body is the default option for Elasticsearch.
+# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
+# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
+# for details
+#es_send_get_body_as: GET
+
+# Option basic-auth username and password for elasticsearch
+#es_username: someusername
+#es_password: somepassword
+
+# The index on es_host which is used for metadata storage
+# This can be a unmapped index, but it is recommended that you run
+# elastalert-create-index to set a mapping
+writeback_index: elastalert_status
+
+# If an alert fails for some reason, ElastAlert will retry
+# sending the alert until this time period has elapsed
+alert_time_limit:
+  days: 2

ara/elastalert/conf/elastalert.yaml

@@ -0,0 +1,49 @@
+# The elasticsearch hostname for metadata writeback
+# Note that every rule can have its own elasticsearch host
+es_host: elasticsearch
+
+# The elasticsearch port
+es_port: 9200
+
+# This is the folder that contains the rule yaml files
+# Any .yaml file will be loaded as a rule
+rules_folder: rules
+
+# How often ElastAlert will query elasticsearch
+# The unit can be anything from weeks to seconds
+run_every:
+  seconds: 5
+
+# ElastAlert will buffer results from the most recent
+# period of time, in case some log sources are not in real time
+buffer_time:
+  minutes: 1
+
+# Optional URL prefix for elasticsearch
+#es_url_prefix: elasticsearch
+
+# Connect with TLS to elasticsearch
+#use_ssl: True
+
+# Verify TLS certificates
+#verify_certs: True
+
+# GET request with body is the default option for Elasticsearch.
+# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
+# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
+# for details
+#es_send_get_body_as: GET
+
+# Option basic-auth username and password for elasticsearch
+#es_username: someusername
+#es_password: somepassword
+
+# The index on es_host which is used for metadata storage
+# This can be a unmapped index, but it is recommended that you run
+# elastalert-create-index to set a mapping
+writeback_index: elastalert_status
+
+# If an alert fails for some reason, ElastAlert will retry
+# sending the alert until this time period has elapsed
+alert_time_limit:
+  days: 2

ara/elastalert/rule_templates/detection_template.yaml

@@ -0,0 +1,56 @@
+# Rule name, must be unique
+name: Alert on any detection
+
+# Index to search, wildcard supported
+index: bitsensor
+timestamp_field: endpoint.localtime
+
+# Type of alert.
+type: any
+realert: 
+  seconds: 0
+
+# A list of elasticsearch filters used for find events
+# These filters are joined with AND and nested in a filtered query
+# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html
+filter: 
+- query:
+    query_string:
+      query: "_exists_:detections"
+
+include:
+  - endpoint.location
+  - endpoint.name
+  - context.http.userAgent
+  - context.ip
+  - context.php.session.sessionId
+  - detections
+  - meta.user
+
+
+# Enhancement for converting 'detections' array into object, ex. get merged detection type by  
+# 'detections_parsed.type' or get first detection type by 'detection_parsed.0.type' 
+match_enhancements:   
+- "elastalert_modules.bitsensor_enhancement.AlertTextEnhancement"   
+run_enhancements_first: true 
+
+
+alert_subject: ":exclamation: Detection on {}"
+alert_subject_args:
+  - endpoint.name
+
+alert_text_type: alert_text_only
+alert_text: "Triggered at _{}_\n\n*Attacker:*\nIP: {} \nUser-Agent: {}\nDetection: `{}`\n\n:Id: {}\nUser: {}"
+alert_text_args:
+  - endpoint.localtime
+  - context.ip
+  - context.http.userAgent
+  - detections_parsed.type
+  - _id
+  - meta.user
+
+# The alert is use when a match is found
+alert:
+  - slack
+slack_webhook_url: "https://hooks.slack.com/services/"
+slack_username_override: "ElastAlert"

ara/elastalert/rule_templates/error_jira_template.yaml

@@ -0,0 +1,62 @@
+# Rule name, must be unique
+name: Alert on any error
+
+# Index to search, wildcard supported
+index: bitsensor
+timestamp_field: endpoint.localtime
+
+# Type of alert.
+type: any
+realert: 
+  seconds: 0
+
+# A list of elasticsearch filters used for find events
+# These filters are joined with AND and nested in a filtered query
+# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html
+filter: 
+- query:
+    query_string:
+      query: "_exists_:errors"
+
+include:
+  - endpoint.location
+  - endpoint.name
+  - context.http.userAgent
+  - context.ip
+  - errors
+  - meta.user
+
+
+# Enhancement for converting 'detections' array into object, ex. get merged detection type by  
+# 'detections_parsed.type' or get first detection type by 'detection_parsed.0.type' 
+match_enhancements:   
+- "elastalert_modules.bitsensor_enhancement.AlertTextEnhancement"   
+run_enhancements_first: true 
+
+
+alert_subject: "Error on {}"
+alert_subject_args:
+  - endpoint.name
+
+alert_text_type: alert_text_only
+alert_text: "Triggered at _{}_\n\n*Attacker:*\nIP: {} \nUser-Agent: {}\nError: *{}*\n\nId: {}\nUser: {}"
+alert_text_args:
+  - endpoint.localtime
+  - context.ip
+  - context.http.userAgent
+  - errors_parsed.type
+  - _id
+  - meta.user
+
+# The alert is use when a match is found
+alert:
+  - jira
+
+jira_server: https://bitsensor.atlassian.net
+jira_project: SA
+jira_issuetype: Story
+jira_labels: error
+
+# Add jira_acct.txt to rules folder
+# The file is yaml formatted and must contain fields: 'user', 'password'
+jira_account_file: "rules/jira_acct.txt"

ara/elastalert/rule_templates/integration_started_template.yaml

@@ -0,0 +1,54 @@
+# Rule name, must be unique
+name: Integration Started
+
+# Alert on x events in y seconds
+type: frequency
+
+# Alert when this many documents matching the query occur within a timeframe
+num_events: 1
+
+# num_events must occur within this amount of time to trigger an alert
+timeframe:
+  hours: 1
+
+# When the attacker continues, send a new alert after x minutes
+realert:
+  days: 7
+
+query_key:
+  - meta.provider
+  - endpoint.name
+
+include:
+  - meta.provider
+  - endpoint.name
+
+alert_subject: "Integration started on <{}> | <{}|Show Dashboard>"
+alert_subject_args:
+  - endpoint.name
+  - kibana_link
+
+alert_text: |-
+  Integration on {} has started with plugin {}.
+
+alert_text_args:
+  - endpoint.name
+  - meta.provider
+
+# The alert when a match is found
+alert:
+  - slack
+
+slack_webhook_url: "https://hooks.slack.com/services/"
+slack_username_override: "ElastAlert"
+
+# Alert body only cointains a title and text
+alert_text_type: alert_text_only
+
+# Link to BitSensor Kibana Dashboard
+use_kibana4_dashboard: "https://kibana.dashboard.io/app/kibana#/dashboard"
+
+# Index to search, wildcard supported
+index: bitsensor
+timestamp_field: endpoint.localtime
+doc_type: datapoint

ara/elastalert/rule_templates/no_data_template.yaml

@@ -0,0 +1,38 @@
+# Alert when no data has been received for more then 30 seconds.
+
+# Rule name, must be unique
+name: No Data
+
+# Type of alert.
+type: flatline
+
+# Alert when this many documents matching the query occur within a timeframe
+threshold: 1
+use_terms: true
+
+# num_events must occur within this amount of time to trigger an alert
+timeframe:
+  seconds: 30
+
+realert: 
+  minutes: 10
+
+exponential_realert: 
+  hours: 1
+
+doc_type: datapoint
+
+# Index to search, wildcard supported
+index: bitsensor
+timestamp_field: endpoint.localtime
+
+alert_subject: "No data on dashboard"
+
+alert_text_type: alert_text_only
+alert_text: "The stack receives no data. It might be down :("
+
+# The alert is use when a match is found
+alert:
+  - slack
+slack_webhook_url: "https://hooks.slack.com/services/"
+slack_username_override: "ElastAlert"