diff --git a/docker-compose-elk-basic.yml b/docker/docker-compose-elk-basic.yml similarity index 74% rename from docker-compose-elk-basic.yml rename to docker/docker-compose-elk-basic.yml index 9fb512f3..e45b643d 100644 --- a/docker-compose-elk-basic.yml +++ b/docker/docker-compose-elk-basic.yml @@ -2,13 +2,15 @@ version: '3' services: helk-elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1 + image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 container_name: helk-elasticsearch volumes: - ./helk-elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - esdata:/usr/share/elasticsearch/data + - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts + entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh environment: - - "ES_JAVA_OPTS=-Xms6g -Xmx6g" + - "ES_JAVA_OPTS=-Xms4g -Xmx4g" ulimits: memlock: soft: -1 @@ -19,15 +21,17 @@ services: aliases: - helk_elasticsearch.hunt.local helk-logstash: - image: docker.elastic.co/logstash/logstash:6.3.1 + image: docker.elastic.co/logstash/logstash:6.3.2 container_name: helk-logstash volumes: - ./helk-logstash/logstash.yml:/usr/share/logstash/config/logstash.yml - ./helk-logstash/pipeline:/usr/share/logstash/pipeline - ./helk-logstash/output_templates:/usr/share/logstash/output_templates - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti + - ./helk-logstash/scripts:/usr/share/logstash/scripts environment: - - "LS_JAVA_OPTS=-Xms2g -Xmx2g" + - "LS_JAVA_OPTS=-Xms1g -Xmx1g" + entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh restart: always depends_on: - helk-elasticsearch @@ -36,7 +40,7 @@ services: aliases: - helk_logstash.hunt.local helk-kibana: - image: docker.elastic.co/kibana/kibana:6.3.1 + image: docker.elastic.co/kibana/kibana:6.3.2 container_name: helk-kibana volumes: - ./helk-kibana/kibana.yml:/usr/share/kibana/config/kibana.yml @@ -51,13 +55,16 @@ services: aliases: - helk_kibana.hunt.local helk-nginx: - image: cyb3rward0g/helk-nginx:0.0.3 + image: cyb3rward0g/helk-nginx:0.0.6 container_name: helk-nginx volumes: - ./helk-nginx/htpasswd.users:/etc/nginx/htpasswd.users - ./helk-nginx/default:/etc/nginx/sites-available/default + - ./helk-nginx/scripts/:/opt/helk/scripts/ + entrypoint: /opt/helk/scripts/nginx-entrypoint.sh ports: - "80:80" + - "443:443" restart: always depends_on: - helk-kibana @@ -65,12 +72,24 @@ services: helk: aliases: - helk_nginx.hunt.local + helk-jupyter: + image: cyb3rward0g/helk-jupyter:0.0.4 + container_name: helk-jupyter + restart: always + depends_on: + - helk-nginx + networks: + helk: + aliases: + - helk_jupyter.hunt.local helk-spark-master: - image: cyb3rward0g/helk-spark-master:2.3.1 + image: cyb3rward0g/helk-spark-master:2.3.1-a container_name: helk-spark-master + environment: + - SPARK_MASTER_PORT=7077 + - SPARK_MASTER_WEBUI_PORT=8080 ports: - "8080:8080" - - "7077:7077" restart: always depends_on: - helk-elasticsearch @@ -79,11 +98,13 @@ services: aliases: - helk_spark_master.hunt.local helk-spark-worker: - image: cyb3rward0g/helk-spark-worker:2.3.1 + image: cyb3rward0g/helk-spark-worker:2.3.1-a container_name: helk-spark-worker environment: - - SPARK_WORKER_MEMORY=1g + - SPARK_MASTER=spark://helk-spark-master:7077 + - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8081 + - SPARK_WORKER_PORT=42950 ports: - "8081:8081" restart: always @@ -94,11 +115,13 @@ services: aliases: - helk_spark_worker.hunt.local helk-spark-worker2: - image: cyb3rward0g/helk-spark-worker:2.3.1 + image: cyb3rward0g/helk-spark-worker:2.3.1-a container_name: helk-spark-worker2 environment: - - SPARK_WORKER_MEMORY=1g + - SPARK_MASTER=spark://helk-spark-master:7077 + - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8082 + - SPARK_WORKER_PORT=42951 ports: - "8082:8082" restart: always @@ -108,33 +131,20 @@ services: helk: aliases: - helk_spark_worker2.hunt.local - helk-jupyter: - image: cyb3rward0g/helk-jupyter:0.0.2 - container_name: helk-jupyter - ports: - - "8880:8880" - - "4040-4050:4040-4050" - restart: always - depends_on: - - helk-kibana - networks: - helk: - aliases: - - helk_jupyter.hunt.local helk-zookeeper: - image: cyb3rward0g/helk-zookeeper:3.4.10 + image: cyb3rward0g/helk-zookeeper:1.1.1 container_name: helk-zookeeper ports: - "2181:2181" restart: always depends_on: - - helk-elasticsearch + - helk-kibana networks: helk: aliases: - helk_zookeeper.hunt.local helk-kafka-broker: - image: cyb3rward0g/helk-kafka-broker:1.1.0 + image: cyb3rward0g/helk-kafka-broker:1.1.1 container_name: helk-kafka-broker restart: always depends_on: @@ -153,7 +163,7 @@ services: aliases: - helk_kafka_broker.hunt.local helk-kafka-broker2: - image: cyb3rward0g/helk-kafka-broker:1.1.0 + image: cyb3rward0g/helk-kafka-broker:1.1.1 container_name: helk-kafka-broker2 restart: always depends_on: @@ -170,7 +180,7 @@ services: networks: helk: aliases: - - helk_kafka_broker.hunt.local + - helk_kafka_broker2.hunt.local helk-sigma: image: thomaspatzke/helk-sigma container_name: helk-sigma diff --git a/docker-compose-elk-trial.yml b/docker/docker-compose-elk-trial.yml similarity index 76% rename from docker-compose-elk-trial.yml rename to docker/docker-compose-elk-trial.yml index b9f93cec..78300e1c 100644 --- a/docker-compose-elk-trial.yml +++ b/docker/docker-compose-elk-trial.yml @@ -7,8 +7,10 @@ services: volumes: - ./helk-elasticsearch/trial/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - esdata:/usr/share/elasticsearch/data + - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts + entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh environment: - - "ES_JAVA_OPTS=-Xms6g -Xmx6g" + - "ES_JAVA_OPTS=-Xms4g -Xmx4g" ulimits: memlock: soft: -1 @@ -19,15 +21,17 @@ services: aliases: - helk_elasticsearch.hunt.local helk-logstash: - image: docker.elastic.co/logstash/logstash:6.3.1 + image: docker.elastic.co/logstash/logstash:6.3.2 container_name: helk-logstash volumes: - ./helk-logstash/trial/logstash.yml:/usr/share/logstash/config/logstash.yml - ./helk-logstash/trial/pipeline:/usr/share/logstash/pipeline - ./helk-logstash/output_templates:/usr/share/logstash/output_templates - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti + - ./helk-logstash/trial/scripts:/usr/share/logstash/scripts environment: - - "LS_JAVA_OPTS=-Xms2g -Xmx2g" + - "LS_JAVA_OPTS=-Xms1g -Xmx1g" + entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh restart: always depends_on: - helk-elasticsearch @@ -36,7 +40,7 @@ services: aliases: - helk_logstash.hunt.local helk-kibana: - image: docker.elastic.co/kibana/kibana:6.3.1 + image: docker.elastic.co/kibana/kibana:6.3.2 container_name: helk-kibana volumes: - ./helk-kibana/trial/kibana.yml:/usr/share/kibana/config/kibana.yml @@ -51,12 +55,15 @@ services: aliases: - helk_kibana.hunt.local helk-nginx: - image: cyb3rward0g/helk-nginx:0.0.3 + image: cyb3rward0g/helk-nginx:0.0.6 container_name: helk-nginx volumes: - ./helk-nginx/trial/default:/etc/nginx/sites-available/default + - ./helk-nginx/scripts/:/opt/helk/scripts/ + entrypoint: /opt/helk/scripts/nginx-entrypoint.sh ports: - "80:80" + - "443:443" restart: always depends_on: - helk-kibana @@ -64,12 +71,24 @@ services: helk: aliases: - helk_nginx.hunt.local + helk-jupyter: + image: cyb3rward0g/helk-jupyter:0.0.4 + container_name: helk-jupyter + restart: always + depends_on: + - helk-nginx + networks: + helk: + aliases: + - helk_jupyter.hunt.local helk-spark-master: - image: cyb3rward0g/helk-spark-master:2.3.1 + image: cyb3rward0g/helk-spark-master:2.3.1-a container_name: helk-spark-master + environment: + - SPARK_MASTER_PORT=7077 + - SPARK_MASTER_WEBUI_PORT=8080 ports: - "8080:8080" - - "7077:7077" restart: always depends_on: - helk-elasticsearch @@ -78,11 +97,13 @@ services: aliases: - helk_spark_master.hunt.local helk-spark-worker: - image: cyb3rward0g/helk-spark-worker:2.3.1 + image: cyb3rward0g/helk-spark-worker:2.3.1-a container_name: helk-spark-worker environment: - - SPARK_WORKER_MEMORY=1g + - SPARK_MASTER=spark://helk-spark-master:7077 + - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8081 + - SPARK_WORKER_PORT=42950 ports: - "8081:8081" restart: always @@ -93,11 +114,13 @@ services: aliases: - helk_spark_worker.hunt.local helk-spark-worker2: - image: cyb3rward0g/helk-spark-worker:2.3.1 + image: cyb3rward0g/helk-spark-worker:2.3.1-a container_name: helk-spark-worker2 environment: - - SPARK_WORKER_MEMORY=1g + - SPARK_MASTER=spark://helk-spark-master:7077 + - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8082 + - SPARK_WORKER_PORT=42951 ports: - "8082:8082" restart: always @@ -107,21 +130,8 @@ services: helk: aliases: - helk_spark_worker2.hunt.local - helk-jupyter: - image: cyb3rward0g/helk-jupyter:0.0.2 - container_name: helk-jupyter - ports: - - "8880:8880" - - "4040-4050:4040-4050" - restart: always - depends_on: - - helk-kibana - networks: - helk: - aliases: - - helk_jupyter.hunt.local helk-zookeeper: - image: cyb3rward0g/helk-zookeeper:3.4.10 + image: cyb3rward0g/helk-zookeeper:1.1.1 container_name: helk-zookeeper ports: - "2181:2181" @@ -133,7 +143,7 @@ services: aliases: - helk_zookeeper.hunt.local helk-kafka-broker: - image: cyb3rward0g/helk-kafka-broker:1.1.0 + image: cyb3rward0g/helk-kafka-broker:1.1.1 container_name: helk-kafka-broker restart: always depends_on: @@ -152,7 +162,7 @@ services: aliases: - helk_kafka_broker.hunt.local helk-kafka-broker2: - image: cyb3rward0g/helk-kafka-broker:1.1.0 + image: cyb3rward0g/helk-kafka-broker:1.1.1 container_name: helk-kafka-broker2 restart: always depends_on: diff --git a/helk-base/Dockerfile b/docker/helk-base/Dockerfile similarity index 100% rename from helk-base/Dockerfile rename to docker/helk-base/Dockerfile diff --git a/helk-elasticsearch/Dockerfile b/docker/helk-elasticsearch/Dockerfile similarity index 80% rename from helk-elasticsearch/Dockerfile rename to docker/helk-elasticsearch/Dockerfile index 1af6b9d8..46e59989 100644 --- a/helk-elasticsearch/Dockerfile +++ b/docker/helk-elasticsearch/Dockerfile @@ -1,12 +1,12 @@ # HELK script: HELK Elasticsearch Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.1 +# HELK ELK version: 6.3.2 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 # References: # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html -FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.1 +FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Elasticsearch." \ No newline at end of file diff --git a/helk-elasticsearch/elasticsearch.yml b/docker/helk-elasticsearch/elasticsearch.yml similarity index 100% rename from helk-elasticsearch/elasticsearch.yml rename to docker/helk-elasticsearch/elasticsearch.yml diff --git a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh new file mode 100755 index 00000000..427799e0 --- /dev/null +++ b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# HELK script: elasticsearch-entrypoint.sh +# HELK script description: sets elasticsearch configs and starts elasticsearch +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# *********** Looking for ES *************** +if [[ ! -z "$ES_JAVA_OPTS" ]]; then + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting ES_JAVA_OPTS to $ES_JAVA_OPTS" +else + # ****** Setup heap size and memory locking ***** + ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo) + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting ES_HEAP_SIZE to ${ES_MEMORY}.." + export ES_JAVA_OPTS="-Xms${ES_MEMORY}g -Xmx${ES_MEMORY}g" +fi + +# ********** Starting Elasticsearch ***************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.." +/usr/local/bin/docker-entrypoint.sh \ No newline at end of file diff --git a/helk-elasticsearch/trial/Dockerfile b/docker/helk-elasticsearch/trial/Dockerfile similarity index 80% rename from helk-elasticsearch/trial/Dockerfile rename to docker/helk-elasticsearch/trial/Dockerfile index 22e36376..d906c24c 100644 --- a/helk-elasticsearch/trial/Dockerfile +++ b/docker/helk-elasticsearch/trial/Dockerfile @@ -1,15 +1,13 @@ # HELK script: HELK Elasticsearch Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.1 +# HELK ELK version: 6.3.2 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 # References: # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html -# *********** ELK Version *************** - -FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.1 +FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Elasticsearch." diff --git a/helk-elasticsearch/trial/elasticsearch.yml b/docker/helk-elasticsearch/trial/elasticsearch.yml similarity index 100% rename from helk-elasticsearch/trial/elasticsearch.yml rename to docker/helk-elasticsearch/trial/elasticsearch.yml diff --git a/helk-jupyter/Dockerfile b/docker/helk-jupyter/Dockerfile similarity index 50% rename from helk-jupyter/Dockerfile rename to docker/helk-jupyter/Dockerfile index a69483ef..7e851c61 100644 --- a/helk-jupyter/Dockerfile +++ b/docker/helk-jupyter/Dockerfile @@ -14,10 +14,10 @@ USER root # *********** Installing Prerequisites *************** # -qq : No output except for errors RUN echo "[HELK-DOCKER-INSTALLATION-INFO] Extracting templates from packages.." \ - && apt-get install -qqy \ - python3-pip \ - python-tk \ - unzip + && apt-get install -qqy --no-install-recommends \ + curl python3-pip python3-dev python-tk unzip python3-setuptools \ + libcurl4-openssl-dev build-essential libssl-dev libffi-dev \ + libxml2-dev libxslt1-dev zlib1g-dev RUN apt-get -qy clean \ autoremove @@ -27,55 +27,41 @@ RUN pip3 install --upgrade pip # *********** Installing Jupyter Hub Prerequisites RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - -RUN apt-get install -y nodejs +RUN apt-get install -y --no-install-recommends nodejs # *********** Installing HELK python packages *************** RUN pip3 install pandas \ jupyter \ - jupyterlab \ - jupyterhub + jupyterlab==0.33.4 \ + jupyterhub==0.9.1 +# *********** Installing Jupyter Lab Extension - JupyterHub *************** RUN npm install -g configurable-http-proxy -RUN jupyter labextension install @jupyterlab/hub-extension +RUN jupyter labextension install @jupyterlab/hub-extension@0.10.0 -# *********** Creating the right directories *************** -RUN bash -c 'mkdir -pv /opt/helk/{es-hadoop,jupyter}' +# *********** Creating the Jupyter directories *************** +RUN bash -c 'mkdir -pv /opt/helk/{es-hadoop,jupyter,jupyterhub}' +RUN mkdir -v /usr/local/share/jupyter/kernels/pyspark3 +RUN mkdir -v /var/log/spark -# *********** Setting Jupyterhub*********************** +# *********** Configure Jupyterhub *************** ENV JUPYTER_DIR=/opt/helk/jupyter # *********** Adding HELK scripts and files to Container *************** -ADD scripts/jupyter-entrypoint.sh ${JUPYTER_DIR} -ADD notebooks ${JUPYTER_DIR}/notebooks +COPY scripts/jupyter-entrypoint.sh ${JUPYTER_DIR} +COPY notebooks ${JUPYTER_DIR}/notebooks +COPY spark/* ${SPARK_HOME}/conf/ +COPY kernels/pyspark_kernel.json /usr/local/share/jupyter/kernels/pyspark3/kernel.json +COPY jupyterhub/jupyterhub_config.py /opt/helk/jupyter/ # *********** Download ES-Hadoop *************** -ENV ESHADOOP_VERSION=6.3.1 +ENV ESHADOOP_VERSION=6.3.2 RUN wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-${ESHADOOP_VERSION}.zip -P /opt/helk/es-hadoop/ \ && unzip -j /opt/helk/es-hadoop/*.zip -d /opt/helk/es-hadoop/ \ && rm /opt/helk/es-hadoop/*.zip -# *********** Configure Jupyterhub *************** -ENV JUPYTER_LOGS_PATH=${JUPYTER_DIR}/log -ENV JUPYTER_CONSOLE_LOG=${JUPYTER_LOGS_PATH}/jupyter.log -ENV JUPYTER_EXEC=$SPARK_HOME/bin/pyspark -ENV JUPYTER_LOGS=">> $JUPYTER_CONSOLE_LOG 2>&1" - -RUN mkdir -v $JUPYTER_LOGS_PATH -ADD spark/log4j.properties ${SPARK_HOME}/conf/ -ADD spark/spark-defaults.conf ${SPARK_HOME}/conf/ - -# *********** Update Jupyter PySpark Kernel ************* -#ADD kernels/pyspark_kernel.json /usr/local/share/jupyter/kernels/python3/kernel.json - -# ************* Adding SPARK environment variables ************* -ENV PATH=$SPARK_HOME/bin:$PATH -ENV PYSPARK_PYTHON=/usr/bin/python3 -ENV PYSPARK_DRIVER_PYTHON=/usr/local/bin/jupyter -ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip -ENV PYSPARK_DRIVER_PYTHON_OPTS="lab --no-browser --ip=* --port=8880 --allow-root --notebook-dir=/opt/helk/jupyter/notebooks" - -EXPOSE 4040 8880 - +EXPOSE 8000 # *********** RUN HELK *************** WORKDIR ${JUPYTER_DIR} -ENTRYPOINT ["./jupyter-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["./jupyter-entrypoint.sh"] +CMD ["/bin/bash","-c","/usr/local/bin/jupyterhub","-f","/opt/helk/jupyter/jupyterhub_config.py"] \ No newline at end of file diff --git a/docker/helk-jupyter/jupyterhub/jupyterhub_config.py b/docker/helk-jupyter/jupyterhub/jupyterhub_config.py new file mode 100644 index 00000000..5cb145ec --- /dev/null +++ b/docker/helk-jupyter/jupyterhub/jupyterhub_config.py @@ -0,0 +1,17 @@ +# HELK script: HELK JupyterHub Config +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +#c = get_config() + +c.JupyterHub.log_level = 10 + +c.Authenticator.whitelist = {'hunter1','hunter2','hunter3'} +c.Authenticator.admin_users = {'hunter1'} +c.Spawner.cmd = ['jupyter-labhub'] +c.Spawner.notebook_dir = '/opt/helk/jupyterhub' + +c.JupyterHub.hub_ip = 'helk-jupyter' +c.JupyterHub.port = 8000 +c.JupyterHub.base_url = '/jupyter' \ No newline at end of file diff --git a/docker/helk-jupyter/kernels/pyspark_kernel.json b/docker/helk-jupyter/kernels/pyspark_kernel.json new file mode 100644 index 00000000..0ae67c6c --- /dev/null +++ b/docker/helk-jupyter/kernels/pyspark_kernel.json @@ -0,0 +1,17 @@ +{ + "display_name": "PySpark_Python3", + "language": "python", + "argv": [ + "/usr/bin/python3", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}" + ], + "env": { + "SPARK_HOME": "/opt/helk/spark/", + "PYTHONPATH": "/opt/helk/spark/python/:/opt/helk/spark/python/lib/py4j-0.10.7-src.zip", + "PYTHONSTARTUP": "/opt/helk/spark/python/pyspark/shell.py", + "PYSPARK_PYTHON": "/usr/bin/python3" + } +} \ No newline at end of file diff --git a/helk-jupyter/notebooks/basic/Graphframes_Basic_Sysmon_ProcessCreate.ipynb b/docker/helk-jupyter/notebooks/basic/Graphframes_Basic_Sysmon_ProcessCreate.ipynb similarity index 100% rename from helk-jupyter/notebooks/basic/Graphframes_Basic_Sysmon_ProcessCreate.ipynb rename to docker/helk-jupyter/notebooks/basic/Graphframes_Basic_Sysmon_ProcessCreate.ipynb diff --git a/helk-jupyter/notebooks/basic/Spark_Initial_Checks.ipynb b/docker/helk-jupyter/notebooks/basic/Spark_Initial_Checks.ipynb similarity index 100% rename from helk-jupyter/notebooks/basic/Spark_Initial_Checks.ipynb rename to docker/helk-jupyter/notebooks/basic/Spark_Initial_Checks.ipynb diff --git a/helk-jupyter/notebooks/trial/Graphframes_Basic_Sysmon_ProcessCreate.ipynb b/docker/helk-jupyter/notebooks/trial/Graphframes_Basic_Sysmon_ProcessCreate.ipynb similarity index 100% rename from helk-jupyter/notebooks/trial/Graphframes_Basic_Sysmon_ProcessCreate.ipynb rename to docker/helk-jupyter/notebooks/trial/Graphframes_Basic_Sysmon_ProcessCreate.ipynb diff --git a/helk-jupyter/notebooks/trial/Spark_Initial_Checks.ipynb b/docker/helk-jupyter/notebooks/trial/Spark_Initial_Checks.ipynb similarity index 100% rename from helk-jupyter/notebooks/trial/Spark_Initial_Checks.ipynb rename to docker/helk-jupyter/notebooks/trial/Spark_Initial_Checks.ipynb diff --git a/docker/helk-jupyter/scripts/jupyter-entrypoint.sh b/docker/helk-jupyter/scripts/jupyter-entrypoint.sh new file mode 100755 index 00000000..c459fe49 --- /dev/null +++ b/docker/helk-jupyter/scripts/jupyter-entrypoint.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# HELK script: jupyter-entryppoint.sh +# HELK script description: Creates JupyterHub Users +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# ************* Creating JupyterHub Users *************** +declare -a users_index=("hunter1" "hunter2" "hunter3") + +JUPYTERHUB_GID=711 +JUPYTERHUB_UID=711 +JUPYTERHUB_HOME=/opt/helk/jupyterhub +JUPYTER_HOME=/opt/helk/jupyter + +echo "[HELK-JUPYTER-DOCKER-INSTALLATION-INFO] Creating JupyterHub Group..." +groupadd -g ${JUPYTERHUB_GID} jupyterhub + +for u in ${users_index[@]}; do + echo "[HELK-JUPYTER-DOCKER-INSTALLATION-INFO] Creating JupyterHub user ${u} .." + student_password="${u}P@ssw0rd!" + echo $student_password >> /opt/helk/user_credentials.txt + + JUPYTERHUB_USER_DIRECTORY=${JUPYTERHUB_HOME}/${u} + mkdir -v $JUPYTERHUB_USER_DIRECTORY + + useradd -p $(openssl passwd -1 ${student_password}) -u ${JUPYTERHUB_UID} -g ${JUPYTERHUB_GID} -d $JUPYTERHUB_USER_DIRECTORY --no-create-home -s /bin/bash ${u} + + echo "[HELK-JUPYTER-DOCKER-INSTALLATION-INFO] copying notebooks to ${JUPYTERHUB_USER_DIRECTORY} notebooks directory ..." + cp -R ${JUPYTER_HOME}/notebooks ${JUPYTERHUB_USER_DIRECTORY}/notebooks + chown -R ${u}:jupyterhub $JUPYTERHUB_USER_DIRECTORY + chmod 700 -R $JUPYTERHUB_USER_DIRECTORY + + ((JUPYTERHUB_UID=$JUPYTERHUB_UID + 1)) +done + +chmod 777 -R /var/log/spark +chmod 777 -R /opt/helk/spark + +exec "$@" diff --git a/helk-jupyter/spark/log4j.properties b/docker/helk-jupyter/spark/log4j.properties similarity index 99% rename from helk-jupyter/spark/log4j.properties rename to docker/helk-jupyter/spark/log4j.properties index 9aa76e12..65ef1894 100644 --- a/helk-jupyter/spark/log4j.properties +++ b/docker/helk-jupyter/spark/log4j.properties @@ -37,4 +37,4 @@ log4j.logger.parquet=ERROR # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL -log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR \ No newline at end of file +log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR diff --git a/docker/helk-jupyter/spark/spark-defaults.conf b/docker/helk-jupyter/spark/spark-defaults.conf new file mode 100644 index 00000000..ba8f2ba6 --- /dev/null +++ b/docker/helk-jupyter/spark/spark-defaults.conf @@ -0,0 +1,39 @@ +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# HELK References: +# https://spark.apache.org/docs/latest/configuration.html +# https://graphframes.github.io/quick-start.html +# https://spark-packages.org/package/graphframes/graphframes +# https://spark.apache.org/docs/latest/sql-programming-guide.html#pyspark-usage-guide-for-pandas-with-apache-arrow + +# ************ Application Properties **************** +# Logs the effective SparkConf as INFO when a SparkContext is started. Default: false +spark.logConf true +# The cluster manager to connect to. +spark.master spark://helk-spark-master:7077 +# Restarts the driver automatically if it fails with a non-zero exit status +spark.driver.supervise true + +# ************ Runtime Environment **************** +# Sets the number of latest rolling log files that are going to be retained by the system. Older log files will be deleted. +spark.executor.logs.rolling.maxRetainedFiles 20 +# Set the strategy of rolling of executor logs. +spark.executor.logs.rolling.strategy spark.executor.logs.rolling.time.interval +# Comma-separated list of jars to include on the driver and executor classpaths. Globs are allowed. +spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.3.2.jar +# Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths. +# The coordinates should be groupId:artifactId:version. +spark.jars.packages graphframes:graphframes:0.5.0-spark2.1-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0,databricks:spark-sklearn:0.2.3 + +# ************ Spark UI **************** +# Base directory in which Spark events are logged +spark.eventLog.dir /var/log/spark +# Whether to log Spark events, useful for reconstructing the Web UI after the application has finished. +spark.eventLog.enabled true +# Enable running Spark Master as reverse proxy for worker and application UIs. +# In this mode, Spark master will reverse proxy the worker and application UIs to enable access without requiring direct access to their hosts. +spark.ui.reverseProxy true + +spark.sql.execution.arrow.enabled true \ No newline at end of file diff --git a/helk-kafka-base/Dockerfile b/docker/helk-kafka-base/Dockerfile similarity index 85% rename from helk-kafka-base/Dockerfile rename to docker/helk-kafka-base/Dockerfile index 5acd2285..5ebdecfc 100644 --- a/helk-kafka-base/Dockerfile +++ b/docker/helk-kafka-base/Dockerfile @@ -24,10 +24,10 @@ RUN apt-get -qy clean \ RUN bash -c 'mkdir -pv /opt/helk/kafka' # *********** Install Kafka *************** -ENV KAFKA_VERSION=1.1.0 +ENV KAFKA_VERSION=1.1.1 ENV KAFKA_LOGS_PATH=/var/log/kafka ENV KAFKA_CONSOLE_LOG=/var/log/kafka/helk-kafka.log ENV KAFKA_HOME=/opt/helk/kafka/kafka_2.11-${KAFKA_VERSION} -RUN wget -qO- http://mirrors.ocf.berkeley.edu/apache/kafka/1.1.0/kafka_2.11-${KAFKA_VERSION}.tgz | sudo tar xvz -C /opt/helk/kafka/ \ +RUN wget -qO- http://mirrors.ocf.berkeley.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz | sudo tar xvz -C /opt/helk/kafka/ \ && mkdir -v $KAFKA_LOGS_PATH \ No newline at end of file diff --git a/helk-kafka-broker/Dockerfile b/docker/helk-kafka-broker/Dockerfile similarity index 95% rename from helk-kafka-broker/Dockerfile rename to docker/helk-kafka-broker/Dockerfile index 632974f6..99ccd259 100644 --- a/helk-kafka-broker/Dockerfile +++ b/docker/helk-kafka-broker/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-kafka-base:1.1.0 +FROM cyb3rward0g/helk-kafka-base:1.1.1 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kafka Broker." diff --git a/helk-kafka-broker/scripts/kafka-entrypoint.sh b/docker/helk-kafka-broker/scripts/kafka-entrypoint.sh similarity index 99% rename from helk-kafka-broker/scripts/kafka-entrypoint.sh rename to docker/helk-kafka-broker/scripts/kafka-entrypoint.sh index a2855961..39e02ca5 100755 --- a/helk-kafka-broker/scripts/kafka-entrypoint.sh +++ b/docker/helk-kafka-broker/scripts/kafka-entrypoint.sh @@ -32,7 +32,7 @@ fi # *********** Starting Kafka ************** exec $KAFKA_SCRIPT $KAFKA_CONFIG >> $KAFKA_CONSOLE_LOG 2>&1 & -sleep 20 +sleep 30 # *********** Creating Kafka Topics************** declare -a temas=("winlogbeat" "sysmontransformed" "securitytransformed") diff --git a/helk-kafka-broker/server.properties b/docker/helk-kafka-broker/server.properties similarity index 100% rename from helk-kafka-broker/server.properties rename to docker/helk-kafka-broker/server.properties diff --git a/helk-kibana/Dockerfile b/docker/helk-kibana/Dockerfile similarity index 82% rename from helk-kibana/Dockerfile rename to docker/helk-kibana/Dockerfile index 1370fcf6..a4d35264 100644 --- a/helk-kibana/Dockerfile +++ b/docker/helk-kibana/Dockerfile @@ -1,12 +1,12 @@ # HELK script: HELK Kibana Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.1 +# HELK ELK version: 6.3.2 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 # References: # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html -FROM docker.elastic.co/kibana/kibana:6.3.1 +FROM docker.elastic.co/kibana/kibana:6.3.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kibana." \ No newline at end of file diff --git a/helk-kibana/dashboards/ALL-MITRE-ATTACK.json b/docker/helk-kibana/dashboards/ALL-MITRE-ATTACK.json similarity index 100% rename from helk-kibana/dashboards/ALL-MITRE-ATTACK.json rename to docker/helk-kibana/dashboards/ALL-MITRE-ATTACK.json diff --git a/helk-kibana/dashboards/Global_Dashboard.json b/docker/helk-kibana/dashboards/Global_Dashboard.json similarity index 100% rename from helk-kibana/dashboards/Global_Dashboard.json rename to docker/helk-kibana/dashboards/Global_Dashboard.json diff --git a/helk-kibana/dashboards/MITRE-ATTACK-GROUPS.json b/docker/helk-kibana/dashboards/MITRE-ATTACK-GROUPS.json similarity index 100% rename from helk-kibana/dashboards/MITRE-ATTACK-GROUPS.json rename to docker/helk-kibana/dashboards/MITRE-ATTACK-GROUPS.json diff --git a/helk-kibana/dashboards/Sysmon_Dashboard.json b/docker/helk-kibana/dashboards/Sysmon_Dashboard.json similarity index 100% rename from helk-kibana/dashboards/Sysmon_Dashboard.json rename to docker/helk-kibana/dashboards/Sysmon_Dashboard.json diff --git a/helk-kibana/dashboards/Sysmon_Network_Dashboard.json b/docker/helk-kibana/dashboards/Sysmon_Network_Dashboard.json similarity index 100% rename from helk-kibana/dashboards/Sysmon_Network_Dashboard.json rename to docker/helk-kibana/dashboards/Sysmon_Network_Dashboard.json diff --git a/helk-kibana/kibana.yml b/docker/helk-kibana/kibana.yml similarity index 100% rename from helk-kibana/kibana.yml rename to docker/helk-kibana/kibana.yml diff --git a/helk-kibana/scripts/kibana-entrypoint.sh b/docker/helk-kibana/scripts/kibana-entrypoint.sh similarity index 100% rename from helk-kibana/scripts/kibana-entrypoint.sh rename to docker/helk-kibana/scripts/kibana-entrypoint.sh diff --git a/helk-kibana/scripts/kibana-setup.sh b/docker/helk-kibana/scripts/kibana-setup.sh similarity index 100% rename from helk-kibana/scripts/kibana-setup.sh rename to docker/helk-kibana/scripts/kibana-setup.sh diff --git a/helk-kibana/trial/kibana.yml b/docker/helk-kibana/trial/kibana.yml similarity index 97% rename from helk-kibana/trial/kibana.yml rename to docker/helk-kibana/trial/kibana.yml index de967b18..c6604dc4 100644 --- a/helk-kibana/trial/kibana.yml +++ b/docker/helk-kibana/trial/kibana.yml @@ -12,7 +12,7 @@ server.host: "helk-kibana" #server.basePath: "" # The maximum payload size in bytes for incoming server requests. -#server.maxPayloadBytes: 1048576 +server.maxPayloadBytes: 2048576 # The Kibana server's name. This is used for display purposes. server.name: "helk-kibana" @@ -58,11 +58,11 @@ elasticsearch.url: "http://helk-elasticsearch:9200" # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of # the elasticsearch.requestTimeout setting. -#elasticsearch.pingTimeout: 1500 +elasticsearch.pingTimeout: 7500 # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value # must be a positive integer. -elasticsearch.requestTimeout: 60000 +elasticsearch.requestTimeout: 300000 # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side # headers, set this value to [] (an empty list). diff --git a/helk-kibana/trial/scripts/kibana-entrypoint.sh b/docker/helk-kibana/trial/scripts/kibana-entrypoint.sh similarity index 95% rename from helk-kibana/trial/scripts/kibana-entrypoint.sh rename to docker/helk-kibana/trial/scripts/kibana-entrypoint.sh index a4d91173..766e061c 100755 --- a/helk-kibana/trial/scripts/kibana-entrypoint.sh +++ b/docker/helk-kibana/trial/scripts/kibana-entrypoint.sh @@ -14,7 +14,7 @@ until curl -s $ELASTICSEARCH_ACCESS -o /dev/null; do sleep 1 done -# *********** Check if Elasticsearch is up *************** +# *********** Change Kibana and Logstash password *************** echo "[HELK-DOCKER-INSTALLATION-INFO] Submitting a request to change the password of a Kibana and Logstash users .." until curl -s -H 'Content-Type:application/json' -XPUT $ELASTICSEARCH_ACCESS/_xpack/security/user/kibana/_password -d "{\"password\": \"kibanapassword\"}" do diff --git a/helk-kibana/trial/scripts/kibana-setup.sh b/docker/helk-kibana/trial/scripts/kibana-setup.sh similarity index 100% rename from helk-kibana/trial/scripts/kibana-setup.sh rename to docker/helk-kibana/trial/scripts/kibana-setup.sh diff --git a/helk-logstash/Dockerfile b/docker/helk-logstash/Dockerfile similarity index 84% rename from helk-logstash/Dockerfile rename to docker/helk-logstash/Dockerfile index 7e41a469..136c8df3 100644 --- a/helk-logstash/Dockerfile +++ b/docker/helk-logstash/Dockerfile @@ -1,6 +1,6 @@ # HELK script: HELK Logstash Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.1 +# HELK ELK version: 6.3.2 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 @@ -8,6 +8,6 @@ # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html # https://github.com/spujadas/elk-docker/blob/master/Dockerfile -FROM docker.elastic.co/logstash/logstash:6.3.1 +FROM docker.elastic.co/logstash/logstash:6.3.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Logstash." \ No newline at end of file diff --git a/helk-logstash/enrichments/ACE/logstash/03-ace-rabbitmq-input.conf b/docker/helk-logstash/enrichments/ACE/logstash/03-ace-rabbitmq-input.conf similarity index 100% rename from helk-logstash/enrichments/ACE/logstash/03-ace-rabbitmq-input.conf rename to docker/helk-logstash/enrichments/ACE/logstash/03-ace-rabbitmq-input.conf diff --git a/helk-logstash/enrichments/ACE/logstash/55-rabbitmq-elasticsearch-output.conf b/docker/helk-logstash/enrichments/ACE/logstash/55-rabbitmq-elasticsearch-output.conf similarity index 100% rename from helk-logstash/enrichments/ACE/logstash/55-rabbitmq-elasticsearch-output.conf rename to docker/helk-logstash/enrichments/ACE/logstash/55-rabbitmq-elasticsearch-output.conf diff --git a/helk-logstash/enrichments/cti/mitre_attack.csv b/docker/helk-logstash/enrichments/cti/mitre_attack.csv similarity index 100% rename from helk-logstash/enrichments/cti/mitre_attack.csv rename to docker/helk-logstash/enrichments/cti/mitre_attack.csv diff --git a/helk-logstash/logstash.yml b/docker/helk-logstash/logstash.yml similarity index 99% rename from helk-logstash/logstash.yml rename to docker/helk-logstash/logstash.yml index 7b8aac1b..97377cfb 100644 --- a/helk-logstash/logstash.yml +++ b/docker/helk-logstash/logstash.yml @@ -62,7 +62,7 @@ pipeline.batch.size: 500 # # Where to fetch the pipeline configuration for the main pipeline # -# path.config: /etc/LS_SETTINGS_DIRstash/pipeline +#path.config: /usr/share/logstash/pipeline # # Pipeline configuration string for the main pipeline # diff --git a/helk-logstash/output_templates/10-logs-all-default.json b/docker/helk-logstash/output_templates/10-logs-all-default.json similarity index 100% rename from helk-logstash/output_templates/10-logs-all-default.json rename to docker/helk-logstash/output_templates/10-logs-all-default.json diff --git a/helk-logstash/output_templates/50-logs-winevent-all.json b/docker/helk-logstash/output_templates/50-logs-winevent-all.json similarity index 100% rename from helk-logstash/output_templates/50-logs-winevent-all.json rename to docker/helk-logstash/output_templates/50-logs-winevent-all.json diff --git a/helk-logstash/output_templates/60-powershell-direct-template.json b/docker/helk-logstash/output_templates/60-powershell-direct-template.json similarity index 100% rename from helk-logstash/output_templates/60-powershell-direct-template.json rename to docker/helk-logstash/output_templates/60-powershell-direct-template.json diff --git a/helk-logstash/output_templates/60-winevent-application-template.json b/docker/helk-logstash/output_templates/60-winevent-application-template.json similarity index 100% rename from helk-logstash/output_templates/60-winevent-application-template.json rename to docker/helk-logstash/output_templates/60-winevent-application-template.json diff --git a/helk-logstash/output_templates/60-winevent-powershell-template.json b/docker/helk-logstash/output_templates/60-winevent-powershell-template.json similarity index 100% rename from helk-logstash/output_templates/60-winevent-powershell-template.json rename to docker/helk-logstash/output_templates/60-winevent-powershell-template.json diff --git a/helk-logstash/output_templates/60-winevent-security-template.json b/docker/helk-logstash/output_templates/60-winevent-security-template.json similarity index 100% rename from helk-logstash/output_templates/60-winevent-security-template.json rename to docker/helk-logstash/output_templates/60-winevent-security-template.json diff --git a/helk-logstash/output_templates/60-winevent-sysmon-template.json b/docker/helk-logstash/output_templates/60-winevent-sysmon-template.json similarity index 100% rename from helk-logstash/output_templates/60-winevent-sysmon-template.json rename to docker/helk-logstash/output_templates/60-winevent-sysmon-template.json diff --git a/helk-logstash/output_templates/60-winevent-system-template.json b/docker/helk-logstash/output_templates/60-winevent-system-template.json similarity index 100% rename from helk-logstash/output_templates/60-winevent-system-template.json rename to docker/helk-logstash/output_templates/60-winevent-system-template.json diff --git a/helk-logstash/output_templates/60-winevent-wmiactivity-template.json b/docker/helk-logstash/output_templates/60-winevent-wmiactivity-template.json similarity index 100% rename from helk-logstash/output_templates/60-winevent-wmiactivity-template.json rename to docker/helk-logstash/output_templates/60-winevent-wmiactivity-template.json diff --git a/helk-logstash/output_templates/82-logs-not-ip.json b/docker/helk-logstash/output_templates/82-logs-not-ip.json similarity index 100% rename from helk-logstash/output_templates/82-logs-not-ip.json rename to docker/helk-logstash/output_templates/82-logs-not-ip.json diff --git a/helk-logstash/output_templates/91-logs-ip-dst-nat.json b/docker/helk-logstash/output_templates/91-logs-ip-dst-nat.json similarity index 100% rename from helk-logstash/output_templates/91-logs-ip-dst-nat.json rename to docker/helk-logstash/output_templates/91-logs-ip-dst-nat.json diff --git a/helk-logstash/output_templates/91-logs-ip-dst.json b/docker/helk-logstash/output_templates/91-logs-ip-dst.json similarity index 100% rename from helk-logstash/output_templates/91-logs-ip-dst.json rename to docker/helk-logstash/output_templates/91-logs-ip-dst.json diff --git a/helk-logstash/output_templates/91-logs-ip-src-nat.json b/docker/helk-logstash/output_templates/91-logs-ip-src-nat.json similarity index 100% rename from helk-logstash/output_templates/91-logs-ip-src-nat.json rename to docker/helk-logstash/output_templates/91-logs-ip-src-nat.json diff --git a/helk-logstash/output_templates/91-logs-ip-src.json b/docker/helk-logstash/output_templates/91-logs-ip-src.json similarity index 100% rename from helk-logstash/output_templates/91-logs-ip-src.json rename to docker/helk-logstash/output_templates/91-logs-ip-src.json diff --git a/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json b/docker/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json similarity index 100% rename from helk-logstash/output_templates/93-logs-ipv6-dst-nat.json rename to docker/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json diff --git a/helk-logstash/output_templates/93-logs-ipv6-dst.json b/docker/helk-logstash/output_templates/93-logs-ipv6-dst.json similarity index 100% rename from helk-logstash/output_templates/93-logs-ipv6-dst.json rename to docker/helk-logstash/output_templates/93-logs-ipv6-dst.json diff --git a/helk-logstash/output_templates/93-logs-ipv6-src-nat.json b/docker/helk-logstash/output_templates/93-logs-ipv6-src-nat.json similarity index 100% rename from helk-logstash/output_templates/93-logs-ipv6-src-nat.json rename to docker/helk-logstash/output_templates/93-logs-ipv6-src-nat.json diff --git a/helk-logstash/output_templates/93-logs-ipv6-src.json b/docker/helk-logstash/output_templates/93-logs-ipv6-src.json similarity index 100% rename from helk-logstash/output_templates/93-logs-ipv6-src.json rename to docker/helk-logstash/output_templates/93-logs-ipv6-src.json diff --git a/helk-logstash/output_templates/99-logs-any-fields.json b/docker/helk-logstash/output_templates/99-logs-any-fields.json similarity index 100% rename from helk-logstash/output_templates/99-logs-any-fields.json rename to docker/helk-logstash/output_templates/99-logs-any-fields.json diff --git a/helk-logstash/pipeline/0002-kafka-input.conf b/docker/helk-logstash/pipeline/0002-kafka-input.conf similarity index 100% rename from helk-logstash/pipeline/0002-kafka-input.conf rename to docker/helk-logstash/pipeline/0002-kafka-input.conf diff --git a/helk-logstash/pipeline/0003-attack-input.conf b/docker/helk-logstash/pipeline/0003-attack-input.conf similarity index 100% rename from helk-logstash/pipeline/0003-attack-input.conf rename to docker/helk-logstash/pipeline/0003-attack-input.conf diff --git a/helk-logstash/pipeline/0004-beats-input.conf b/docker/helk-logstash/pipeline/0004-beats-input.conf similarity index 100% rename from helk-logstash/pipeline/0004-beats-input.conf rename to docker/helk-logstash/pipeline/0004-beats-input.conf diff --git a/helk-logstash/pipeline/0098-all-filter.conf b/docker/helk-logstash/pipeline/0098-all-filter.conf similarity index 100% rename from helk-logstash/pipeline/0098-all-filter.conf rename to docker/helk-logstash/pipeline/0098-all-filter.conf diff --git a/helk-logstash/pipeline/0099-all-finpgerint-hash-filter.conf b/docker/helk-logstash/pipeline/0099-all-fingerprint-hash-filter.conf similarity index 100% rename from helk-logstash/pipeline/0099-all-finpgerint-hash-filter.conf rename to docker/helk-logstash/pipeline/0099-all-fingerprint-hash-filter.conf diff --git a/helk-logstash/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf b/docker/helk-logstash/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf similarity index 84% rename from helk-logstash/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf rename to docker/helk-logstash/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf index bb9d546b..1b0b8693 100644 --- a/helk-logstash/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf +++ b/docker/helk-logstash/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf @@ -1,3 +1,8 @@ +# HELK winevent-remove-winlogbeats-prepend-of-eventdata filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { # Use the following to get rid of the prepended "event_data" nest that (elastic) winlogbeats adds to windows logs if [type] == "wineventlog" and [beat] { diff --git a/helk-logstash/pipeline/1216-attack-filter.conf b/docker/helk-logstash/pipeline/1216-attack-filter.conf similarity index 100% rename from helk-logstash/pipeline/1216-attack-filter.conf rename to docker/helk-logstash/pipeline/1216-attack-filter.conf diff --git a/helk-logstash/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf b/docker/helk-logstash/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf similarity index 95% rename from helk-logstash/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf rename to docker/helk-logstash/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf index b97d363c..84457a22 100644 --- a/helk-logstash/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf +++ b/docker/helk-logstash/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf @@ -1,3 +1,8 @@ +# HELK winevent-cleanup-no-dashes-only-values filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { if [event_id] { mutate { add_field => { "z_logstash_pipeline" => "1500" } } diff --git a/helk-logstash/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf b/docker/helk-logstash/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf similarity index 96% rename from helk-logstash/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf rename to docker/helk-logstash/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf index 9d81b4db..0b80def7 100644 --- a/helk-logstash/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf +++ b/docker/helk-logstash/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf @@ -1,3 +1,8 @@ +# HELK winevent-conversions-ip-conversions-basic filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { # Use this to determine if windows event log or not (for now, until we are properly marking all windows log types as something like "log_type: winevent") if [event_id] { @@ -150,11 +155,11 @@ filter { # Seen in the following EventIDs (not necessarily exhaustive) # Microsoft-Windows-TerminalServices-RDPClient/Operational:1102,1024 # Only perform on the above EIDs because otherwise it may be values that are incomprehensible - else if [Value] and [wef][channel] == "Microsoft-Windows-TerminalServices-RDPClient/Operational" { - if [wef.eid] == 1102 { + else if [Value] and [log_name] == "Microsoft-Windows-TerminalServices-RDPClient/Operational" { + if [event_id] == 1102 { mutate { rename => { "Value" => "dst_ip_addr" } } } - else if [wef.eid] == 1024 { + else if [event_id] == 1024 { mutate { rename => { "Value" => "dst_ip_addr" } } } } diff --git a/helk-logstash/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf b/docker/helk-logstash/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf similarity index 96% rename from helk-logstash/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf rename to docker/helk-logstash/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf index d5f385f6..83dd0c90 100644 --- a/helk-logstash/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf +++ b/docker/helk-logstash/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf @@ -1,3 +1,8 @@ +# HELK winevent-cleanup-lowercasing-windows-is-case-sensitive filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { if [event_id] { mutate { add_field => { "z_logstash_pipeline" => "1522" } } diff --git a/helk-logstash/pipeline/1531-winevent-sysmon-filter.conf b/docker/helk-logstash/pipeline/1531-winevent-sysmon-filter.conf similarity index 100% rename from helk-logstash/pipeline/1531-winevent-sysmon-filter.conf rename to docker/helk-logstash/pipeline/1531-winevent-sysmon-filter.conf diff --git a/helk-logstash/pipeline/1532-winevent-security-filter.conf b/docker/helk-logstash/pipeline/1532-winevent-security-filter.conf similarity index 100% rename from helk-logstash/pipeline/1532-winevent-security-filter.conf rename to docker/helk-logstash/pipeline/1532-winevent-security-filter.conf diff --git a/helk-logstash/pipeline/1533-winevent-system-filter.conf b/docker/helk-logstash/pipeline/1533-winevent-system-filter.conf similarity index 100% rename from helk-logstash/pipeline/1533-winevent-system-filter.conf rename to docker/helk-logstash/pipeline/1533-winevent-system-filter.conf diff --git a/helk-logstash/pipeline/1534-winevent-application-filter.conf b/docker/helk-logstash/pipeline/1534-winevent-application-filter.conf similarity index 100% rename from helk-logstash/pipeline/1534-winevent-application-filter.conf rename to docker/helk-logstash/pipeline/1534-winevent-application-filter.conf diff --git a/helk-logstash/pipeline/1535-winevent-wmiactivity-filter.conf b/docker/helk-logstash/pipeline/1535-winevent-wmiactivity-filter.conf similarity index 100% rename from helk-logstash/pipeline/1535-winevent-wmiactivity-filter.conf rename to docker/helk-logstash/pipeline/1535-winevent-wmiactivity-filter.conf diff --git a/helk-logstash/pipeline/1543-winevent-conversions-process-cli-filter.conf b/docker/helk-logstash/pipeline/1543-winevent-conversions-process-cli-filter.conf similarity index 91% rename from helk-logstash/pipeline/1543-winevent-conversions-process-cli-filter.conf rename to docker/helk-logstash/pipeline/1543-winevent-conversions-process-cli-filter.conf index 4774eb99..3021dc0d 100644 --- a/helk-logstash/pipeline/1543-winevent-conversions-process-cli-filter.conf +++ b/docker/helk-logstash/pipeline/1543-winevent-conversions-process-cli-filter.conf @@ -1,3 +1,8 @@ +# HELK winevent-conversions-process-cli filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { if [event_id] { diff --git a/helk-logstash/pipeline/1544-winevent-cleanup-other.conf b/docker/helk-logstash/pipeline/1544-winevent-cleanup-other.conf similarity index 63% rename from helk-logstash/pipeline/1544-winevent-cleanup-other.conf rename to docker/helk-logstash/pipeline/1544-winevent-cleanup-other.conf index e139e488..2833fcda 100644 --- a/helk-logstash/pipeline/1544-winevent-cleanup-other.conf +++ b/docker/helk-logstash/pipeline/1544-winevent-cleanup-other.conf @@ -1,3 +1,8 @@ +# HELK winevent-cleanup-other filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { if [event_id] { if [user_logon_guid] { diff --git a/helk-logstash/pipeline/2511-winevent-powershell-filter.conf b/docker/helk-logstash/pipeline/2511-winevent-powershell-filter.conf similarity index 100% rename from helk-logstash/pipeline/2511-winevent-powershell-filter.conf rename to docker/helk-logstash/pipeline/2511-winevent-powershell-filter.conf diff --git a/helk-logstash/pipeline/2512-winevent-security-schtasks-filter.conf b/docker/helk-logstash/pipeline/2512-winevent-security-schtasks-filter.conf similarity index 99% rename from helk-logstash/pipeline/2512-winevent-security-schtasks-filter.conf rename to docker/helk-logstash/pipeline/2512-winevent-security-schtasks-filter.conf index 9eec12af..86f0ce80 100644 --- a/helk-logstash/pipeline/2512-winevent-security-schtasks-filter.conf +++ b/docker/helk-logstash/pipeline/2512-winevent-security-schtasks-filter.conf @@ -2,6 +2,7 @@ # HELK build Stage: Alpha # Author: Nate Guagenti (@neu5ron) # License: GPL-3.0 + filter { if [log_name] == "Security" { # event_id 4698 for Created Scheduled Task diff --git a/helk-logstash/pipeline/8012-dst-ip-cleanups-filter.conf b/docker/helk-logstash/pipeline/8012-dst-ip-cleanups-filter.conf similarity index 97% rename from helk-logstash/pipeline/8012-dst-ip-cleanups-filter.conf rename to docker/helk-logstash/pipeline/8012-dst-ip-cleanups-filter.conf index cef874c3..fe87a829 100644 --- a/helk-logstash/pipeline/8012-dst-ip-cleanups-filter.conf +++ b/docker/helk-logstash/pipeline/8012-dst-ip-cleanups-filter.conf @@ -1,3 +1,8 @@ +# HELK dst-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" diff --git a/helk-logstash/pipeline/8013-src-ip-cleanups-filter.conf b/docker/helk-logstash/pipeline/8013-src-ip-cleanups-filter.conf similarity index 97% rename from helk-logstash/pipeline/8013-src-ip-cleanups-filter.conf rename to docker/helk-logstash/pipeline/8013-src-ip-cleanups-filter.conf index 21253a9b..7987070c 100644 --- a/helk-logstash/pipeline/8013-src-ip-cleanups-filter.conf +++ b/docker/helk-logstash/pipeline/8013-src-ip-cleanups-filter.conf @@ -1,3 +1,8 @@ +# HELK src-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" diff --git a/helk-logstash/pipeline/8014-dst-nat-ip-cleanups-filter.conf b/docker/helk-logstash/pipeline/8014-dst-nat-ip-cleanups-filter.conf similarity index 97% rename from helk-logstash/pipeline/8014-dst-nat-ip-cleanups-filter.conf rename to docker/helk-logstash/pipeline/8014-dst-nat-ip-cleanups-filter.conf index 6d6bae8b..47bb8821 100644 --- a/helk-logstash/pipeline/8014-dst-nat-ip-cleanups-filter.conf +++ b/docker/helk-logstash/pipeline/8014-dst-nat-ip-cleanups-filter.conf @@ -1,3 +1,8 @@ +# HELK dst-nat-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" diff --git a/helk-logstash/pipeline/8015-src-nat-ip-cleanups-filter.conf b/docker/helk-logstash/pipeline/8015-src-nat-ip-cleanups-filter.conf similarity index 97% rename from helk-logstash/pipeline/8015-src-nat-ip-cleanups-filter.conf rename to docker/helk-logstash/pipeline/8015-src-nat-ip-cleanups-filter.conf index 897ed729..f0bb431c 100644 --- a/helk-logstash/pipeline/8015-src-nat-ip-cleanups-filter.conf +++ b/docker/helk-logstash/pipeline/8015-src-nat-ip-cleanups-filter.conf @@ -1,3 +1,8 @@ +# HELK src-nat-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" diff --git a/helk-logstash/pipeline/8112-dst-ip-filter.conf b/docker/helk-logstash/pipeline/8112-dst-ip-filter.conf similarity index 98% rename from helk-logstash/pipeline/8112-dst-ip-filter.conf rename to docker/helk-logstash/pipeline/8112-dst-ip-filter.conf index a6b7ddca..fa58787a 100644 --- a/helk-logstash/pipeline/8112-dst-ip-filter.conf +++ b/docker/helk-logstash/pipeline/8112-dst-ip-filter.conf @@ -1,3 +1,8 @@ +# HELK dst-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { # If dst_ip_addr field exists from previous config settings if [dst_ip_addr] { diff --git a/helk-logstash/pipeline/8113-src-ip-filter.conf b/docker/helk-logstash/pipeline/8113-src-ip-filter.conf similarity index 98% rename from helk-logstash/pipeline/8113-src-ip-filter.conf rename to docker/helk-logstash/pipeline/8113-src-ip-filter.conf index e25999d8..b583ec0c 100644 --- a/helk-logstash/pipeline/8113-src-ip-filter.conf +++ b/docker/helk-logstash/pipeline/8113-src-ip-filter.conf @@ -1,3 +1,8 @@ +# HELK src-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { # If src_ip_addr field exists from previous config settings if [src_ip_addr] { diff --git a/helk-logstash/pipeline/8114-dst-nat-ip-filter.conf b/docker/helk-logstash/pipeline/8114-dst-nat-ip-filter.conf similarity index 98% rename from helk-logstash/pipeline/8114-dst-nat-ip-filter.conf rename to docker/helk-logstash/pipeline/8114-dst-nat-ip-filter.conf index efd696af..d4bef355 100644 --- a/helk-logstash/pipeline/8114-dst-nat-ip-filter.conf +++ b/docker/helk-logstash/pipeline/8114-dst-nat-ip-filter.conf @@ -1,3 +1,8 @@ +# HELK dst-nat-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { # If dst_nat_ip_addr field exists from previous config settings if [dst_nat_ip_addr] { diff --git a/helk-logstash/pipeline/8115-src-nat-ip-filter.conf b/docker/helk-logstash/pipeline/8115-src-nat-ip-filter.conf similarity index 98% rename from helk-logstash/pipeline/8115-src-nat-ip-filter.conf rename to docker/helk-logstash/pipeline/8115-src-nat-ip-filter.conf index 75b3963b..fa246bf9 100644 --- a/helk-logstash/pipeline/8115-src-nat-ip-filter.conf +++ b/docker/helk-logstash/pipeline/8115-src-nat-ip-filter.conf @@ -1,3 +1,8 @@ +# HELK src-nat-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + filter { # If src_nat_ip_addr field exists from previous config settings if [src_nat_ip_addr] { diff --git a/helk-logstash/pipeline/9950-winevent-sysmon-output.conf b/docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf similarity index 94% rename from helk-logstash/pipeline/9950-winevent-sysmon-output.conf rename to docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf index 77d9a533..940a3547 100644 --- a/helk-logstash/pipeline/9950-winevent-sysmon-output.conf +++ b/docker/helk-logstash/pipeline/9950-winevent-sysmon-output.conf @@ -9,7 +9,7 @@ output { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-sysmon-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" - document_type => "_doc" + #document_type => "_doc" } kafka { bootstrap_servers => "helk-kafka-broker:9092" diff --git a/helk-logstash/pipeline/9951-winevent-security-output.conf b/docker/helk-logstash/pipeline/9951-winevent-security-output.conf similarity index 94% rename from helk-logstash/pipeline/9951-winevent-security-output.conf rename to docker/helk-logstash/pipeline/9951-winevent-security-output.conf index f7307771..37fe93fb 100644 --- a/helk-logstash/pipeline/9951-winevent-security-output.conf +++ b/docker/helk-logstash/pipeline/9951-winevent-security-output.conf @@ -9,7 +9,7 @@ output { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-security-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" - document_type => "_doc" + #document_type => "_doc" } kafka { bootstrap_servers => "helk-kafka-broker:9092" diff --git a/helk-logstash/pipeline/9952-winevent-system-output.conf b/docker/helk-logstash/pipeline/9952-winevent-system-output.conf similarity index 91% rename from helk-logstash/pipeline/9952-winevent-system-output.conf rename to docker/helk-logstash/pipeline/9952-winevent-system-output.conf index a5b386c0..e02a67d6 100644 --- a/helk-logstash/pipeline/9952-winevent-system-output.conf +++ b/docker/helk-logstash/pipeline/9952-winevent-system-output.conf @@ -9,7 +9,7 @@ output { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-system-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" - document_type => "_doc" + #document_type => "_doc" } } } \ No newline at end of file diff --git a/helk-logstash/pipeline/9953-winevent-application-output.conf b/docker/helk-logstash/pipeline/9953-winevent-application-output.conf similarity index 92% rename from helk-logstash/pipeline/9953-winevent-application-output.conf rename to docker/helk-logstash/pipeline/9953-winevent-application-output.conf index d0ecd0e7..1825b78e 100644 --- a/helk-logstash/pipeline/9953-winevent-application-output.conf +++ b/docker/helk-logstash/pipeline/9953-winevent-application-output.conf @@ -9,7 +9,7 @@ output { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-application-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" - document_type => "_doc" + #document_type => "_doc" } } } \ No newline at end of file diff --git a/helk-logstash/pipeline/9954-winevent-powershell-output.conf b/docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf similarity index 92% rename from helk-logstash/pipeline/9954-winevent-powershell-output.conf rename to docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf index 1ce908e6..d01b9d96 100644 --- a/helk-logstash/pipeline/9954-winevent-powershell-output.conf +++ b/docker/helk-logstash/pipeline/9954-winevent-powershell-output.conf @@ -9,7 +9,7 @@ output { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-powershell-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" - document_type => "_doc" + #document_type => "_doc" } } } \ No newline at end of file diff --git a/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf b/docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf similarity index 92% rename from helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf rename to docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf index 6cd932f5..d6684266 100644 --- a/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf +++ b/docker/helk-logstash/pipeline/9955-winevent-wmiactivity-output.conf @@ -9,7 +9,7 @@ output { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-wmiactivity-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" - document_type => "_doc" + #document_type => "_doc" } } } \ No newline at end of file diff --git a/helk-logstash/pipeline/9956-attack-output.conf b/docker/helk-logstash/pipeline/9956-attack-output.conf similarity index 90% rename from helk-logstash/pipeline/9956-attack-output.conf rename to docker/helk-logstash/pipeline/9956-attack-output.conf index 6e711027..3a412dd4 100644 --- a/helk-logstash/pipeline/9956-attack-output.conf +++ b/docker/helk-logstash/pipeline/9956-attack-output.conf @@ -9,7 +9,7 @@ output { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "mitre-attack-%{+YYYY.MM.dd}" - document_type => "_doc" + #document_type => "_doc" } } } diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh new file mode 100755 index 00000000..785552a7 --- /dev/null +++ b/docker/helk-logstash/scripts/logstash-entrypoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# HELK script: logstash-entrypoint.sh +# HELK script description: Pushes output templates to ES and starts Logstash +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# ********* Setting LS_JAVA_OPTS *************** +if [[ ! -z "$LS_JAVA_OPTS" ]]; then + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting LS_JAVA_OPTS to $LS_JAVA_OPTS" +else + # ****** Setup heap size ***** + LS_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo) + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting LS_HEAP_SIZE to ${LS_MEMORY}.." + export LS_JAVA_OPTS="-Xms${LS_MEMORY}g -Xmx${LS_MEMORY}g" +fi + +# *********** Looking for ES *************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.." +until curl -s helk-elasticsearch:9200 -o /dev/null; do + sleep 1 +done + +echo "[HELK-DOCKER-INSTALLATION-INFO] Uploading templates to elasticsearch.." +DIR=/usr/share/logstash/output_templates +for file in ${DIR}/*.json +do + template_name=$(echo $file | sed -r ' s/^.*\/[0-9]+\-//'); + echo "[HELK-DOCKER-INSTALLATION-INFO] Uploading $template_name template to elasticsearch.."; + curl -s -H 'Content-Type: application/json' -XPUT "http://helk-elasticsearch:9200/_template/$template_name" -d@${file}; +done + +# ********** Install Plugin ***************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstsh plugins.." +logstash-plugin install logstash-filter-prune + +# ********** Starting Logstash ***************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.." +/usr/local/bin/docker-entrypoint \ No newline at end of file diff --git a/helk-logstash/trial/logstash.yml b/docker/helk-logstash/trial/logstash.yml similarity index 100% rename from helk-logstash/trial/logstash.yml rename to docker/helk-logstash/trial/logstash.yml diff --git a/helk-logstash/trial/pipeline/02-kafka-input.conf b/docker/helk-logstash/trial/pipeline/0002-kafka-input.conf similarity index 98% rename from helk-logstash/trial/pipeline/02-kafka-input.conf rename to docker/helk-logstash/trial/pipeline/0002-kafka-input.conf index 4bddc082..03cdc229 100644 --- a/helk-logstash/trial/pipeline/02-kafka-input.conf +++ b/docker/helk-logstash/trial/pipeline/0002-kafka-input.conf @@ -4,8 +4,7 @@ # License: GPL-3.0 input { - kafka - { + kafka { bootstrap_servers => "helk-kafka-broker:9092,helk-kafka-broker2:9093" topics => ["winlogbeat"] decorate_events => true diff --git a/helk-logstash/trial/pipeline/03-attack-input.conf b/docker/helk-logstash/trial/pipeline/0003-attack-input.conf similarity index 96% rename from helk-logstash/trial/pipeline/03-attack-input.conf rename to docker/helk-logstash/trial/pipeline/0003-attack-input.conf index b198d520..7ef4ec35 100644 --- a/helk-logstash/trial/pipeline/03-attack-input.conf +++ b/docker/helk-logstash/trial/pipeline/0003-attack-input.conf @@ -5,8 +5,7 @@ # License: GPL-3.0 input { - file - { + file { path => "/usr/share/logstash/cti/mitre_attack.csv" start_position => "beginning" sincedb_path => "/dev/null" diff --git a/docker/helk-logstash/trial/pipeline/0004-beats-input.conf b/docker/helk-logstash/trial/pipeline/0004-beats-input.conf new file mode 100644 index 00000000..be6befcc --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/0004-beats-input.conf @@ -0,0 +1,11 @@ +# HELK Kafka input conf file +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +input { + beats { + port => 5044 + include_codec_tag => false + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/0098-all-filter.conf b/docker/helk-logstash/trial/pipeline/0098-all-filter.conf new file mode 100644 index 00000000..b8dd7c61 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/0098-all-filter.conf @@ -0,0 +1,19 @@ +# HELK All filter conf file +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +filter { + if [message] { + mutate { + add_field => { + "z_logstash_pipeline" => "0098" + "log_ingest_timestamp" => "%{@timestamp}" + } + copy => { + "message" => "z_original_message" + "type" => "z_logstash_type" + } + } + } +} \ No newline at end of file diff --git a/helk-logstash/trial/pipeline/09-all-filter.conf b/docker/helk-logstash/trial/pipeline/0099-all-fingerprint-hash-filter.conf similarity index 81% rename from helk-logstash/trial/pipeline/09-all-filter.conf rename to docker/helk-logstash/trial/pipeline/0099-all-fingerprint-hash-filter.conf index ff7a48c3..f933ea48 100644 --- a/helk-logstash/trial/pipeline/09-all-filter.conf +++ b/docker/helk-logstash/trial/pipeline/0099-all-fingerprint-hash-filter.conf @@ -5,6 +5,7 @@ filter { if [message] { + mutate { add_field => { "z_logstash_pipeline" => "0099" } } fingerprint { source => "message" target => "[@metadata][log_hash]" diff --git a/docker/helk-logstash/trial/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf b/docker/helk-logstash/trial/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf new file mode 100644 index 00000000..1b0b8693 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1010-winevent-remove-winlogbeats-prepend-of-eventdata.conf @@ -0,0 +1,33 @@ +# HELK winevent-remove-winlogbeats-prepend-of-eventdata filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + # Use the following to get rid of the prepended "event_data" nest that (elastic) winlogbeats adds to windows logs + if [type] == "wineventlog" and [beat] { + ruby { + code => " + eventdata = event.get('event_data') + # Sometimes does not exist, so check that first -- then move the nests + if !eventdata.nil? + eventdata.each {|k, v| + if eventdata.to_s != '(NULL)' + event.set(k, v) + end + } + end + # Finally remove the nest completely + event.remove('event_data') + " + tag_on_exception => "_rubyexception_1010" + #code => " + # event.get('event_data').each {|k, v| + # event.set(k, v) + # } + # event.remove('event_data') + #" + #tag_on_exception => "_rubyexception_1010" + } + } +} \ No newline at end of file diff --git a/helk-logstash/trial/pipeline/16-attack-filter.conf b/docker/helk-logstash/trial/pipeline/1216-attack-filter.conf similarity index 100% rename from helk-logstash/trial/pipeline/16-attack-filter.conf rename to docker/helk-logstash/trial/pipeline/1216-attack-filter.conf diff --git a/docker/helk-logstash/trial/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf b/docker/helk-logstash/trial/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf new file mode 100644 index 00000000..84457a22 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1500-winevent-cleanup-no-dashes-only-values-filter.conf @@ -0,0 +1,86 @@ +# HELK winevent-cleanup-no-dashes-only-values filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + if [event_id] { + mutate { add_field => { "z_logstash_pipeline" => "1500" } } + + # Remove specific keys/fields that have "-"/dash has the only value values + # Command Line will be done later on in pipeline because it is all sorts of random fields especially when we custom parse some event IDs + # #TONOTE:Prune does not work on nested fields, so we are moving this into the beginning of Windows logs pipeline + prune { + blacklist_values => [ + "AccessList", "^\-$", + "AccessReason", "^\-$", + "AccountExpires", "^\-$", + "AccountName", "^\-$", + "AdditionalInfo", "^\-$", + "Address", "^\-$", + "AppCorrelationID", "^\-$", + "CallerProcessName", "^\-$", + "ClientAddress", "^\-$", + "ClientIP", "^\-$", + "ClientIPAddress", "^\-$", + "CompatibleIds", "^\-$", + "ComputerAccountChange", "^\-$", + "ConnectedViaIPAddress", "^\-$", + "DCIPAddress", "^\-$", + "DestinationIp", "^\-$", + "DisplayName", "^\-$", + "DnsHostName", "^\-$", + "Domain", "^\-$", + "Dummy", "^\-$", + "HomeDirectory", "^\-$", + "HomePath", "^\-$", + "Identity", "^\-$", + "ipAddress", "^\-$", + "IpAddress", "^\-$", + "IPAddress", "^\-$", + "IpPort", "^\-$", + "IPString", "^\-$", + "LaunchedViaIPAddress", "^\-$", + "LmPackageName", "^\-$", + "LocationInformation", "^\-$", + "LogonGuid", "^\-$", + "LogonHours", "^\-$", + "NewUacValue", "^\-$", + "ObjectName", "^\-$", + "ObjectType", "^\-$", + "OldUacValue", "^\-$", + "PasswordLastSet", "^\-$", + "PreAuthType", "^\-$", + "PrimaryGroupId", "^\-$", + "PrivilegeList", "^\-$", + "ProcessName", "^\-$", + "ProfilePath", "^\-$", + "RestrictedAdminMode", "^\-$", + "SamAccountName", "^\-$", + "ScriptPath", "^\-$", + "ServerIpAddress", "^\-$", + "Service", "^\-$", + "ServicePrincipalNames", "^\-$", + "SourceIp", "^\-$", + "SidHistory", "^\-$", + "SidList", "^\-$", + "SubjectDomainName", "^\-$", + "SubjectUserName", "^\-$", + "SubjectUserSid", "^\-$", + "TargetDomainName", "^\-$", + "TargetOutboundDomainName", "^\-$", + "TargetOutboundUserName", "^\-$", + "TargetUserName", "^\-$", + "TargetUserSid", "^\-$", + "TraceMessage", "^\-$", + "TransmittedServices", "^\-$", + "UserAccountControl", "^\-$", + "UserID", "^\-$", + "UserParameters", "^\-$", + "UserPrincipalName", "^\-$", + "UserWorkstations", "^\-$", + "WorkstationName", "^\-$" + ] + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf b/docker/helk-logstash/trial/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf new file mode 100644 index 00000000..0b80def7 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1521-winevent-conversions-ip-conversions-basic-filter.conf @@ -0,0 +1,186 @@ +# HELK winevent-conversions-ip-conversions-basic filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + # Use this to determine if windows event log or not (for now, until we are properly marking all windows log types as something like "log_type: winevent") + if [event_id] { + mutate { add_field => { "z_logstash_pipeline" => "1521" } } + + # Since Sysmon may be the most common EventIDs with IPs lets do that first. This will contain source and destination IPs. + # Seen in the following EventIDs (not necessarily exhaustive) + # Microsoft-Windows-Sysmon/Operational:3 + # Parse "SourceIp" field and then afterwards if it exists then the DestinationIp should exist as well (see note above) + if [SourceIp] { + if [SourceIsIpv6] == 'false' { + mutate { + rename => { "SourceIp" => "src_ip_addr" } + remove_field => [ "SourceIsIpv6" ] + } + } + else { + mutate { + rename => { "SourceIp" => "ipv6_src_addr" } + remove_field => [ "SourceIsIpv6" ] + } + } + # Parse "DestinationIp" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Microsoft-Windows-Sysmon/Operational:3 + if [DestinationIp] { + if [DestinationIsIpv6] == 'false' { + mutate { + rename => { "DestinationIp" => "dst_ip_addr" } + remove_field => [ "DestinationIsIpv6" ] + } + } + else { + mutate { + rename => { "DestinationIp" => "ipv6_dst_addr" } + remove_field => [ "DestinationIsIpv6" ] + } + } + } + } + + # If not Sysmon IP field try all the rest of the possible IP fields as of 2018-03-19 and going through 3,000 Windows EventIDs + else { + + #### Parse all other possible Source IP Address fields, none of the following two fields would exist in any one windows event.. therefore, use "if"/"else if" statements + # Parse "IpAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Security:4624, Security:4625, Security:4648, Security:4770, Security:4771, Security:4768, Security:4769, Security:5140, Security:5145 + if [IpAddress] { + if [IpAddress] =~ "^\d{1,3}\." { + mutate { rename => { "IpAddress" => "src_ip_addr" } } + } + else { + # First try to match IPv6 & IPv4 combined + # Last try IPv6 only + grok { + match => { + "IpAddress" => + [ + "%{IPV6:ipv6_src_addr} %{IPV4:src_ip_addr}", + "%{IPV6:ipv6_src_addr}" + ] + } + keep_empty_captures => false + named_captures_only => true + tag_on_failure => [ "_IpAddress_grokparsefailure", "_grokparsefailure", "_parsefailure", "_windows_ip_parsefailure" ] + tag_on_timeout => "_groktimeout" + # Timeout .250 seconds + timeout_millis => 250 + remove_field => [ "IpAddress" ] + } + } + } + + # Parse "SourceAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Security:5152,5154,5156,5157,5158 + else if [SourceAddress] { + mutate { rename => { "SourceAddress" => "src_ip_addr" } } + } + + # Parse "ClientAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Security:4778,4779 + else if [ClientAddress] { + mutate { rename => { "ClientAddress" => "src_ip_addr" } } + } + + # Parse "ClientIPAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Citrix-XenDesktop-BrokerMonitor/Operational:4,5,44,45 + else if [ClientIPAddress] { + mutate { rename => { "ClientIPAddress" => "src_ip_addr" } } + } + + # Parse "ClientIP" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Microsoft-Windows-RemoteDesktopServices-RdpCoreTS/Operational:131 + else if [ClientIP] { + # First lets substitute any characters that have been seen that would cause bad parsing/values + mutate { gsub => [ "ClientIP", "[\[\]]", "" ] } + if [ClientIP] =~ "^\d{1,3}\." { + dissect { + mapping => { "ClientIP" => "%{src_ip_addr}:%{port_src}" } + tag_on_failure => [ "_dissectfailure", "_parsefailure" ] + remove_field => [ "ClientIP" ] + } + } + } + + # Parse "IPString" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Microsoft-Windows-RemoteDesktopServices-RdpCoreTS/Operational:139,140 + else if [IPString] { + mutate { rename => { "IPString" => "src_ip_addr" } } + } + #### ^ DONE w/ All Src IP parsing #### + + + #### Parse all other possible Destination IP Address fields, none of the following two fields would exist in any one windows event.. therefore, use "if"/"else if" statements + # Parse "DestAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Application:1039 SourceName Citrix Broker Service + if [DestAddress] { + mutate { rename => { "DestAddress" => "dst_ip_addr" } } + } + + # Parse "LaunchedViaIPAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Citrix-XenDesktop-BrokerMonitor/Operational:4,5,44,45 + else if [LaunchedViaIPAddress] { + mutate { rename => { "LaunchedViaIPAddress" => "dst_ip_addr" } } + } + + # Parse "MachineIpAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Citrix-XenDesktop-BrokerMonitor/Operational:10 + else if [MachineIpAddress] { + mutate { rename => { "MachineIpAddress" => "dst_ip_addr" } } + } + + # Parse "ipAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Application:1039 SourceName Citrix Broker Service + else if [ipAddress] { + mutate { rename => { "ipAddress" => "dst_ip_addr" } } + } + + # Parse "Value" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Microsoft-Windows-TerminalServices-RDPClient/Operational:1102,1024 + # Only perform on the above EIDs because otherwise it may be values that are incomprehensible + else if [Value] and [log_name] == "Microsoft-Windows-TerminalServices-RDPClient/Operational" { + if [event_id] == 1102 { + mutate { rename => { "Value" => "dst_ip_addr" } } + } + else if [event_id] == 1024 { + mutate { rename => { "Value" => "dst_ip_addr" } } + } + } + # Parse "TargetServer" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Security:5378 + else if [TargetServer] { + mutate { gsub => [ "TargetServer", "TERMSRV\/", "" ] } + mutate { rename => { "TargetServer" => "dst_ip_addr" } } + } + #### ^ DONE w/ All Dst IP parsing #### + + + #### Parse all possible Destination NAT IP Address fields, none of the following two fields would exist in any one windows event.. therefore, use "if"/"else if" statements + # Parse "ConnectedViaIPAddress" field + # Seen in the following EventIDs (not necessarily exhaustive) + # Citrix-XenDesktop-BrokerMonitor/Operational:4,44 + if [ConnectedViaIPAddress] { + mutate { rename => { "ConnectedViaIPAddress" => "dst_nat_ip_addr" } } + } + #### ^ DONE w/ All Dst NAT IP parsing #### + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf b/docker/helk-logstash/trial/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf new file mode 100644 index 00000000..83dd0c90 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1522-winevent-cleanup-lowercasing-windows-is-case-insensitive.conf @@ -0,0 +1,148 @@ +# HELK winevent-cleanup-lowercasing-windows-is-case-sensitive filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + if [event_id] { + mutate { add_field => { "z_logstash_pipeline" => "1522" } } + + mutate { + lowercase => [ + "AccountDistinguishedName", + "AccountName", + "ActionName", + "AllowedToDelegateTo", + "ApplicationPath", + "AttributeLDAPDisplayName", + "AuthenticationPackageName", + "CACommonName", + "CallerProcessName", + "CalloutName", + "ClientName", + "ClientUserName", + "Command Name", + "CommandName", + "CommandPath", + "ConnectionName", + "CorruptedFilePath", + "CrashedAppName", + "CurrentDirectory", + "DCDomainName", + "DestinationHostname", + "DestinationPortName", + "DeviceName", + "DfsPath", + "DisplayName", + "displayName", + "DnsHostName", + "Domain", + "DriveName", + "DSName", + "DSObjectName", + "ExtensibleModulePath", + "FileName", + "Filename", + "FilePath", + "GPODisplayName", + "GPOFileSystemPath", + "GPOName", + "GroupName", + "HiveName", + "HomeDirectory", + "HomePath", + "HostName", + "Host Name", + "HostOSName", + "Host OS Name", + "HostOSservicepackName", + "Host OS service pack Name", + "HostNameChanged", + "ImageName", + "ImagePath", + "InstancePath", + "KeepPropertyCanonicalName", + "KeyName", + "Key Name", + "KeyName", + "KeyFilePath", + "LocalPath", + "LogonProcessName", + "MachineName", + "MappedName", + "MemberName", + "NAME", + "NameServer", + "NamespaceName", + "NetworkName", + "NewProcessName", + "NewTargetUserName", + "ObjectName", + "OldTargetUserName", + "OldTargetUserName", + "ParentProcessName", + "Path", + "PortName", + "PrincipalCNName", + "PrincipalDomainName", + "PrincipalSamName", + "Process Name", + "ProcessName", + "ProdessName", + "ProcessPath", + "processPath", + "ProfilePath", + "QNAME", + "QueryName", + "RelativeTargetName", + "SamAccountName", + "Script Name", + "ScriptName", + "ScriptPath", + "SecurityPackageName", + "ServerName", + "serverName", + "ServiceFileName", + "ServiceName", + "ServicePrincipalNames", + "SessionName", + "ShareName", + "ShareLocalPath", + "SnapshotPath", + "SourceHostname", + "SourcePortName", + "StateName", + "SubjectDomainName", + "SubjectUserDomainName", + "SubjectUserName", + "SwitchFName", + "TargetDomainName", + "TargetFilename", + "Targetname", + "TargetOutboundDomainName", + "TargetOutboundUserName", + "TargetProcessName", + "TargetRealm", + "TargetServerName", + "TargetUserName", + "TargetUserName", + "TaskName", + "TransportProtocolName", + "UncPath", + "User", + "user", + "UserName", + "Username", + "userName", + "username", + "UserPrincipalName", + "VhdFileName", + "VolumeName", + "wmiClassName", + "Workstation", + "WorkstationName", + "XPath" + ] + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/1531-winevent-sysmon-filter.conf b/docker/helk-logstash/trial/pipeline/1531-winevent-sysmon-filter.conf new file mode 100644 index 00000000..96f86bfe --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1531-winevent-sysmon-filter.conf @@ -0,0 +1,286 @@ +# HELK sysmon filter conf file +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +filter { + if [log_name] == "Microsoft-Windows-Sysmon/Operational"{ + mutate { add_field => { "z_logstash_pipeline" => "1531" } } + mutate { + rename => { + "[user][domain]" => "user_reporter_domain" + "[user][identifier]" => "user_reporter_sid" + "[user][name]" => "user_reporter_name" + "[user][type]" => "user_reporter_type" + "computer_name" => "host_name" + } + } + if [provider_guid] { + mutate { gsub => [ "provider_guid", "[{}]", "" ] } + } + if [RuleName] { + kv { + source => "RuleName" + field_split => "," + value_split => "=" + prefix => "mitre_" + transform_key => "lowercase" + } + } + if [Image] { + if [Image] =~ /^(\w*$)|^(\w*\..*$)/ { + mutate { copy => {"Image" => "process_name"} } + } + else { + grok { + match => { "Image" => ".*\\%{GREEDYDATA:process_name}" } + tag_on_failure => [ "_Image__grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + } + mutate { + rename => { + "Image" => "process_path" + "ProcessGuid" => "process_guid" + "ProcessId" => "process_id" + } + gsub => [ "process_guid", "[{}]", "" ] + } + } + if [ParentImage] { + grok { + match => { "ParentImage" => ".*\\%{GREEDYDATA:process_parent_name}" } + tag_on_failure => [ "_ParentImage__grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + } + if [Hashes] { + kv { + source => "Hashes" + field_split => "," + value_split => "=" + prefix => "hash_" + transform_key => "lowercase" + } + } + if [TargetImage] { + grok { + match => { "SourceImage" => ".*\\%{GREEDYDATA:process_name}" } + tag_on_failure => [ "_SourceImage__grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + grok { + match => { "TargetImage" => ".*\\%{GREEDYDATA:target_process_name}" } + tag_on_failure => [ "_TargetImage__grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + mutate { + rename => { + "SourceImage" => "process_path" + "SourceProcessGUID" => "process_guid" + "SourceProcessId" => "process_id" + "TargetImage" => "target_process_path" + "TargetProcessGUID" => "target_process_guid" + "TargetProcessId" => "target_process_id" + } + gsub => [ + "process_guid", "[{}]", "", + "target_process_guid", "[{}]", "" + ] + } + } + if [User] { + grok { + match => { "User" => "%{GREEDYDATA:user_domain}\\%{GREEDYDATA:user_name}" } + tag_on_failure => [ "_User__grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + } + if [event_id] == 1 { + mutate { + add_field => { "action" => "processcreate" } + rename => { + "CommandLine" => "process_command_line" + "CurrentDirectory" => "process_current_directory" + "ParentImage" => "process_parent_path" + "ParentCommandLine" => "process_parent_command_line" + "IntegrityLevel" => "process_integrity_level" + "LogonGuid" => "user_logon_guid" + "ParentProcessGuid" => "process_parent_guid" + "ParentProcessId" => "process_parent_id" + "TerminalSessionId" => "user_session_id" + "FileVersion" => "file_version" + "Description" => "file_description" + "Product" => "file_product" + "Company" => "file_company" + } + gsub => [ "process_parent_guid", "[{}]", "" ] + } + } + if [event_id] == 2 { + mutate { + add_field => { "action" => "filecreatetime" } + rename => { "TargetFileName" => "file_name" } + } + } + if [event_id] == 3 { + mutate { + add_field => { "action" => "networkconnect" } + rename => { + "DestinationHostname" => "dst_host_name" + "DestinationPort" => "dst_port" + "DestinationPortName" => "dst_port_name" + "Initiated" => "network_initiated" + "Protocol" => "network_protocol" + "SourceHostname" => "src_host_name" + "SourcePort" => "src_port" + "SourcePortName" => "src_port_name" + } + } + } + if [event_id] == 4 { + mutate { + add_field => { "action" => "sysmonservicestatechanged" } + rename => { + "State" => "service_state" + "Version" => "sysmon_version" + "SchemaVersion" => "sysmon_schema_version" + } + } + } + if [event_id] == 6 { + mutate { + add_field => { "action" => "driverload" } + rename => { + "ImageLoaded" => "driver_loaded" + "Signature" => "driver_signature" + "SignatureStatus" => "driver_signature_status" + "Signed" => "driver_signed" + } + } + } + if [event_id] == 7 { + mutate { + add_field => { "action" => "imageload" } + rename => { + "ImageLoaded" => "module_loaded" + "Signature" => "module_signature" + "SignatureStatus" => "module_signature_status" + "Signed" => "module_signed" + } + } + } + if [event_id] == 8 { + mutate { + add_field => { "action" => "createremotethread" } + rename => { + "NewThreadId" => "thread_new_id" + "StartAddress" => "thread_start_address" + "StartFunction" => "thread_start_function" + "StartModule" => "thread_start_module" + } + } + } + if [event_id] == 9 { + mutate { + add_field => { "action" => "rawaccessread" } + rename => { "Device" => "device_name" } + } + } + if [event_id] == 10 { + mutate { + add_field => { "action" => "processaccess" } + rename => { + "CallTrace" => "process_call_trace" + "GrantedAccess" => "process_granted_access" + "SourceThreadId" => "thread_id" + } + } + } + if [event_id] == 11 { + mutate { + add_field => { "action" => "filecreate" } + rename => { "TargetFilename" => "file_name" } + } + } + if [event_id] == 12 or [event_id] == 13 or [event_id] == 14 { + mutate { + add_field => { "action" => "registryevent" } + rename => { + "EventType" => "event_type" + "TargetObject" => "registry_key_path" + "Details" => "registry_key_value" + } + } + } + if [event_id] == 15 { + mutate { + add_field => { "action" => "filecreatestreamhash" } + rename => { + "TargetFilename" => "file_name" + "Hash" => "hash" + } + } + } + if [event_id] == 16 { + kv { + source => "ConfigurationFileHash" + value_split => "=" + prefix => "sysmon_config_hash_" + transform_key => "lowercase" + } + mutate { + add_field => { "action" => "sysmonconfigstatechanged" } + rename => { + "State" => "sysmon_configuration_state" + "Configuration" => "sysmon_configuration" + } + } + } + if [event_id] == 18 or [event_id] == 17 { + mutate { + add_field => { "action" => "pipeevent" } + rename => { "PipeName" => "pipe_name" } + } + } + if [event_id] == 19 or [event_id] == 20 or [event_id] == 21 { + mutate { + add_field => { "action" => "wmievent" } + rename => { + "EventType" => "wmi_event_type" + "Operation" => "wmi_operation" + "EventNamespace" => "wmi_namespace" + "Name" => "wmi_name" + "Query" => "wmi_query" + "Type" => "wmi_type" + "Destination" => "wmi_destination" + } + } + } + date { + timezone => "UTC" + match => [ "UtcTime", "YYYY-MM-dd HH:mm:ss.SSS" ] + tag_on_failure => [ "_sysmon_UtcTime_datefailure", "_sysmon_datefailure", "_dateparsefailure" ] + } + date { + timezone => "UTC" + match => [ "CreationUtcTime", "YYYY-MM-dd HH:mm:ss.SSS" ] + target => "@date_creation" + tag_on_failure => [ "_sysmon_CreationUtcTime_datefailure", "_sysmon_datefailure", "_dateparsefailure" ] + } + date { + timezone => "UTC" + match => [ "PreviousCreationUtcTime", "YYYY-MM-dd HH:mm:ss.SSS" ] + target => "@date_creation_previous" + tag_on_failure => [ "_sysmon_PreviousCreationUtcTime_datefailure", "_sysmon_datefailure", "_dateparsefailure" ] + } + mutate { + rename => { "User" => "user_account"} + remove_field => [ + "message", + "Hashes", + "ConfigurationFileHash", + "UtcTime", + "CreationUtcTime", + "PreviousCreationUtcTime", + "[user]" + ] + } + } +} + diff --git a/docker/helk-logstash/trial/pipeline/1532-winevent-security-filter.conf b/docker/helk-logstash/trial/pipeline/1532-winevent-security-filter.conf new file mode 100644 index 00000000..55cd5e29 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1532-winevent-security-filter.conf @@ -0,0 +1,807 @@ +# HELK winevent-security filter conf file +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +filter { + if [log_name] == "Security"{ + mutate { add_field => { "z_logstash_pipeline" => "1532" } } + if [ProcessName] { + grok { + match => { "ProcessName" => ".*\\%{GREEDYDATA:process_name}" } + tag_on_failure => [ "_ProcessName_grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + } + if [event_id] == 4611 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4611.md + mutate { + rename => { + "SubjectUserSid" => "user_sid" + "SubjectUserName" => "user_name" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "LogonProcessName" => "logon_process_name" + } + } + } + if [event_id] == 4616 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4616.md + mutate { + rename => { + "NewTime" => "@date_new_time" + "PreviousTime" => "@date_previous_time" + "SubjectUserSid" => "user_sid" + "SubjectUserName" => "user_name" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + } + } + } + if [event_id] == 4624 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4624.md + mutate { + rename => { + "SubjectUserSid" => "user_reporter_sid" + "SubjectUserName" => "user_reporter_name" + "SubjectDomainName" => "user_reporter_domain" + "SubjectLogonId" => "reporter_logon_id" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "TargetLogonId" => "user_logon_id" + "LogonType" => "logon_type" + "RestrictedAdminMode" => "logon_restricted_adminmode" + "VirtualAccount" => "logon_virtual_account" + "ElevatedToken" => "logon_elevated_token" + "TargetUserName" => "user_name" + "TargetDomainName" => "user_domain" + "TargetLinkedLogonId" => "user_linked_logon_id" + "TargetOutboundDomainName" => "user_network_account_domain" + "TargetOutboundUserName" => "user_network_account_name" + "TargetUserSid" => "user_sid" + "ImpersonationLevel" => "impersonation_level" + "LogonGuid" => "user_logon_guid" + "WorkstationName" => "src_host_name" + "IpPort" => "src_port" + "LogonProcessName" => "logon_process_name" + "AuthenticationPackageName" => "logon_authentication_package_name" + "TransmittedServices" => "logon_transmitted_services" + "LmPackageName" => "logon_package_name" + "KeyLength" => "logon_key_length" + } + } + if "logon_elevated_token" == "Yes"{ + mutate { add_tag => ["elevated_logon"] } + } + } + if [event_id] == 4625 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4625.md + mutate { + rename => { + "SubjectUserSid" => "user_reporter_sid" + "SubjectUserName" => "user_reporter_name" + "SubjectDomainName" => "user_reporter_domain" + "SubjectLogonId" => "reporter_logon_id" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "LogonType" => "logon_type" + "TargetUserName" => "user_name" + "TargetDomainName" => "user_domain" + "TargetUserSid" => "user_sid" + "WorkstationName" => "src_host" + "IpPort" => "src_port" + "LogonProcessName" => "logon_process_name" + "AuthenticationPackageName" => "logon_authentication_package_name" + "TransmittedServices" => "logon_transmitted_services" + "LmPackageName" => "logon_package_name" + "KeyLength" => "logon_key_length" + "FailureReason" => "logon_failure_reason" + "Status" => "logon_failure_status" + "SubStatus" => "logon_failure_substatus" + } + } + if "logon_elevated_token" == "Yes"{ + mutate { add_tag => ["elevated_logon"] } + } + } + if [event_id] == 4627 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4627.md + mutate { + rename => { + "SubjectUserSid" => "user_reporter_sid" + "SubjectUserName" => "user_reporter_name" + "SubjectDomainName" => "user_reporter_domain" + "SubjectLogonId" => "reporter_logon_id" + "TargetLogonId" => "user_logon_id" + "LogonType" => "logon_type" + "TargetUserName" => "user_name" + "TargetDomainName" => "user_domain" + "TargetUserSid" => "user_sid" + "GroupMembership" => "user_group_membership" + } + remove_field => [ + "EventCountTotal", + "EventIdx" + ] + } + } + if [event_id] == 4634 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4634.md + mutate { + rename => { + "LogonType" => "logon_type" + "TargetDomainName" => "user_domain" + "TargetLogonId" => "user_logon_id" + "TargetUserName" => "user_name" + "TargetUserSid" => "user_sid" + } + } + } + if [event_id] == 4647 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4647.md + mutate { + rename => { + "TargetDomainName" => "user_domain" + "TargetLogonId" => "user_logon_id" + "TargetUserName" => "user_name" + "TargetUserSid" => "user_sid" + } + } + } + if [event_id] == 4648 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4648.md + mutate { + rename => { + "SubjectUserSid" => "user_sid" + "SubjectUserName" => "user_name" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "LogonGuid" => "user_logon_guid" + "TargetUserName" => "target_user_name" + "TargetDomainName" => "target_user_domain" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "TargetLogonId" => "target_user_logon_id" + "TargetServerName" => "service_host_name" + "TargetInfo" => "service_host_info" + "TargetLogonGuid" => "target_user_logon_guid" + "IpPort" => "src_port" + } + gsub => [ "target_user_logon_guid", "[{}]", "" ] + } + } + if [event_id] == 4656 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4656.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "AccessList" => "object_access_list_requested" + "AccessMask" => "object_access_mask_requested" + "AccessReason" => "object_access_reason" + "ObjectName" => "object_name" + "ObjectServer" => "object_server" + "ObjectType" => "object_type" + "HandleId" => "object_access_handle_id" + "PrivilegeList" => "object_privilege_list" + "TransactionId" => "object_access_transaction_id" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "ResourceAttributes" => "object_resource_attributes" + "RestrictedSidCount" => "object_restricted_sid_count" + } + } + } + if [event_id] == 4657 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4657.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ObjectName" => "object_name" + "ObjectValueName" => "object_value_name" + "HandleId" => "object_access_handle_id" + "OperationType" => "object_operation_type" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "OldValueType" => "object_value_old_type" + "OldValue" => "object_value_old" + "NewValueType" => "object_value_new_type" + "NewValue" => "object_value_new" + } + } + } + if [event_id] == 4658 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4658.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "ObjectServer" => "object_server" + "HandleId" => "object_access_handle_id" + } + } + } + if [event_id] == 4661 or [event_id] == 4662 or [event_id] == 4663 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4661.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4662.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4663.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "Properties" => "object_properties" + "AccessMask" => "object_access_mask_requested" + "HandleId" => "object_access_handle_id" + "ObjectName" => "object_name" + "ObjectServer" => "object_server" + "ObjectType" => "object_type" + "AdditionalInfo2" => "object_additional_info2" + "OperationType" => "object_operation_type" + "AdditionalInfo" => "object_additional_info" + "AccessList" => "object_access_list_requested" + "ResourceAttributes" => "object_resource_attributes" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "AccessReason" => "object_access_reason" + "PrivilegeList" => "object_privilege_list" + "TransactionId" => "object_access_transaction_id" + "RestrictedSidCount" => "object_restricted_sid_count" + } + } + } + if [event_id] == 4670 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4670.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "HandleId" => "object_access_handle_id" + "NewSd" => "object_new_sddl" + "ObjectName" => "object_name" + "ObjectServer" => "object_server" + "ObjectType" => "object_type" + "OldSd" => "object_old_sddl" + } + } + } + if [event_id] == 4672 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4672.md + mutate { + rename => { + "PrivilegeList" => "logon_privileges_assigned" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + } + } + } + if [event_id] == 4673 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4673.md + mutate { + rename => { + "PrivilegeList" => "service_privilege_list" + "ObjectServer" => "object_server" + "Service" => "service_name" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + } + } + } + if [event_id] == 4674 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4674.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "AccessMask" => "object_access_mask_requested" + "HandleId" => "object_access_handle_id" + "ObjectName" => "object_name" + "ObjectServer" => "object_server" + "ObjectType" => "object_type" + "PrivilegeList" => "object_privilege_list" + } + } + } + if [event_id] == 4688 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4688.md + grok { + match => { "NewProcessName" => ".*\\%{GREEDYDATA:process_name}" } + tag_on_failure => [ "_NewProcessName_grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + if [ParentProcessName] { + grok { + match => { "ParentProcessName" => ".*\\%{GREEDYDATA:process_parent_name}" } + tag_on_failure => [ "_ParentProcessName_grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + } + mutate { + rename => { + "NewProcessId" => "process_id" + "NewProcessName" => "process_path" + "CommandLine" => "command_line" + "ParentProcessName" => "process_parent_path" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "user_principal_domain" + "TargetUserSid" => "user_principal_sid" + "TargetUserName" => "user_principal_name" + "TargetLogonId" => "user_principal_id" + "MandatoryLabel" => "process_mandatory_level" + "ProcessId" => "process_parent_id" + "TokenElevationType" => "process_token_elevation_type" + } + } + } + if [event_id] == 4689 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4689.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "Status" => "process_status" + } + } + } + if [event_id] == 4690 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4690.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "SourceHandleId" => "process_handle_id" + "SourceProcessId" => "process_id" + "TargetHandleId" => "target_process_handle_id" + "TargetProcessId" => "target_process_id" + } + } + } + if [event_id] == 4697 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4697.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ServiceAccount" => "service_account_name" + "ServiceFileName" => "service_image_path" + "ServiceName" => "service_name" + "ServiceStartType" => "service_start_type" + "ServiceType" => "service_type" + } + } + } + if [event_id] == 4701 or [event_id] == 4702 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4701.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4702.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TaskContentNew" => "task_new_content" + "TaskName" => "task_name" + } + } + } + if [event_id] == 4703 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4703.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "user_target_domain" + "TargetLogonId" => "user_target_logon_id" + "TargetUserSid" => "user_target_sid" + "TargetUserName" => "user_target_name" + "DisabledPrivilegeList" => "user_target_disabled_privilegelist" + "EnabledPrivilegeList" => "user_target_enabled_privilegelist" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + } + } + } + if [event_id] == 4719 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4719.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "AuditPolicyChanges" => "policy_changes" + "CategoryId" => "policy_category_id" + "SubcategoryGuid" => "policy_subcategory_guid" + "SubcategoryId" => "policy_subcategory_id" + } + } + } + if [event_id] == 4724 or [event_id] == 4725 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4724.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4725.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "user_target_domain" + "TargetSid" => "user_target_sid" + "TargetUserName" => "user_target_name" + } + } + } + if [event_id] == 4726 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4726.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "user_target_domain" + "TargetSid" => "user_target_sid" + "TargetUserName" => "user_target_name" + "PrivilegeList" => "user_privilege_list" + } + } + } + if [event_id] == 4728 or [event_id] == 4729 { + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "group_domain" + "TargetSid" => "group_sid" + "TargetUserName" => "group_name" + "MemberName" => "group_member_name" + "MemberSid" => "group_member_sid" + "PrivilegeList" => "group_privilege_list" + } + } + } + if [event_id] == 4732 or [event_id] == 4733 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4732.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4733.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "group_domain" + "TargetSid" => "group_sid" + "TargetUserName" => "group_name" + "MemberName" => "group_member_name" + "MemberSid" => "group_member_sid" + "PrivilegeList" => "group_privilege_list" + } + } + } + if [event_id] == 4738 or [event_id] == 4720 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4738.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4720.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "user_target_domain" + "TargetSid" => "user_target_sid" + "TargetUserName" => "user_target_name" + "AccountExpires" => "user_attribute_account_expires" + "AllowedToDelegateTo" => "user_attribute_allowed_todelegate" + "DisplayName" => "user_attribute_display_name" + "Dummy" => "user_attribute_dummy" + "HomeDirectory" => "user_attribute_home_directory" + "HomePath" => "user_attribute_home_path" + "LogonHours" => "user_attribute_logon_hours" + "NewUacValue" => "user_attribute_new_uacvalue" + "OldUacValue" => "user_attribute_old_uacvalue" + "PasswordLastSet" => "user_attribute_password_lastset" + "PrimaryGroupId" => "[user_attribute_primary_group_id" + "PrivilegeList" => "user_attribute_privilege_list" + "ProfilePath" => "user_attribute_profile_path" + "SamAccountName" => "user_attribute_samaccount_name" + "ScriptPath" => "user_attribute_script_path" + "SidHistory" => "user_attribute_sid_history" + } + } + } + if [event_id] == 4768 or [event_id] == 4769 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4768.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4769.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4771.md + mutate { + rename => { + "TargetDomainName" => "user_domain" + "TargetUserName" => "user_name" + "IpPort" => "src_port" + "PreAuthType" => "service_ticket_preauthtype" + "LogonGuid" => "user_logon_guid" + "ServiceName" => "service_ticket_name" + "ServiceSid" => "service_ticket_id" + "Status" => "service_ticket_status" + "TicketEncryptionType" => "ticket_encryption_type" + "TicketOptions" => "ticket_options" + "FailureCode" => "ticket_failure_code" + "TransmittedServices" => "service_ticket_requested" + "TargetSid" => "user_sid" + } + } + } + if [event_id] == 4797 { + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "service_enumerated_domain" + "TargetUserName" => "service_enumerated_name" + "Workstation" => "host_name" + } + } + } + if [event_id] == 4798 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4798.md + mutate { + rename => { + "CallerProcessId" => "process_id" + "CallerProcessName" => "process_path" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "user_domain_enumerated" + "TargetSid" => "user_sid_enumerated" + "TargetUserName" => "user_name_enumerated" + } + } + } + if [event_id] == 4799 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4799.md + mutate { + rename => { + "CallerProcessId" => "process_id" + "CallerProcessName" => "process_path" + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "TargetDomainName" => "group_domain_enumerated" + "TargetSid" => "group_sid_enumerated" + "TargetUserName" => "group_name_enumerated" + } + } + } + if [event_id] == 4800 or [event_id] == 4801 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4800.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4801.md + mutate { + rename => { + "TargetDomainName" => "user_domain" + "TargetUserSid" => "user_sid" + "TargetUserName" => "user_name" + "TargetLogonId" => "user_logon_id" + "SessionId" => "user_session_id" + } + } + } + if [event_id] == 4907 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4907.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ProcessId" => "process_id" + "ProcessName" => "process_path" + "HandleId" => "object_access_handle_id" + "NewSd" => "object_new_sddl" + "ObjectName" => "object_name" + "ObjectServer" => "object_server" + "ObjectType" => "object_type" + "OldSd" => "object_old_sddl" + } + } + } + if [event_id] == 4957 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4957.md + mutate { + rename => { + "RuleAttr" => "firewall_rule_attr" + "RuleId" => "firewall_rule_id" + "RuleName" => "firewall_rule_name" + } + } + } + if [event_id] == 5058 or [event_id] == 5059 or [event_id] == 5061 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5058.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5059.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5061.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "AlgorithmName" => "key_algorithm_name" + "KeyName" => "key_name" + "KeyType" => "key_type" + "Operation" => "key_operation" + "ProviderName" => "key_provider_name" + "ReturnCode" => "key_return_code" + } + } + } + if [event_id] == 5136 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5136.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "OpCorrelationID" => "dsoperation_correlation_id" + "AppCorrelationID" => "dsoperation_app_correlation_id" + "DSName" => "dsobject_domain" + "DSType" => "dsobject_domain_type" + "ObjectDN" => "dsobject_dn" + "ObjectGUID" => "dsobject_guid" + "ObjectClass" => "dsobject_class" + "AttributeLDAPDisplayName" => "dsobject_attribute_name" + "AttributeSyntaxOID" => "dsobject_attribute_type" + "AttributeValue" => "dsobject_attribute_value" + "OperationType" => "dsoperation_type" + } + } + } + if [event_id] == 5140 or [event_id] == 5145 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5140.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5145.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "AccessList" => "object_access_list_requested" + "AccessMask" => "object_access_mask_requested" + "AccessReason" => "user_access_reason" + "IpPort" => "src_port" + "ObjectType" => "object_type" + "RelativeTargetName" => "share_relative_target_name" + "ShareLocalPath" => "share_local_path" + "ShareName" => "share_name" + } + } + } + if [event_id] == 5152 or [event_id] == 5154 or [event_id] == 5156 or [event_id] == 5158 or [event_id] == 5157 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5152.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5154.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5156.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5157.md + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5158.md + grok { + match => { + "Application" => ".*\\%{GREEDYDATA:process_name}" + } + tag_on_failure => [ "_Application_grokparsefailure", "_grokparsefailure", "_parsefailure" ] + } + mutate { + rename => { + "Application" => "process_path" + "DestPort" => "dst_port" + "Direction" => "network_direction" + "FilterRTID" => "network_filter_rtid" + "LayerName" => "network_layer_name" + "LayerRTID" => "network_layer_rtid" + "ProcessID" => "process_id" + "Protocol" => "network_protocol" + "RemoteMachineID" => "dst_host_name_id" + "RemoteUserID" => "dst_user_id" + "SourcePort" => "src_port" + "ProcessId" => "process_id" + } + } + } + if [event_id] == 5447 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5447.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "AccessList" => "object_access_list_requested" + "AccessMask" => "object_access_mask_requested" + "AccessReason" => "user_access_reason" + "IpPort" => "src_port" + "ObjectType" => "object_type" + "RelativeTargetName" => "share_relative_target_name" + "ShareLocalPath" => "share_local_path" + "ShareName" => "share_name" + "Action" => "filtering_action" + "CalloutKey" => "filtering_callout_key" + "CalloutName" => "filtering_callout_name" + "ChangeType" => "filtering_change_type" + "Conditions" => "filtering_conditions" + "FilterId" => "filtering_id" + "FilterKey" => "filtering_key" + "FilterName" => "filtering_name" + "FilterType" => "filtering_type" + "LayerId" => "filtering_layer_id" + "LayerKey" => "filtering_layer_key" + "LayerName" => "filtering_layer_name" + "ProcessId" => "process_id" + "ProviderKey" => "filtering_provider_key" + "ProviderName" => "filtering_provider_name" + "UserName" => "user_name" + "UserSid" => "user_sid" + "Weight" => "filtering_weight" + } + } + } + if [event_id] == 6416 { + # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-6416.md + mutate { + rename => { + "SubjectDomainName" => "user_domain" + "SubjectLogonId" => "user_logon_id" + "SubjectUserName" => "user_name" + "SubjectUserSid" => "user_sid" + "ClassId" => "device_class_id" + "ClassName" => "device_class_name" + "CompatibleIds" => "device_compatible_ids" + "DeviceDescription" => "device_description" + "DeviceId" => "device_id" + "LocationInformation" => "device_location_information" + "VendorIds" => "device_vendor_ids" + } + } + } + mutate { rename => { "computer_name" => "host_name" } } + } +} + diff --git a/helk-logstash/trial/pipeline/13-winevent-system-filter.conf b/docker/helk-logstash/trial/pipeline/1533-winevent-system-filter.conf similarity index 50% rename from helk-logstash/trial/pipeline/13-winevent-system-filter.conf rename to docker/helk-logstash/trial/pipeline/1533-winevent-system-filter.conf index f64c9e59..0bc5a302 100644 --- a/helk-logstash/trial/pipeline/13-winevent-system-filter.conf +++ b/docker/helk-logstash/trial/pipeline/1533-winevent-system-filter.conf @@ -5,29 +5,32 @@ filter { if [log_name] == "System" { + mutate { add_field => { "z_logstash_pipeline" => "1533" } } if [event_id] == 7045 { # https://cyberwardog.blogspot.com/2017/04/chronicles-of-threat-hunter-hunting-for_11.html mutate { rename => { - "[event_data][AccountName]" => "service_account_name" - "[event_data][ImagePath]" => "service_image_path" - "[event_data][ServiceName]" => "service_name" - "[event_data][ServiceType]" => "service_type" - "[event_data][StartType]" => "service_start_type" + "AccountName" => "service_account_name" + "ImagePath" => "service_image_path" + "ServiceName" => "service_name" + "ServiceType" => "service_type" + "StartType" => "service_start_type" } - add_tag => ["new_service"] - remove_field => ["message"] - remove_field => "host" - remove_field => "process_id" + add_tag => [ "new_service" ] + remove_field => [ + "message", + "host", + "process_id" + ] } } if [event_id] == 16 { mutate { rename => { - "[event_data][HiveName]" => "hive_name" - "[event_data][HiveNameLength]" => "hive_name_length" - "[event_data][KeysUpdated]" => "hive_keys_updated" - "[event_data][DirtyPages]" => "hive_dirty_pages" + "HiveName" => "hive_name" + "HiveNameLength" => "hive_name_length" + "KeysUpdated" => "hive_keys_updated" + "DirtyPages" => "hive_dirty_pages" } } } diff --git a/helk-logstash/trial/pipeline/14-winevent-application-filter.conf b/docker/helk-logstash/trial/pipeline/1534-winevent-application-filter.conf similarity index 61% rename from helk-logstash/trial/pipeline/14-winevent-application-filter.conf rename to docker/helk-logstash/trial/pipeline/1534-winevent-application-filter.conf index 32de1bc4..1e12b477 100644 --- a/helk-logstash/trial/pipeline/14-winevent-application-filter.conf +++ b/docker/helk-logstash/trial/pipeline/1534-winevent-application-filter.conf @@ -6,19 +6,16 @@ filter { if [log_name] == "Application"{ if [source_name] == "Microsoft-Windows-Security-SPP"{ + mutate { add_field => { "z_logstash_pipeline" => "1534" } } if [event_id] == 16384 { mutate { rename => { - "[event_data][param1]" => "spp_restart_scheduled" - "[event_data][param2]" => "spp_restart_reason" + "param1" => "spp_restart_scheduled" + "param2" => "spp_restart_reason" } } } } - mutate { - rename => { - "computer_name" => "host_name" - } - } + mutate { rename => { "computer_name" => "host_name" } } } } \ No newline at end of file diff --git a/helk-logstash/trial/pipeline/15-winevent-wmiactivity-filter.conf b/docker/helk-logstash/trial/pipeline/1535-winevent-wmiactivity-filter.conf similarity index 97% rename from helk-logstash/trial/pipeline/15-winevent-wmiactivity-filter.conf rename to docker/helk-logstash/trial/pipeline/1535-winevent-wmiactivity-filter.conf index 44c3a024..c9b9ec5a 100644 --- a/helk-logstash/trial/pipeline/15-winevent-wmiactivity-filter.conf +++ b/docker/helk-logstash/trial/pipeline/1535-winevent-wmiactivity-filter.conf @@ -5,6 +5,7 @@ filter { if [log_name] == "Microsoft-Windows-WMI-Activity/Operational"{ + mutate { add_field => { "z_logstash_pipeline" => "1535" } } mutate { rename => { "[user][domain]" => "user_reporter_domain" @@ -14,10 +15,10 @@ filter { "computer_name" => "host_name" } } - if [event_data][User] { + if [User] { grok { - match => { "[event_data][User]" => "%{GREEDYDATA:user_domain}\\%{GREEDYDATA:user_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] + match => { "User" => "%{GREEDYDATA:user_domain}\\%{GREEDYDATA:user_name}" } + tag_on_failure => [ "_User_grokparsefailure", "_grokparsefailure", "_parsefailure" ] } } @@ -236,11 +237,7 @@ filter { } # Common to all events - mutate { - convert => { - "process_id" => "integer" - } - } + mutate { convert => { "process_id" => "integer" } } } } diff --git a/docker/helk-logstash/trial/pipeline/1543-winevent-conversions-process-cli-filter.conf b/docker/helk-logstash/trial/pipeline/1543-winevent-conversions-process-cli-filter.conf new file mode 100644 index 00000000..3021dc0d --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1543-winevent-conversions-process-cli-filter.conf @@ -0,0 +1,47 @@ +# HELK winevent-conversions-process-cli filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + + if [event_id] { + + if [user_logon_id] { + mutate { add_field => { "z_logstash_pipeline" => "1543_1" } } + mutate { gsub => [ "user_logon_id", "0x", "" ]} + ruby { + code => "event.set('user_logon_id', event.get('user_logon_id').to_s.hex)" + tag_on_exception => "_rubyexception_1543_1" + } + } + if [process_id] { + mutate { add_field => { "z_logstash_pipeline" => "1543_2" } } + mutate { gsub => [ "process_id", "0x", "" ]} + ruby { + code => "event.set('process_id', event.get('process_id').to_s.hex)" + tag_on_exception => "_rubyexception_1543_2" + } + } + if [process_parent_id] { + mutate { add_field => { "z_logstash_pipeline" => "1543_3" } } + mutate { gsub => [ "process_parent_id", "0x", "" ]} + ruby { + code => "event.set('process_parent_id', event.get('process_parent_id').to_s.hex)" + tag_on_exception => "_rubyexception_1543_3" + } + } + if [target_process_id] { + mutate { add_field => { "z_logstash_pipeline" => "1543_4" } } + mutate { gsub => [ "target_process_id", "0x", "" ]} + ruby { + code => "event.set('target_process_id', event.get('target_process_id').to_s.hex)" + tag_on_exception => "_rubyexception_1543_4" + } + } + if [ProdessName] { + mutate { rename => { "ProdessName" => "ProcessName" } } + } + + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/1544-winevent-cleanup-other.conf b/docker/helk-logstash/trial/pipeline/1544-winevent-cleanup-other.conf new file mode 100644 index 00000000..2833fcda --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/1544-winevent-cleanup-other.conf @@ -0,0 +1,13 @@ +# HELK winevent-cleanup-other filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + if [event_id] { + if [user_logon_guid] { + mutate { add_field => { "z_logstash_pipeline" => "1544" } } + mutate { gsub => [ "user_logon_guid", "[{}]", "" ] } + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/2511-winevent-powershell-filter.conf b/docker/helk-logstash/trial/pipeline/2511-winevent-powershell-filter.conf new file mode 100644 index 00000000..5ecc5ad2 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/2511-winevent-powershell-filter.conf @@ -0,0 +1,207 @@ +# HELK powershell filter conf file +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# Reference: +# Nate Guagenti (@neu5ron) https://gist.github.com/neu5ron/450289373db61d5c8d7378e79455ef07#file-511-windows-event-powershell-operational-conf + +filter { + if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell" { + if [event_id] == 4103 { + mutate { add_field => { "z_logstash_pipeline" => "2511_1" } } + mutate { + add_field => { + "PayloadInvocation" => "%{Payload}" + "PayloadParams" => "%{Payload}" + } + gsub => [ + "ContextInfo", " ", "", + "ContextInfo", " = ", "=" + ] + } + kv { + source => "ContextInfo" + field_split => "\r\n" + value_split => "=" + remove_char_key => " " + allow_duplicate_values => false + include_keys => [ + "Severity", + "HostName", + "HostVersion", + "HostID", + "HostApplication", + "EngineVersion", + "RunspaceID", + "PipelineID", + "CommandName", + "CommandType", + "ScriptName", + "CommandPath", + "SequenceNumber", + "ConnectedUser", + "ShellID" + ] + } + mutate { + gsub => [ + "PayloadInvocation", "CommandInvocation\(.*\)", "CommandInvocation", + "PayloadInvocation", "ParameterBinding.*\r\n", "", + "PayloadParams", "CommandInvocation.*\r\n", "", + "PayloadParams", "ParameterBinding\(\S+\): ", "|||SPLITMEHEHE|||", + "Payload", "CommandInvocation.*\r\n", "", + "Payload", "ParameterBinding.*\r\n", "" + ] + } + kv { + source => "PayloadInvocation" + field_split => "\n" + value_split => ":" + allow_duplicate_values => false + target => "[powershell]" + include_keys => [ "CommandInvocation" ] + } + ruby { + code => " + params_split = event.get('PayloadParams').split('|||SPLITMEHEHE|||') + params_split = params_split.drop(1) + params_split_length = params_split.length + all_names = Array.new + all_values = Array.new + all_values_non_alphanumeric = Array.new + all_contains_net = Array.new + all_ascii_only = Array.new + + for param in params_split + slice_and_dice = param.index('; value=') + param_name = param.slice(6..slice_and_dice-2) + param_value = param.slice(param.index('value=')..-1)[6..-1] + param_value = param_value.strip + param_value[0] = '' + param_value[-1] = '' + value_non_alphanumeric = param_value.gsub(/[A-Za-z0-9\s]+/i, '') + value_lowercased = param_value.downcase + if !param_value.nil? + all_values.push(param_value) + all_values_non_alphanumeric.push(value_non_alphanumeric) + end + if !param_name.nil? + all_names.push(param_name) + end + end + all_names = all_names.uniq + all_values = all_values.uniq + all_values_non_alphanumeric = all_values_non_alphanumeric.uniq + event.set('[powershell][param][name]', all_names) + event.set('[powershell][param][value]', all_values) + event.set('[powershell][param][value_nonalphanumeric]', all_values_non_alphanumeric) + " + tag_on_exception => "_rubyexception_2511_1" + } + prune { blacklist_values => [ "Payload", "^$" ] } + mutate { + rename => { + "CommandName" => "[powershell][command][name]" + "CommandPath" => "[powershell][command][path]" + "CommandType" => "[powershell][command][type]" + "ConnectedUser" => "[powershell][connected_user]" + "EngineVersion" => "[powershell][engine_version]" + "HostApplication" => "[powershell][host][application]" + "HostID" => "[powershell][host][id]" + "HostName" => "[powershell][host][name]" + "HostVersion" => "[powershell][host][version]" + "PipelineID" => "[powershell][pipeline_id]" + "RunspaceID" => "[powershell][runspace_id]" + "Scriptname" => "[powershell][script][name]" + "SequenceNumber" => "[powershell][sequence_number]" + "ShellID" => "[powershell][shell_id]" + "Payload" => "[powershell][remaining_payload]" + } + remove_field => [ + "Severity", + "EventType", + "Keywords", + "message", + "Opcode", + "PayloadInvocation", + "PayloadParams", + "Payload", + "ContextInfo" + ] + convert => { + "[powershell][pipeline_id]" => "integer" + "[powershell][sequence_number]" => "integer" + } + } + } + if [event_id] == 4104 { + mutate { add_field => { "z_logstash_pipeline" => "2511_2" } } + if [ScriptBlockText] { + mutate { remove_field => [ "message" ] } + } + else { + # Lets use GSUB to make sure we can get things to split on / make it easier more efficient to split on + grok { + match => { + "message" => "^Creating Scriptblock text \(%{INT:MessageNumber} of %{INT:MessageTotal}\):\r\n%{GREEDYDATA:ScriptBlockText}\r\n\r\nScriptBlock ID: %{UUID:ScriptBlockId}\r\nPath: %{DATA:Path}$" + } + break_on_match => true + keep_empty_captures => false + named_captures_only => true + # Sometimes the ScriptBlockText literally could be empty, so you may see this in _grokparesfailure and would then cause a ruby failure below. + tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] + tag_on_timeout => "_groktimeout" + # Timeout 1.5 seconds + timeout_millis => 1500 + remove_field => [ "message" ] + } + } + mutate { + rename => { + "MessageNumber" => "[powershell][scriptblock][message_number]" + "MessageTotal" => "[powershell][scriptblock][message_total]" + "ScriptBlockId" => "[powershell][scriptblock][id]" + "ScriptBlockText" => "[powershell][scriptblock][text]" + "Path" => "[powershell][script][path]" + } + } + } + if [event_id] == 400 or [event_id] == 600 { + kv { + source => "param3" + field_split => "\n" + value_split => "=" + trim_key => "\t" + allow_duplicate_values => false + } + mutate { + rename => { + "ProviderName" => "[powershell][providername]" + "NewProviderState" => "[powershell][newproviderstate]" + "SequenceNumber" => "[powershell][sequence_number]" + "HostName" => "[powershell][host][name]" + "HostVersion" => "[powershell][host][version]" + "HostId" => "[powershell][host][id]" + "HostApplication" => "[powershell][host][application]" + "EngineVersion" => "[powershell][engine_version]" + "RunspaceId" => "[powershell][runspace_id]" + "PipelineId" => "[powershell][pipeline_id]" + "CommandName" => "[powershell][command][name]" + "CommandType" => "[powershell][command][type]" + "ScriptName" => "[powershell][script][name]" + "CommandPath" => "[powershell][command][path]" + "CommandLine" => "[powershell][command][line]" + "NewEngineState" => "[powershell][newengine_state]" + "PreviousEngineState" => "[powershell][previousengine_state]" + } + remove_field => [ + "message", + "param1", + "param2", + "param3" + ] + } + } + } +} diff --git a/helk-logstash/trial/pipeline/16-winevent-security-schtasks-filter.conf b/docker/helk-logstash/trial/pipeline/2512-winevent-security-schtasks-filter.conf similarity index 97% rename from helk-logstash/trial/pipeline/16-winevent-security-schtasks-filter.conf rename to docker/helk-logstash/trial/pipeline/2512-winevent-security-schtasks-filter.conf index 0dd72d05..86f0ce80 100644 --- a/helk-logstash/trial/pipeline/16-winevent-security-schtasks-filter.conf +++ b/docker/helk-logstash/trial/pipeline/2512-winevent-security-schtasks-filter.conf @@ -2,11 +2,12 @@ # HELK build Stage: Alpha # Author: Nate Guagenti (@neu5ron) # License: GPL-3.0 + filter { if [log_name] == "Security" { # event_id 4698 for Created Scheduled Task if [event_id] == 4698 { - mutate { add_field => { "z_logstash_pipeline" => "0511" } } + mutate { add_field => { "z_logstash_pipeline" => "2512" } } # Copy Message field incase we bork/mess it up mutate { copy => { "Message" => "deleteme" } } # #TONOTE: encoding is UTF-16 diff --git a/docker/helk-logstash/trial/pipeline/8012-dst-ip-cleanups-filter.conf b/docker/helk-logstash/trial/pipeline/8012-dst-ip-cleanups-filter.conf new file mode 100644 index 00000000..fe87a829 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8012-dst-ip-cleanups-filter.conf @@ -0,0 +1,86 @@ +# HELK dst-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" + + if [dst_ip_addr] { + # Add pipeline field + mutate { add_field => { "z_logstash_pipeline" => "8012" } } + ruby { + code => " + temp_ip_addresses = event.get('dst_ip_addr') + ip_addresses = Array.new + + # Determine if the IP field is an array if not make it an array + if temp_ip_addresses.is_a? Enumerable + ip_addresses = temp_ip_addresses.uniq + else + ip_addresses.push(temp_ip_addresses) + end + + clean_ip_addresses_v4 = Array.new + clean_ip_addresses_v6 = Array.new + not_ip_addresses = Array.new + + ipv6_regex = /((([0-9a-f]{1,4}:){7}([0-9a-f]{1,4}|:))|(([0-9a-f]{1,4}:){6}(:[0-9a-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){5}(((:[0-9a-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){4}(((:[0-9a-f]{1,4}){1,3})|((:[0-9a-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){3}(((:[0-9a-f]{1,4}){1,4})|((:[0-9a-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){2}(((:[0-9a-f]{1,4}){1,5})|((:[0-9a-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){1}(((:[0-9a-f]{1,4}){1,6})|((:[0-9a-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9a-f]{1,4}){1,7})|((:[0-9a-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?/ + + for ip_address in ip_addresses + #### General Cleanup + # Remove ending '.' + ip_address = ip_address.chomp + # Remove preceeding '.'# Don't ask.. reverse + chomp + reverse up to 16 times faster + ip_address = ip_address.reverse.chomp('.').reverse + # Remove ending or beginning whitespace + ip_address = ip_address.lstrip.rstrip + # Remove things that would make an IP a share but we want the IP :) + ip_address = ip_address.gsub(/^\\:?/, '') + # Downcase/lowercase for checking if possible ipv6 + ip_address = ip_address.downcase + + # Likely IPv4 #Check if at most 15 characters (a fully represented IPv4 address with periods\.) and minimum of 7 + ip_address_length = ip_address.length + if !ip_address.include?(':') && !( /[a-z]/ === ip_address ) && ip_address_length <= 15 && ip_address_length >= 7 && ip_address.ascii_only? + # Remove any preceeding zeroes in each octet + temp_ip = Array.new + ip_address.split('.').each do |octet| + octet = octet.to_i.to_s + temp_ip.push(octet) + end + ip_address = temp_ip.join('.') + clean_ip_addresses_v4.push(ip_address) + + # Likely IPv6 #Check if at most 39 characters (a fully represented IPv6 address with colons\:) and minimum of 2 + elsif ip_address_length <= 39 && ip_address_length >= 2 && ip_address.ascii_only? + if ipv6_regex === ip_address + clean_ip_addresses_v6.push(ip_address) + else + not_ip_addresses.push(ip_address) + end + else + not_ip_addresses.push(ip_address) + end + end + + # Set the new IP addresses + if !clean_ip_addresses_v4.empty? + event.set('dst_ip_addr', clean_ip_addresses_v4) + # Set the number of ip addresses so we can use array or non array later in pipeline + event.set('temp_number_of_ipv4_dst_addresses', clean_ip_addresses_v4.length) + else + event.remove('dst_ip_addr') + end + if !clean_ip_addresses_v6.empty? + event.set('ipv6_dst_addr', clean_ip_addresses_v6) + #TODO:eventually set ipv6 number of ip addresses + end + if !not_ip_addresses.empty? + event.set('not_ip_dst', not_ip_addresses) + end + " + tag_on_exception => "_rubyexception_8012" + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8013-src-ip-cleanups-filter.conf b/docker/helk-logstash/trial/pipeline/8013-src-ip-cleanups-filter.conf new file mode 100644 index 00000000..7987070c --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8013-src-ip-cleanups-filter.conf @@ -0,0 +1,86 @@ +# HELK src-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" + + if [src_ip_addr] { + # Add pipeline field + mutate { add_field => { "z_logstash_pipeline" => "8013" } } + ruby { + code => " + temp_ip_addresses = event.get('src_ip_addr') + ip_addresses = Array.new + + # Determine if the IP field is an array if not make it an array + if temp_ip_addresses.is_a? Enumerable + ip_addresses = temp_ip_addresses.uniq + else + ip_addresses.push(temp_ip_addresses) + end + + clean_ip_addresses_v4 = Array.new + clean_ip_addresses_v6 = Array.new + not_ip_addresses = Array.new + + ipv6_regex = /((([0-9a-f]{1,4}:){7}([0-9a-f]{1,4}|:))|(([0-9a-f]{1,4}:){6}(:[0-9a-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){5}(((:[0-9a-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){4}(((:[0-9a-f]{1,4}){1,3})|((:[0-9a-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){3}(((:[0-9a-f]{1,4}){1,4})|((:[0-9a-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){2}(((:[0-9a-f]{1,4}){1,5})|((:[0-9a-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){1}(((:[0-9a-f]{1,4}){1,6})|((:[0-9a-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9a-f]{1,4}){1,7})|((:[0-9a-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?/ + + for ip_address in ip_addresses + #### General Cleanup + # Remove ending '.' + ip_address = ip_address.chomp + # Remove preceeding '.'# Don't ask.. reverse + chomp + reverse up to 16 times faster + ip_address = ip_address.reverse.chomp('.').reverse + # Remove ending or beginning whitespace + ip_address = ip_address.lstrip.rstrip + # Remove things that would make an IP a share but we want the IP :) + ip_address = ip_address.gsub(/^\\:?/, '') + # Downcase/lowercase for checking if possible ipv6 + ip_address = ip_address.downcase + + # Likely IPv4 #Check if at most 15 characters (a fully represented IPv4 address with periods\.) and minimum of 7 + ip_address_length = ip_address.length + if !ip_address.include?(':') && !( /[a-z]/ === ip_address ) && ip_address_length <= 15 && ip_address_length >= 7 && ip_address.ascii_only? + # Remove any preceeding zeroes in each octet + temp_ip = Array.new + ip_address.split('.').each do |octet| + octet = octet.to_i.to_s + temp_ip.push(octet) + end + ip_address = temp_ip.join('.') + clean_ip_addresses_v4.push(ip_address) + + # Likely IPv6 #Check if at most 39 characters (a fully represented IPv6 address with colons\:) and minimum of 2 + elsif ip_address_length <= 39 && ip_address_length >= 2 && ip_address.ascii_only? + if ipv6_regex === ip_address + clean_ip_addresses_v6.push(ip_address) + else + not_ip_addresses.push(ip_address) + end + else + not_ip_addresses.push(ip_address) + end + end + + # Set the new IP addresses + if !clean_ip_addresses_v4.empty? + event.set('src_ip_addr', clean_ip_addresses_v4) + # Set the number of ip addresses so we can use array or non array later in pipeline + event.set('temp_number_of_ipv4_src_addresses', clean_ip_addresses_v4.length) + else + event.remove('src_ip_addr') + end + if !clean_ip_addresses_v6.empty? + event.set('ipv6_src_addr', clean_ip_addresses_v6) + #TODO:eventually set ipv6 number of ip addresses + end + if !not_ip_addresses.empty? + event.set('not_ip_src', not_ip_addresses) + end + " + tag_on_exception => "_rubyexception_8013" + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8014-dst-nat-ip-cleanups-filter.conf b/docker/helk-logstash/trial/pipeline/8014-dst-nat-ip-cleanups-filter.conf new file mode 100644 index 00000000..47bb8821 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8014-dst-nat-ip-cleanups-filter.conf @@ -0,0 +1,86 @@ +# HELK dst-nat-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" + + if [dst_nat_ip_addr] { + # Add pipeline field + mutate { add_field => { "z_logstash_pipeline" => "8014" } } + ruby { + code => " + temp_ip_addresses = event.get('dst_nat_ip_addr') + ip_addresses = Array.new + + # Determine if the IP field is an array if not make it an array + if temp_ip_addresses.is_a? Enumerable + ip_addresses = temp_ip_addresses.uniq + else + ip_addresses.push(temp_ip_addresses) + end + + clean_ip_addresses_v4 = Array.new + clean_ip_addresses_v6 = Array.new + not_ip_addresses = Array.new + + ipv6_regex = /((([0-9a-f]{1,4}:){7}([0-9a-f]{1,4}|:))|(([0-9a-f]{1,4}:){6}(:[0-9a-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){5}(((:[0-9a-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){4}(((:[0-9a-f]{1,4}){1,3})|((:[0-9a-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){3}(((:[0-9a-f]{1,4}){1,4})|((:[0-9a-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){2}(((:[0-9a-f]{1,4}){1,5})|((:[0-9a-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){1}(((:[0-9a-f]{1,4}){1,6})|((:[0-9a-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9a-f]{1,4}){1,7})|((:[0-9a-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?/ + + for ip_address in ip_addresses + #### General Cleanup + # Remove ending '.' + ip_address = ip_address.chomp + # Remove preceeding '.'# Don't ask.. reverse + chomp + reverse up to 16 times faster + ip_address = ip_address.reverse.chomp('.').reverse + # Remove ending or beginning whitespace + ip_address = ip_address.lstrip.rstrip + # Remove things that would make an IP a share but we want the IP :) + ip_address = ip_address.gsub(/^\\:?/, '') + # Downcase/lowercase for checking if possible ipv6 + ip_address = ip_address.downcase + + # Likely IPv4 #Check if at most 15 characters (a fully represented IPv4 address with periods\.) and minimum of 7 + ip_address_length = ip_address.length + if !ip_address.include?(':') && !( /[a-z]/ === ip_address ) && ip_address_length <= 15 && ip_address_length >= 7 && ip_address.ascii_only? + # Remove any preceeding zeroes in each octet + temp_ip = Array.new + ip_address.split('.').each do |octet| + octet = octet.to_i.to_s + temp_ip.push(octet) + end + ip_address = temp_ip.join('.') + clean_ip_addresses_v4.push(ip_address) + + # Likely IPv6 #Check if at most 39 characters (a fully represented IPv6 address with colons\:) and minimum of 2 + elsif ip_address_length <= 39 && ip_address_length >= 2 && ip_address.ascii_only? + if ipv6_regex === ip_address + clean_ip_addresses_v6.push(ip_address) + else + not_ip_addresses.push(ip_address) + end + else + not_ip_addresses.push(ip_address) + end + end + + # Set the new IP addresses + if !clean_ip_addresses_v4.empty? + event.set('dst_nat_ip_addr', clean_ip_addresses_v4) + # Set the number of ip addresses so we can use array or non array later in pipeline + event.set('temp_number_of_ipv4_dst_nat_addresses', clean_ip_addresses_v4.length) + else + event.remove('dst_nat_ip_addr') + end + if !clean_ip_addresses_v6.empty? + event.set('ipv6_dst_nat_addr', clean_ip_addresses_v6) + #TODO:eventually set ipv6 number of ip addresses + end + if !not_ip_addresses.empty? + event.set('not_ip_dst_nat', not_ip_addresses) + end + " + tag_on_exception => "_rubyexception_8014" + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8015-src-nat-ip-cleanups-filter.conf b/docker/helk-logstash/trial/pipeline/8015-src-nat-ip-cleanups-filter.conf new file mode 100644 index 00000000..f0bb431c --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8015-src-nat-ip-cleanups-filter.conf @@ -0,0 +1,86 @@ +# HELK src-nat-ip-cleanups filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + #TONOTE: On all IPs below, even though we have determined that it is IPv4 and IPv6 and will alter rename the field if it is an IPv6 address differently than IPv4 -- We still need to keep it as that single field for this configuration -- because sometimes a single IP type can already have both IPv6 and IPv4 from before -- examples are Cisco ASA Logs and Windows EventID "4769" in Channel "Security" + + if [src_nat_ip_addr] { + # Add pipeline field + mutate { add_field => { "z_logstash_pipeline" => "8015" } } + ruby { + code => " + temp_ip_addresses = event.get('src_nat_ip_addr') + ip_addresses = Array.new + + # Determine if the IP field is an array if not make it an array + if temp_ip_addresses.is_a? Enumerable + ip_addresses = temp_ip_addresses.uniq + else + ip_addresses.push(temp_ip_addresses) + end + + clean_ip_addresses_v4 = Array.new + clean_ip_addresses_v6 = Array.new + not_ip_addresses = Array.new + + ipv6_regex = /((([0-9a-f]{1,4}:){7}([0-9a-f]{1,4}|:))|(([0-9a-f]{1,4}:){6}(:[0-9a-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){5}(((:[0-9a-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){4}(((:[0-9a-f]{1,4}){1,3})|((:[0-9a-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){3}(((:[0-9a-f]{1,4}){1,4})|((:[0-9a-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){2}(((:[0-9a-f]{1,4}){1,5})|((:[0-9a-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){1}(((:[0-9a-f]{1,4}){1,6})|((:[0-9a-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9a-f]{1,4}){1,7})|((:[0-9a-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?/ + + for ip_address in ip_addresses + #### General Cleanup + # Remove ending '.' + ip_address = ip_address.chomp + # Remove preceeding '.'# Don't ask.. reverse + chomp + reverse up to 16 times faster + ip_address = ip_address.reverse.chomp('.').reverse + # Remove ending or beginning whitespace + ip_address = ip_address.lstrip.rstrip + # Remove things that would make an IP a share but we want the IP :) + ip_address = ip_address.gsub(/^\\:?/, '') + # Downcase/lowercase for checking if possible ipv6 + ip_address = ip_address.downcase + + # Likely IPv4 #Check if at most 15 characters (a fully represented IPv4 address with periods\.) and minimum of 7 + ip_address_length = ip_address.length + if !ip_address.include?(':') && !( /[a-z]/ === ip_address ) && ip_address_length <= 15 && ip_address_length >= 7 && ip_address.ascii_only? + # Remove any preceeding zeroes in each octet + temp_ip = Array.new + ip_address.split('.').each do |octet| + octet = octet.to_i.to_s + temp_ip.push(octet) + end + ip_address = temp_ip.join('.') + clean_ip_addresses_v4.push(ip_address) + + # Likely IPv6 #Check if at most 39 characters (a fully represented IPv6 address with colons\:) and minimum of 2 + elsif ip_address_length <= 39 && ip_address_length >= 2 && ip_address.ascii_only? + if ipv6_regex === ip_address + clean_ip_addresses_v6.push(ip_address) + else + not_ip_addresses.push(ip_address) + end + else + not_ip_addresses.push(ip_address) + end + end + + # Set the new IP addresses + if !clean_ip_addresses_v4.empty? + event.set('src_nat_ip_addr', clean_ip_addresses_v4) + # Set the number of ip addresses so we can use array or non array later in pipeline + event.set('temp_number_of_ipv4_src_nat_addresses', clean_ip_addresses_v4.length) + else + event.remove('src_nat_ip_addr') + end + if !clean_ip_addresses_v6.empty? + event.set('ipv6_src_nat_addr', clean_ip_addresses_v6) + #TODO:eventually set ipv6 number of ip addresses + end + if !not_ip_addresses.empty? + event.set('not_ip_src_nat', not_ip_addresses) + end + " + tag_on_exception => "_rubyexception_8015" + } + } +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8112-dst-ip-filter.conf b/docker/helk-logstash/trial/pipeline/8112-dst-ip-filter.conf new file mode 100644 index 00000000..fa58787a --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8112-dst-ip-filter.conf @@ -0,0 +1,136 @@ +# HELK dst-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + # If dst_ip_addr field exists from previous config settings + if [dst_ip_addr] { + mutate { add_field => { "z_logstash_pipeline" => "8112" } } + + #TODO:could eventually make even faster by doing a if not begins/regex-starts with 0,1,2 then just immediatly set as ip is public else check the rest of stuff. + ruby { + code => " + # Get the ip address from the event + ip_addresses = event.get('dst_ip_addr') + + ip_addresses_public = Array.new + ip_addresses_type = Array.new + + for ip_address in ip_addresses + # Set IP type and public for now so easier checking later + ip_public = true + ip_type = 'public' + + # Private/RFC1918 + if ip_address.start_with?( '10.', '192.168.' ) + ip_public = false + ip_type = 'private' + + # (Local)link-local RFC3927 + elsif ip_address.start_with?( '169.254.' ) + ip_public = false + ip_type = 'local' + + # Loopback RFC1122-3.2.1.3 + elsif ip_address.start_with?( '127.' ) + ip_public = false + ip_type = 'loopback' + + # RFC 1122 + elsif ip_address.start_with?('0.') + ip_public = false + ip_type = 'this_rfc1122' + + # IPv6 to IP4 anycast RFC3068 + elsif ip_address.start_with?( '192.88.99.' ) + ip_public = false + ip_type = '6to4' + + # Reserved RFC5736, RFC1122-3.2.1.3, RFC2544, RFC5737 + elsif ip_address.start_with?( '0.', '192.0.0.', '192.0.1.', '192.0.2.', '192.18.', '192.19.', '198.51.100.', '203.0.113.' ) + ip_public = false + ip_type = 'reserved' + + # Private/RFC-1918 -- continued -- 172.16.0.0-17.31.255.255 + elsif ip_address.start_with?( '172.' ) + # Check if 2nd octet is in range(between) 16 to 31 + if ip_address.split('.')[1].to_i.between?(16,31) + ip_public = false + ip_type = 'private' + end + + # Private/RFC-1918 -- continued -- 100.64.0.1 - 100.127.255.254 + elsif ip_address.start_with?( '100.' ) + # Check if 2nd octet is in range(between) 64 to 127 + if ip_address.split('.')[1].to_i.between?(64,127) + ip_public = false + ip_type = 'private' + end + + # The remaining possible NON public/routable IPs begin with 2 and are either multicast or broadcast + elsif ip_address.start_with?( '2' ) + # Broadcast + if ip_address == '255.255.255.255' + ip_public = false + ip_type = 'broadcast' + + # Multicast + # Check if 1st octet is in range(between) 224 to 255 + elsif ip_address.split('.')[0].to_i.between?(224,255) + ip_public = false + ip_type = 'multicast' + end + end + # set parameters for array + ip_addresses_public.push(ip_public) + ip_addresses_type.push(ip_type) + end + + # Use to make array versus non array + # and then Set event parameters accordingly + if event.get('temp_number_of_ipv4_src_addresses') == 1 + event.set('dst_ip_addr', ip_addresses[0]) + event.set('dst_ip_public', ip_addresses_public[0]) + event.set('dst_ip_type', ip_addresses_type[0]) + else + event.set('dst_ip_public', ip_addresses_public) + event.set('dst_ip_type', ip_addresses_type) + # ip is already array so no need to set accordingly + end + " + tag_on_exception => "_rubyexception_8112" + remove_field => [ "temp_number_of_ipv4_dst_addresses" ] + } + + # Perform GeoIP enrichment if is public / internet routable + if [dst_ip_public] { + # Geo Location + geoip { + source => "dst_ip_addr" + target => "meta_dst_ip_geo" + default_database_type => "City" + # database => "/usr/share/logstash/GeoIP/GeoLite2-City.mmdb" + # #TONOTE:It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter of the same geoip_type share the same cache. The last declared cache size will win. The reason for this is that there would be no benefit to having multiple caches for different instances at different points in the pipeline, that would just increase the number of cache misses and waste memory. + cache_size => 90000 + remove_field => [ "[meta_dst_ip_geo][ip]", "[meta_dst_ip_geo][real_region_name]" ] + #tag_on_failure => [ "_geoip_ip_dst_failure", "_geoip_ip_dst_location_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + #QnVpbHQgYnkgTmF0ZSBHdWFnZW50aSBAbmV1NXJvbg== + # Geo ASName and ASNumber / BGP AS Info + geoip { + source => "dst_ip_addr" + target => "meta_dst_ip_geo" + default_database_type => "ASN" + # database => "/usr/share/logstash/GeoIP/GeoLite2-ASN.mmdb" + remove_field => [ "[meta_dst_ip_geo][ip]" ] + # tag_on_failure => [ "_geoip_ip_dst_failure", "_geoip_ip_dst_as_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + } + } + +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8113-src-ip-filter.conf b/docker/helk-logstash/trial/pipeline/8113-src-ip-filter.conf new file mode 100644 index 00000000..b583ec0c --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8113-src-ip-filter.conf @@ -0,0 +1,135 @@ +# HELK src-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + # If src_ip_addr field exists from previous config settings + if [src_ip_addr] { + mutate { add_field => { "z_logstash_pipeline" => "8113" } } + + #TODO:could eventually make even faster by doing a if not begins/regex-starts with 0,1,2 then just immediatly set as ip is public else check the rest of stuff. + ruby { + code => " + # Get the ip address from the event + ip_addresses = event.get('src_ip_addr') + + ip_addresses_public = Array.new + ip_addresses_type = Array.new + + for ip_address in ip_addresses + # Set IP type and public for now so easier checking later + ip_public = true + ip_type = 'public' + + # Private/RFC1918 + if ip_address.start_with?( '10.', '192.168.' ) + ip_public = false + ip_type = 'private' + + # (Local)link-local RFC3927 + elsif ip_address.start_with?( '169.254.' ) + ip_public = false + ip_type = 'local' + + # Loopback RFC1122-3.2.1.3 + elsif ip_address.start_with?( '127.' ) + ip_public = false + ip_type = 'loopback' + + # RFC 1122 + elsif ip_address.start_with?('0.') + ip_public = false + ip_type = 'this_rfc1122' + + # IPv6 to IP4 anycast RFC3068 + elsif ip_address.start_with?( '192.88.99.' ) + ip_public = false + ip_type = '6to4' + + # Reserved RFC5736, RFC1122-3.2.1.3, RFC2544, RFC5737 + elsif ip_address.start_with?( '0.', '192.0.0.', '192.0.1.', '192.0.2.', '192.18.', '192.19.', '198.51.100.', '203.0.113.' ) + ip_public = false + ip_type = 'reserved' + + # Private/RFC-1918 -- continued -- 172.16.0.0-17.31.255.255 + elsif ip_address.start_with?( '172.' ) + # Check if 2nd octet is in range(between) 16 to 31 + if ip_address.split('.')[1].to_i.between?(16,31) + ip_public = false + ip_type = 'private' + end + + # Private/RFC-1918 -- continued -- 100.64.0.1 - 100.127.255.254 + elsif ip_address.start_with?( '100.' ) + # Check if 2nd octet is in range(between) 64 to 127 + if ip_address.split('.')[1].to_i.between?(64,127) + ip_public = false + ip_type = 'private' + end + + # The remaining possible NON public/routable IPs begin with 2 and are either multicast or broadcast + elsif ip_address.start_with?( '2' ) + # Broadcast + if ip_address == '255.255.255.255' + ip_public = false + ip_type = 'broadcast' + + # Multicast + # Check if 1st octet is in range(between) 224 to 255 + elsif ip_address.split('.')[0].to_i.between?(224,255) + ip_public = false + ip_type = 'multicast' + end + end + # set parameters for array + ip_addresses_public.push(ip_public) + ip_addresses_type.push(ip_type) + end + + # Use to make array versus non array + # and then Set event parameters accordingly + if event.get('temp_number_of_ipv4_src_addresses') == 1 + event.set('src_ip_addr', ip_addresses[0]) + event.set('src_ip_public', ip_addresses_public[0]) + event.set('src_ip_type', ip_addresses_type[0]) + else + event.set('src_ip_public', ip_addresses_public) + event.set('src_ip_type', ip_addresses_type) + # ip is already array so no need to set accordingly + end + " + tag_on_exception => "_rubyexception_8113" + } + + # Perform GeoIP enrichment if is public / internet routable + if [src_ip_public] { + # Geo Location + geoip { + source => "src_ip_addr" + target => "meta_src_ip_geo" + default_database_type => "City" + # database => "/usr/share/logstash/GeoIP/GeoLite2-City.mmdb" + # #TONOTE:It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter of the same geoip_type share the same cache. The last declared cache size will win. The reason for this is that there would be no benefit to having multiple caches for different instances at different points in the pipeline, that would just increase the number of cache misses and waste memory. + cache_size => 90000 + remove_field => [ "[meta_src_ip_geo][ip]", "[meta_src_ip_geo][real_region_name]" ] + # tag_on_failure => [ "_geoip_ip_src_location_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + #QnVpbHQgYnkgTmF0ZSBHdWFnZW50aSBAbmV1NXJvbg== + # Geo ASName and ASNumber / BGP AS Info + geoip { + source => "src_ip_addr" + target => "meta_src_ip_geo" + default_database_type => "ASN" + # database => "/usr/share/logstash/GeoIP/GeoLite2-ASN.mmdb" + remove_field => [ "[meta_src_ip_geo][ip]" ] + # tag_on_failure => [ "_geoip_ip_src_as_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + } + } + +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8114-dst-nat-ip-filter.conf b/docker/helk-logstash/trial/pipeline/8114-dst-nat-ip-filter.conf new file mode 100644 index 00000000..d4bef355 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8114-dst-nat-ip-filter.conf @@ -0,0 +1,136 @@ +# HELK dst-nat-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + # If dst_nat_ip_addr field exists from previous config settings + if [dst_nat_ip_addr] { + mutate { add_field => { "z_logstash_pipeline" => "8114" } } + + #TODO:could eventually make even faster by doing a if not begins/regex-starts with 0,1,2 then just immediatly set as ip is public else check the rest of stuff. + ruby { + code => " + # Get the ip address from the event + ip_addresses = event.get('dst_nat_ip_addr') + + ip_addresses_public = Array.new + ip_addresses_type = Array.new + + for ip_address in ip_addresses + # Set IP type and public for now so easier checking later + ip_public = true + ip_type = 'public' + + # Private/RFC1918 + if ip_address.start_with?( '10.', '192.168.' ) + ip_public = false + ip_type = 'private' + + # (Local)link-local RFC3927 + elsif ip_address.start_with?( '169.254.' ) + ip_public = false + ip_type = 'local' + + # Loopback RFC1122-3.2.1.3 + elsif ip_address.start_with?( '127.' ) + ip_public = false + ip_type = 'loopback' + + # RFC 1122 + elsif ip_address.start_with?('0.') + ip_public = false + ip_type = 'this_rfc1122' + + # IPv6 to IP4 anycast RFC3068 + elsif ip_address.start_with?( '192.88.99.' ) + ip_public = false + ip_type = '6to4' + + # Reserved RFC5736, RFC1122-3.2.1.3, RFC2544, RFC5737 + elsif ip_address.start_with?( '0.', '192.0.0.', '192.0.1.', '192.0.2.', '192.18.', '192.19.', '198.51.100.', '203.0.113.' ) + ip_public = false + ip_type = 'reserved' + + # Private/RFC-1918 -- continued -- 172.16.0.0-17.31.255.255 + elsif ip_address.start_with?( '172.' ) + # Check if 2nd octet is in range(between) 16 to 31 + if ip_address.split('.')[1].to_i.between?(16,31) + ip_public = false + ip_type = 'private' + end + + # Private/RFC-1918 -- continued -- 100.64.0.1 - 100.127.255.254 + elsif ip_address.start_with?( '100.' ) + # Check if 2nd octet is in range(between) 64 to 127 + if ip_address.split('.')[1].to_i.between?(64,127) + ip_public = false + ip_type = 'private' + end + + # The remaining possible NON public/routable IPs begin with 2 and are either multicast or broadcast + elsif ip_address.start_with?( '2' ) + # Broadcast + if ip_address == '255.255.255.255' + ip_public = false + ip_type = 'broadcast' + + # Multicast + # Check if 1st octet is in range(between) 224 to 255 + elsif ip_address.split('.')[0].to_i.between?(224,255) + ip_public = false + ip_type = 'multicast' + end + end + # set parameters for array + ip_addresses_public.push(ip_public) + ip_addresses_type.push(ip_type) + end + + # Use to make array versus non array + # and then Set event parameters accordingly + if event.get('temp_number_of_ipv4_dst_nat_addresses') == 1 + event.set('dst_nat_ip_addr', ip_addresses[0]) + event.set('dst_nat_ip_public', ip_addresses_public[0]) + event.set('dst_nat_ip_type', ip_addresses_type[0]) + else + event.set('dst_nat_ip_public', ip_addresses_public) + event.set('dst_nat_ip_type', ip_addresses_type) + # ip is already array so no need to set accordingly + end + " + tag_on_exception => "_rubyexception_8114" + remove_field => [ "temp_number_of_ipv4_dst_nat_addresses" ] + } + + # Perform GeoIP enrichment if is public / internet routable + if [dst_nat_ip_public] { + # Geo Location + geoip { + source => "dst_nat_ip_addr" + target => "meta_dst_nat_ip_geo" + default_database_type => "City" + # database => "/usr/share/logstash/GeoIP/GeoLite2-City.mmdb" + # #TONOTE:It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter of the same geoip_type share the same cache. The last declared cache size will win. The reason for this is that there would be no benefit to having multiple caches for different instances at different points in the pipeline, that would just increase the number of cache misses and waste memory. + cache_size => 90000 + remove_field => [ "[meta_dst_nat_ip_geo][ip]", "[meta_dst_nat_ip_geo][real_region_name]" ] + # tag_on_failure => [ "_geo_ip_dst_nat_location_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + #QnVpbHQgYnkgTmF0ZSBHdWFnZW50aSBAbmV1NXJvbg== + # Geo ASName and ASNumber / BGP AS Info + geoip { + source => "dst_nat_ip_addr" + target => "meta_dst_nat_ip_geo" + default_database_type => "ASN" + # database => "/usr/share/logstash/GeoIP/GeoLite2-ASN.mmdb" + remove_field => [ "[meta_dst_nat_ip_geo][ip]" ] + # tag_on_failure => [ "_geo_ip_dst_nat_as_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + } + } + +} \ No newline at end of file diff --git a/docker/helk-logstash/trial/pipeline/8115-src-nat-ip-filter.conf b/docker/helk-logstash/trial/pipeline/8115-src-nat-ip-filter.conf new file mode 100644 index 00000000..fa246bf9 --- /dev/null +++ b/docker/helk-logstash/trial/pipeline/8115-src-nat-ip-filter.conf @@ -0,0 +1,136 @@ +# HELK src-nat-ip filter conf +# HELK build Stage: Alpha +# Author: Nate Guagenti (@neu5ron) +# License: GPL-3.0 + +filter { + # If src_nat_ip_addr field exists from previous config settings + if [src_nat_ip_addr] { + mutate { add_field => { "z_logstash_pipeline" => "8115" } } + + #TODO:could eventually make even faster by doing a if not begins/regex-starts with 0,1,2 then just immediatly set as ip is public else check the rest of stuff. + ruby { + code => " + # Get the ip address from the event + ip_addresses = event.get('src_nat_ip_addr') + + ip_addresses_public = Array.new + ip_addresses_type = Array.new + + for ip_address in ip_addresses + # Set IP type and public for now so easier checking later + ip_public = true + ip_type = 'public' + + # Private/RFC1918 + if ip_address.start_with?( '10.', '192.168.' ) + ip_public = false + ip_type = 'private' + + # (Local)link-local RFC3927 + elsif ip_address.start_with?( '169.254.' ) + ip_public = false + ip_type = 'local' + + # Loopback RFC1122-3.2.1.3 + elsif ip_address.start_with?( '127.' ) + ip_public = false + ip_type = 'loopback' + + # RFC 1122 + elsif ip_address.start_with?('0.') + ip_public = false + ip_type = 'this_rfc1122' + + # IPv6 to IP4 anycast RFC3068 + elsif ip_address.start_with?( '192.88.99.' ) + ip_public = false + ip_type = '6to4' + + # Reserved RFC5736, RFC1122-3.2.1.3, RFC2544, RFC5737 + elsif ip_address.start_with?( '0.', '192.0.0.', '192.0.1.', '192.0.2.', '192.18.', '192.19.', '198.51.100.', '203.0.113.' ) + ip_public = false + ip_type = 'reserved' + + # Private/RFC-1918 -- continued -- 172.16.0.0-17.31.255.255 + elsif ip_address.start_with?( '172.' ) + # Check if 2nd octet is in range(between) 16 to 31 + if ip_address.split('.')[1].to_i.between?(16,31) + ip_public = false + ip_type = 'private' + end + + # Private/RFC-1918 -- continued -- 100.64.0.1 - 100.127.255.254 + elsif ip_address.start_with?( '100.' ) + # Check if 2nd octet is in range(between) 64 to 127 + if ip_address.split('.')[1].to_i.between?(64,127) + ip_public = false + ip_type = 'private' + end + + # The remaining possible NON public/routable IPs begin with 2 and are either multicast or broadcast + elsif ip_address.start_with?( '2' ) + # Broadcast + if ip_address == '255.255.255.255' + ip_public = false + ip_type = 'broadcast' + + # Multicast + # Check if 1st octet is in range(between) 224 to 255 + elsif ip_address.split('.')[0].to_i.between?(224,255) + ip_public = false + ip_type = 'multicast' + end + end + # set parameters for array + ip_addresses_public.push(ip_public) + ip_addresses_type.push(ip_type) + end + + # Use to make array versus non array + # and then Set event parameters accordingly + if event.get('temp_number_of_ipv4_src_nat_addresses') == 1 + event.set('src_nat_ip_addr', ip_addresses[0]) + event.set('src_nat_ip_public', ip_addresses_public[0]) + event.set('src_nat_ip_type', ip_addresses_type[0]) + else + event.set('src_nat_ip_public', ip_addresses_public) + event.set('src_nat_ip_type', ip_addresses_type) + # ip is already array so no need to set accordingly + end + " + tag_on_exception => "_rubyexception_8115" + remove_field => [ "temp_number_of_ipv4_src_nat_addresses" ] + } + + # Perform GeoIP enrichment if is public / internet routable + if [src_nat_ip_public] { + # Geo Location + geoip { + source => "src_nat_ip_addr" + target => "meta_src_nat_ip_geo" + default_database_type => "City" + # database => "/usr/share/logstash/GeoIP/GeoLite2-City.mmdb" + # #TONOTE:It is important to note that this config value is global to the geoip_type. That is to say all instances of the geoip filter of the same geoip_type share the same cache. The last declared cache size will win. The reason for this is that there would be no benefit to having multiple caches for different instances at different points in the pipeline, that would just increase the number of cache misses and waste memory. + cache_size => 90000 + remove_field => [ "[meta_src_nat_ip_geo][ip]", "[meta_src_nat_ip_geo][real_region_name]" ] + # tag_on_failure => [ "_geo_ip_src_nat_location_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + #QnVpbHQgYnkgTmF0ZSBHdWFnZW50aSBAbmV1NXJvbg== + # Geo ASName and ASNumber / BGP AS Info + geoip { + source => "src_nat_ip_addr" + target => "meta_src_nat_ip_geo" + default_database_type => "ASN" + # database => "/usr/share/logstash/GeoIP/GeoLite2-ASN.mmdb" + remove_field => [ "[meta_src_nat_ip_geo][ip]" ] + # tag_on_failure => [ "_geo_ip_src_nat_as_lookup_failure", "_geoip_lookup_failure" ] + # Do not tag, so we can use tags (if they only just even exist, versus each tag within) as a method to determine parsing failure(s) (much) easier. + tag_on_failure => [ ] + } + } + } + +} \ No newline at end of file diff --git a/helk-logstash/trial/pipeline/50-winevent-sysmon-output.conf b/docker/helk-logstash/trial/pipeline/9950-winevent-sysmon-output.conf similarity index 76% rename from helk-logstash/trial/pipeline/50-winevent-sysmon-output.conf rename to docker/helk-logstash/trial/pipeline/9950-winevent-sysmon-output.conf index 7d25bd86..86b824ab 100644 --- a/helk-logstash/trial/pipeline/50-winevent-sysmon-output.conf +++ b/docker/helk-logstash/trial/pipeline/9950-winevent-sysmon-output.conf @@ -8,10 +8,8 @@ output { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-sysmon-%{+YYYY.MM.dd}" - template => "/usr/share/logstash/output_templates/winevent-sysmon-template.json" - template_name => "logs-endpoint-winevent-sysmon" - template_overwrite => true document_id => "%{[@metadata][log_hash]}" + #document_type => "_doc" user => 'elastic' password => 'elasticpassword' } diff --git a/helk-logstash/trial/pipeline/51-winevent-security-output.conf b/docker/helk-logstash/trial/pipeline/9951-winevent-security-output.conf similarity index 75% rename from helk-logstash/trial/pipeline/51-winevent-security-output.conf rename to docker/helk-logstash/trial/pipeline/9951-winevent-security-output.conf index f35d4ead..6cb5544b 100644 --- a/helk-logstash/trial/pipeline/51-winevent-security-output.conf +++ b/docker/helk-logstash/trial/pipeline/9951-winevent-security-output.conf @@ -8,10 +8,8 @@ output { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-security-%{+YYYY.MM.dd}" - template => "/usr/share/logstash/output_templates/winevent-security-template.json" - template_name => "logs-endpoint-winevent-security" - template_overwrite => true document_id => "%{[@metadata][log_hash]}" + #document_type => "_doc" user => 'elastic' password => 'elasticpassword' } diff --git a/helk-logstash/trial/pipeline/52-winevent-system-output.conf b/docker/helk-logstash/trial/pipeline/9952-winevent-system-output.conf similarity index 70% rename from helk-logstash/trial/pipeline/52-winevent-system-output.conf rename to docker/helk-logstash/trial/pipeline/9952-winevent-system-output.conf index 8c63e09b..2f7fde4a 100644 --- a/helk-logstash/trial/pipeline/52-winevent-system-output.conf +++ b/docker/helk-logstash/trial/pipeline/9952-winevent-system-output.conf @@ -8,10 +8,8 @@ output { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-system-%{+YYYY.MM.dd}" - template => "/usr/share/logstash/output_templates/winevent-system-template.json" - template_name => "logs-endpoint-winevent-system" - template_overwrite => true document_id => "%{[@metadata][log_hash]}" + #document_type => "_doc" user => 'elastic' password => 'elasticpassword' } diff --git a/helk-logstash/trial/pipeline/53-winevent-application-output.conf b/docker/helk-logstash/trial/pipeline/9953-winevent-application-output.conf similarity index 69% rename from helk-logstash/trial/pipeline/53-winevent-application-output.conf rename to docker/helk-logstash/trial/pipeline/9953-winevent-application-output.conf index 72728a9b..ca16a573 100644 --- a/helk-logstash/trial/pipeline/53-winevent-application-output.conf +++ b/docker/helk-logstash/trial/pipeline/9953-winevent-application-output.conf @@ -8,10 +8,8 @@ output { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-application-%{+YYYY.MM.dd}" - template => "/usr/share/logstash/output_templates/winevent-application-template.json" - template_name => "logs-endpoint-winevent-application" - template_overwrite => true document_id => "%{[@metadata][log_hash]}" + #document_type => "_doc" user => 'elastic' password => 'elasticpassword' } diff --git a/helk-logstash/trial/pipeline/54-winevent-powershell-output.conf b/docker/helk-logstash/trial/pipeline/9954-winevent-powershell-output.conf similarity index 93% rename from helk-logstash/trial/pipeline/54-winevent-powershell-output.conf rename to docker/helk-logstash/trial/pipeline/9954-winevent-powershell-output.conf index 5945aa6e..a786d503 100644 --- a/helk-logstash/trial/pipeline/54-winevent-powershell-output.conf +++ b/docker/helk-logstash/trial/pipeline/9954-winevent-powershell-output.conf @@ -7,9 +7,9 @@ output { if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell"{ elasticsearch { hosts => ["helk-elasticsearch:9200"] - manage_template => false index => "logs-endpoint-winevent-powershell-%{+YYYY.MM.dd}" document_id => "%{[@metadata][log_hash]}" + #document_type => "_doc" user => 'elastic' password => 'elasticpassword' } diff --git a/helk-logstash/trial/pipeline/55-winevent-wmiactivity-output.conf b/docker/helk-logstash/trial/pipeline/9955-winevent-wmiactivity-output.conf similarity index 70% rename from helk-logstash/trial/pipeline/55-winevent-wmiactivity-output.conf rename to docker/helk-logstash/trial/pipeline/9955-winevent-wmiactivity-output.conf index ec985604..ad744c4d 100644 --- a/helk-logstash/trial/pipeline/55-winevent-wmiactivity-output.conf +++ b/docker/helk-logstash/trial/pipeline/9955-winevent-wmiactivity-output.conf @@ -8,10 +8,8 @@ output { elasticsearch { hosts => ["helk-elasticsearch:9200"] index => "logs-endpoint-winevent-wmiactivity-%{+YYYY.MM.dd}" - template => "/usr/share/logstash/output_templates/winevent-wmiactivity-template.json" - template_name => "logs-endpoint-winevent-wmiactivity" - template_overwrite => true document_id => "%{[@metadata][log_hash]}" + #document_type => "_doc" user => 'elastic' password => 'elasticpassword' } diff --git a/helk-logstash/trial/pipeline/56-attack-output.conf b/docker/helk-logstash/trial/pipeline/9956-attack-output.conf similarity index 55% rename from helk-logstash/trial/pipeline/56-attack-output.conf rename to docker/helk-logstash/trial/pipeline/9956-attack-output.conf index 619aabeb..a89c6173 100644 --- a/helk-logstash/trial/pipeline/56-attack-output.conf +++ b/docker/helk-logstash/trial/pipeline/9956-attack-output.conf @@ -7,10 +7,11 @@ output { if "attack" in [tags]{ elasticsearch { - hosts => ["helk-elasticsearch:9200"] - index => "mitre-attack-%{+YYYY.MM.dd}" - user => 'elastic' - password => 'elasticpassword' + hosts => ["helk-elasticsearch:9200"] + index => "mitre-attack-%{+YYYY.MM.dd}" + #document_type => "_doc" + user => 'elastic' + password => 'elasticpassword' } } -} \ No newline at end of file +} diff --git a/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh b/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh new file mode 100755 index 00000000..ee7c81f8 --- /dev/null +++ b/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# HELK script: logstash-setup.sh +# HELK script description: Pushes output templates to ES and starts Logstash +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# ********* Setting LS_JAVA_OPTS *************** +if [[ ! -z "$LS_JAVA_OPTS" ]]; then + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting LS_JAVA_OPTS to $LS_JAVA_OPTS" +else + # ****** Setup heap size ***** + LS_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo) + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting LS_HEAP_SIZE to ${LS_MEMORY}.." + export LS_JAVA_OPTS="-Xms${LS_MEMORY}g -Xmx${LS_MEMORY}g" +fi + +ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200 +# *********** Looking for ES *************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.." +until curl -s $ELASTICSEARCH_ACCESS -o /dev/null; do + sleep 1 +done + +DIR=/usr/share/logstash/output_templates +for file in ${DIR}/*.json + do + template_name=$(echo $file | sed -r ' s/^.*\/[0-9]+\-//'); + curl -H 'Content-Type: application/json' -XPUT "$ELASTICSEARCH_ACCESS/_template/$template_name" -d@${file}; +done + +exec "$@" + + diff --git a/helk-nginx/Dockerfile b/docker/helk-nginx/Dockerfile similarity index 93% rename from helk-nginx/Dockerfile rename to docker/helk-nginx/Dockerfile index a340a389..d7f2fd71 100644 --- a/helk-nginx/Dockerfile +++ b/docker/helk-nginx/Dockerfile @@ -22,7 +22,7 @@ RUN apt-get -qy clean \ autoremove # *********** Adding HELK scripts and files to Container *************** -ADD scripts/nginx-entrypoint.sh /opt/helk/scripts/ +COPY scripts/nginx-entrypoint.sh /opt/helk/scripts/ RUN chmod +x /opt/helk/scripts/nginx-entrypoint.sh # *********** Installing Nginx *************** @@ -30,6 +30,6 @@ RUN apt-get install -qqy nginx RUN apt-get update -qq # *********** RUN HELK *************** -EXPOSE 80 +EXPOSE 80 443 WORKDIR "/opt/helk/scripts/" ENTRYPOINT ["./nginx-entrypoint.sh"] \ No newline at end of file diff --git a/docker/helk-nginx/default b/docker/helk-nginx/default new file mode 100644 index 00000000..3e9e7f6b --- /dev/null +++ b/docker/helk-nginx/default @@ -0,0 +1,114 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} +server { + listen 80; + server_name 127.0.0.1; + return 301 https://$host$request_uri; +} + +server { + ## Log location ## + #access_log /var/log/nginx/kibana.access.log; + #error_log /var/log/nginx/kibana.error.log; + ## End ## + + ## (Web) Server Configuration ## + listen 443 ssl; + ssl_certificate_key /etc/ssl/private/HELK_Nginx.key; + ssl_certificate /etc/ssl/certs/HELK_Nginx.crt; + ssl_session_cache shared:SSL:10m; + ## End ## + + ## Hide Version ## + server_tokens off; + ## End ## + + ## Some Hardening/Security ## + # Web Server Attack (ie: XSS, Clickjacking) + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + add_header X-Frame-Options "SAMEORIGIN"; + # BufferOverflow Hardening + client_body_buffer_size 100K; + client_header_buffer_size 1k; + client_max_body_size 150k; + # TLS/SSL + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4:@STRENGTH"; + ssl_prefer_server_ciphers on; + add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;"; + # HTTP Methods + # DELETE is required for some index management + if ($request_method !~ ^(GET|PUT|POST|DELETE)$ ) { + return 444; + } + ## End ## + + ## Timeout definitions ## + proxy_connect_timeout 900; + proxy_send_timeout 600; + proxy_read_timeout 600; + client_body_timeout 10; + client_header_timeout 10; + keepalive_timeout 10 10; + send_timeout 10; + ## End ## + + ## Performance Tuning ## + gzip on; + gzip_comp_level 1; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + ## End ## + + ## Kibana Settings / Kibana Proxy ## + # Kibana generates really long URI's so need to set this + large_client_header_buffers 4 16k; + + # Cache static resources that are regularly requested + location ~ \.(jpg|png|ico|svg|woff2)$ { + proxy_pass http://helk-kibana:5601; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + #proxy_cache kibana_cache; + add_header Cache-Control "public"; + expires 12h; + } + # Proxy forward to elasticsearch + location / { + ## Local Authentication ## + auth_basic "Restricted Access"; + auth_basic_user_file /etc/nginx/htpasswd.users; + ## End ## + + proxy_pass http://helk-kibana:5601; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } + ## End ## + + ## Jupyter Proxy ## + location /jupyter/ { + if ($scheme = 'http') { + #Insecure, lets go to https + rewrite ^/(.*)$ https://$host/$1 redirect; + } + proxy_pass http://helk-jupyter:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # websocket headers + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + ## End ## +} diff --git a/helk-nginx/htpasswd.users b/docker/helk-nginx/htpasswd.users similarity index 100% rename from helk-nginx/htpasswd.users rename to docker/helk-nginx/htpasswd.users diff --git a/helk-nginx/scripts/nginx-entrypoint.sh b/docker/helk-nginx/scripts/nginx-entrypoint.sh similarity index 69% rename from helk-nginx/scripts/nginx-entrypoint.sh rename to docker/helk-nginx/scripts/nginx-entrypoint.sh index 15550546..d0f49d11 100755 --- a/helk-nginx/scripts/nginx-entrypoint.sh +++ b/docker/helk-nginx/scripts/nginx-entrypoint.sh @@ -18,6 +18,16 @@ until curl -s helk-elasticsearch:9200 -o /dev/null; do sleep 1 done +# ************* Creating JupyterHub Certificate *********** +openssl req \ + -x509 \ + -nodes \ + -days 365 \ + -newkey rsa:2048 \ + -keyout /etc/ssl/private/HELK_Nginx.key \ + -out /etc/ssl/certs/HELK_Nginx.crt \ + -subj "/C=US/ST=VA/L=VA/O=HELK/OU=HELK Nginx/CN=HELK" + echo "[HELK-DOCKER-INSTALLATION-INFO] Starting remaining services.." service nginx restart diff --git a/helk-nginx/default b/docker/helk-nginx/trial/default similarity index 94% rename from helk-nginx/default rename to docker/helk-nginx/trial/default index 0af0e68d..82447ddf 100644 --- a/helk-nginx/default +++ b/docker/helk-nginx/trial/default @@ -25,11 +25,6 @@ server { server_tokens off; ## End ## - ## Local Authentication ## - auth_basic "Restricted Access"; - auth_basic_user_file /etc/nginx/htpasswd.users; - ## End ## - ## Some Hardening/Security ## # Web Server Attack (ie: XSS, Clickjacking) add_header X-Frame-Options DENY; @@ -81,7 +76,7 @@ server { proxy_set_header Connection 'upgrade'; proxy_set_header Host $host; proxy_cache_bypass $http_upgrade; - proxy_cache kibana_cache; + #proxy_cache kibana_cache; add_header Cache-Control "public"; expires 12h; } @@ -92,7 +87,7 @@ server { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; + proxy_cache_bypass $http_upgrade; } ## End ## @@ -111,4 +106,4 @@ server { proxy_set_header Connection $connection_upgrade; } ## End ## -} +} \ No newline at end of file diff --git a/helk-sigma/Dockerfile b/docker/helk-sigma/Dockerfile similarity index 100% rename from helk-sigma/Dockerfile rename to docker/helk-sigma/Dockerfile diff --git a/helk-sigma/scripts/sigma-entrypoint.sh b/docker/helk-sigma/scripts/sigma-entrypoint.sh similarity index 100% rename from helk-sigma/scripts/sigma-entrypoint.sh rename to docker/helk-sigma/scripts/sigma-entrypoint.sh diff --git a/helk-sigma/scripts/update-sigma.sh b/docker/helk-sigma/scripts/update-sigma.sh similarity index 100% rename from helk-sigma/scripts/update-sigma.sh rename to docker/helk-sigma/scripts/update-sigma.sh diff --git a/helk-spark-base/Dockerfile b/docker/helk-spark-base/Dockerfile similarity index 100% rename from helk-spark-base/Dockerfile rename to docker/helk-spark-base/Dockerfile diff --git a/helk-spark-master/Dockerfile b/docker/helk-spark-master/Dockerfile similarity index 69% rename from helk-spark-master/Dockerfile rename to docker/helk-spark-master/Dockerfile index 65e794b1..561c6b54 100644 --- a/helk-spark-master/Dockerfile +++ b/docker/helk-spark-master/Dockerfile @@ -11,12 +11,9 @@ ENV DEBIAN_FRONTEND noninteractive USER sparkuser -ADD scripts/spark-master-entrypoint.sh ${SPARK_HOME}/ - -ENV SPARK_MASTER_HOST helk-spark-master -ENV SPARK_MASTER_PORT 7077 -ENV SPARK_MASTER_WEBUI_PORT 8080 +COPY scripts/spark-master-entrypoint.sh ${SPARK_HOME}/sbin/ +COPY spark-defaults.conf ${SPARK_HOME}/conf/ EXPOSE 8080 7077 -WORKDIR $SPARK_HOME +WORKDIR $SPARK_HOME/sbin ENTRYPOINT ["./spark-master-entrypoint.sh"] \ No newline at end of file diff --git a/docker/helk-spark-master/scripts/spark-master-entrypoint.sh b/docker/helk-spark-master/scripts/spark-master-entrypoint.sh new file mode 100755 index 00000000..72d40436 --- /dev/null +++ b/docker/helk-spark-master/scripts/spark-master-entrypoint.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# HELK script: spark-master-entrypoint.sh +# HELK script description: Starts Spark Master Service +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 +# Reference: +# https://github.com/apache/spark/blob/master/sbin/start-master.sh (Modified to not execute daemon script) + + +# Starts the master on the machine this script is executed on. +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +# NOTE: This exact class name is matched downstream by SparkSubmit. +# Any changes need to be reflected there. +CLASS="org.apache.spark.deploy.master.Master" + +if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then + echo "Usage: ./sbin/start-master.sh [options]" + pattern="Usage:" + pattern+="\|Using Spark's default log4j profile:" + pattern+="\|Registered signal handlers for" + + "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2 + exit 1 +fi + +ORIGINAL_ARGS="$@" + +. "${SPARK_HOME}/sbin/spark-config.sh" + +. "${SPARK_HOME}/bin/load-spark-env.sh" + +if [ "$SPARK_MASTER_PORT" = "" ]; then + SPARK_MASTER_PORT=7077 +fi + +if [ "$SPARK_MASTER_HOST" = "" ]; then + case `uname` in + (SunOS) + SPARK_MASTER_HOST="`/usr/sbin/check-hostname | awk '{print $NF}'`" + ;; + (*) + SPARK_MASTER_HOST="`hostname -f`" + ;; + esac +fi + +if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then + SPARK_MASTER_WEBUI_PORT=8080 +fi + +echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Spark Master Service.." +$SPARK_HOME/bin/spark-class $CLASS \ + --host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \ + $ORIGINAL_ARGS \ No newline at end of file diff --git a/docker/helk-spark-master/spark-defaults.conf b/docker/helk-spark-master/spark-defaults.conf new file mode 100644 index 00000000..8808075c --- /dev/null +++ b/docker/helk-spark-master/spark-defaults.conf @@ -0,0 +1,14 @@ +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# HELK References: +# https://spark.apache.org/docs/latest/configuration.html +# https://graphframes.github.io/quick-start.html +# https://spark-packages.org/package/graphframes/graphframes +# https://spark.apache.org/docs/latest/sql-programming-guide.html#pyspark-usage-guide-for-pandas-with-apache-arrow + +# ************ Spark UI **************** +# Enable running Spark Master as reverse proxy for worker and application UIs. +# In this mode, Spark master will reverse proxy the worker and application UIs to enable access without requiring direct access to their hosts. +spark.ui.reverseProxy true \ No newline at end of file diff --git a/helk-spark-worker/Dockerfile b/docker/helk-spark-worker/Dockerfile similarity index 76% rename from helk-spark-worker/Dockerfile rename to docker/helk-spark-worker/Dockerfile index 1060c8b1..639f2a42 100644 --- a/helk-spark-worker/Dockerfile +++ b/docker/helk-spark-worker/Dockerfile @@ -11,10 +11,10 @@ ENV DEBIAN_FRONTEND noninteractive USER sparkuser -ADD scripts/spark-worker-entrypoint.sh ${SPARK_HOME}/ - -ENV SPARK_MASTER "spark://helk-spark-master:7077" +COPY scripts/spark-worker-entrypoint.sh ${SPARK_HOME}/sbin +COPY spark-defaults.conf ${SPARK_HOME}/conf/ EXPOSE $SPARK_WORKER_WEBUI_PORT -WORKDIR $SPARK_HOME + +WORKDIR $SPARK_HOME/sbin ENTRYPOINT ["./spark-worker-entrypoint.sh"] \ No newline at end of file diff --git a/docker/helk-spark-worker/scripts/spark-worker-entrypoint.sh b/docker/helk-spark-worker/scripts/spark-worker-entrypoint.sh new file mode 100755 index 00000000..7bf4ef1f --- /dev/null +++ b/docker/helk-spark-worker/scripts/spark-worker-entrypoint.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# HELK script: spark-worker-entrypoint.sh +# HELK script description: Starts Spark Worker Service +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 +# Reference: +# https://github.com/apache/spark/blob/master/sbin/start-slave.sh (Modified to not execute daemon script) + +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +# NOTE: This exact class name is matched downstream by SparkSubmit. +# Any changes need to be reflected there. +CLASS="org.apache.spark.deploy.worker.Worker" + +#if [[ $# -lt 1 ]] || [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then +if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then + echo "Usage: ./sbin/start-slave.sh [options] " + pattern="Usage:" + pattern+="\|Using Spark's default log4j profile:" + pattern+="\|Registered signal handlers for" + + "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2 + exit 1 +fi + +. "${SPARK_HOME}/sbin/spark-config.sh" + +. "${SPARK_HOME}/bin/load-spark-env.sh" + +# First argument should be the master; we need to store it aside because we may +# need to insert arguments between it and the other arguments +#MASTER=$1 +#shift + +# Determine desired worker port +if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then + SPARK_WORKER_WEBUI_PORT=8081 +fi + +if [ "$SPARK_WORKER_PORT" = "" ]; then + PORT_FLAG= + PORT_NUM= +else + PORT_FLAG="--port" + PORT_NUM="$SPARK_WORKER_PORT" +fi + +$SPARK_HOME/bin/spark-class $CLASS \ + --webui-port $SPARK_WORKER_WEBUI_PORT $PORT_FLAG $PORT_NUM $SPARK_MASTER + +# Start up the appropriate number of workers on this machine. +# quick local function to start a worker +#function start_instance { +# WORKER_NUM=$1 +# shift + +# if [ "$SPARK_WORKER_PORT" = "" ]; then +# PORT_FLAG= +# PORT_NUM= +# else +# PORT_FLAG="--port" +# PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 )) +# fi +# WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 )) + +# $SPARK_HOME/bin/spark-class $CLASS $WORKER_NUM \ +# --webui-port "$WEBUI_PORT" $PORT_FLAG $PORT_NUM $MASTER "$@" +#} + +#if [ "$SPARK_WORKER_INSTANCES" = "" ]; then +# start_instance 1 "$@" +#else +# for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do +# start_instance $(( 1 + $i )) "$@" +# done +#fi \ No newline at end of file diff --git a/docker/helk-spark-worker/spark-defaults.conf b/docker/helk-spark-worker/spark-defaults.conf new file mode 100644 index 00000000..8808075c --- /dev/null +++ b/docker/helk-spark-worker/spark-defaults.conf @@ -0,0 +1,14 @@ +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# HELK References: +# https://spark.apache.org/docs/latest/configuration.html +# https://graphframes.github.io/quick-start.html +# https://spark-packages.org/package/graphframes/graphframes +# https://spark.apache.org/docs/latest/sql-programming-guide.html#pyspark-usage-guide-for-pandas-with-apache-arrow + +# ************ Spark UI **************** +# Enable running Spark Master as reverse proxy for worker and application UIs. +# In this mode, Spark master will reverse proxy the worker and application UIs to enable access without requiring direct access to their hosts. +spark.ui.reverseProxy true \ No newline at end of file diff --git a/docker/helk-zeppelin/Dockerfile b/docker/helk-zeppelin/Dockerfile new file mode 100644 index 00000000..7e153ef3 --- /dev/null +++ b/docker/helk-zeppelin/Dockerfile @@ -0,0 +1,70 @@ +# HELK script: HELK Zeppelin Dockerfile +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 +# Reference: +# https://github.com/apache/zeppelin/blob/master/scripts/docker/zeppelin/bin/Dockerfile +# https://hub.docker.com/r/apache/zeppelin/~/dockerfile/ + +FROM cyb3rward0g/helk-spark-base:2.3.1 +LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" +LABEL description="Dockerfile base for the HELK Zeppelin." + +ENV DEBIAN_FRONTEND noninteractive + +USER root +# *********** Installing Prerequisites *************** +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN apt-get -y update && \ + apt-get install -y git openjdk-8-jdk \ + libfontconfig git build-essential chrpath \ + libssl-dev libxft-dev libfreetype6 libfreetype6-dev \ + libfontconfig1 libfontconfig1-dev python3-pip && \ + rm -rf /var/lib/apt/lists/* + +# *********** Upgrading PIP *************** +RUN pip3 install --upgrade pip + +RUN apt-get -y update && \ + apt-get install -y gfortran && \ + # numerical/algebra packages + apt-get install -y libblas-dev libatlas-dev liblapack-dev && \ + # font, image for matplotlib + apt-get install -y libpng-dev libfreetype6-dev libxft-dev && \ + # for tkinter + apt-get install -y python-tk libxml2-dev libxslt-dev zlib1g-dev && \ + pip3 install numpy && \ + pip3 install matplotlib + +# *********** Creating the right directories *************** +RUN bash -c 'mkdir -pv /opt/helk/{es-hadoop,zeppelin}' + +# ************** Install Zeppelin ********************* +ENV Z_VERSION=0.8.0 +ENV SPARK_CYPHER_VERSION=0.1.6 +ENV Z_HOME="/opt/helk/zeppelin" +ENV ZEPPELIN_PORT=9090 + +RUN wget -qO- http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz | sudo tar xvz -C ${Z_HOME} --strip-components=1 + +# *********** Install CAPS *************** +RUN wget https://github.com/opencypher/cypher-for-apache-spark/releases/download/${SPARK_CYPHER_VERSION}/spark-cypher-${SPARK_CYPHER_VERSION}-cluster.jar -P ${Z_HOME} + +# *********** Download ES-Hadoop *************** +ENV ESHADOOP_VERSION=6.3.1 +RUN wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsearch-hadoop-${ESHADOOP_VERSION}.zip -P /opt/helk/es-hadoop/ \ + && unzip -j /opt/helk/es-hadoop/*.zip -d /opt/helk/es-hadoop/ \ + && rm /opt/helk/es-hadoop/*.zip + +# *********** Add Spark User rights to Zeppelin Folder *************** +RUN chown -R ${SPARK_USER}:${SPARK_USER} ${Z_HOME} /opt/helk/es-hadoop + +USER $SPARK_UID + +# *********** Add Custom SPark Conf ****** +ADD spark-defaults.conf ${SPARK_HOME}/conf/ + +EXPOSE 9090 + +WORKDIR ${Z_HOME} +CMD ["bin/zeppelin.sh"] \ No newline at end of file diff --git a/docker/helk-zeppelin/conf/interpreter.json b/docker/helk-zeppelin/conf/interpreter.json new file mode 100644 index 00000000..9186b7ad --- /dev/null +++ b/docker/helk-zeppelin/conf/interpreter.json @@ -0,0 +1,1658 @@ +{ + "interpreterSettings": { + "ignite": { + "id": "ignite", + "name": "ignite", + "group": "ignite", + "properties": { + "ignite.config.url": { + "name": "ignite.config.url", + "value": "", + "type": "url" + }, + "ignite.peerClassLoadingEnabled": { + "name": "ignite.peerClassLoadingEnabled", + "value": true, + "type": "checkbox" + }, + "ignite.clientMode": { + "name": "ignite.clientMode", + "value": true, + "type": "checkbox" + }, + "ignite.jdbc.url": { + "name": "ignite.jdbc.url", + "value": "jdbc:ignite:cfg://default-ignite-jdbc.xml", + "type": "string" + }, + "ignite.addresses": { + "name": "ignite.addresses", + "value": "127.0.0.1:47500..47509", + "type": "textarea" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "ignite", + "class": "org.apache.zeppelin.ignite.IgniteInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": false + } + }, + { + "name": "ignitesql", + "class": "org.apache.zeppelin.ignite.IgniteSqlInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "python": { + "id": "python", + "name": "python", + "group": "python", + "properties": { + "zeppelin.python": { + "name": "zeppelin.python", + "value": "python", + "type": "string" + }, + "zeppelin.ipython.launch.timeout": { + "name": "zeppelin.ipython.launch.timeout", + "value": "30000", + "type": "number" + }, + "zeppelin.python.useIPython": { + "name": "zeppelin.python.useIPython", + "value": true, + "type": "checkbox" + }, + "zeppelin.ipython.grpc.message_size": { + "name": "zeppelin.ipython.grpc.message_size", + "value": "33554432", + "type": "number" + }, + "zeppelin.python.maxResult": { + "name": "zeppelin.python.maxResult", + "value": "1000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "python", + "class": "org.apache.zeppelin.python.PythonInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionSupport": true + } + }, + { + "name": "ipython", + "class": "org.apache.zeppelin.python.IPythonInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.python.PythonInterpreterPandasSql", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": false + } + }, + { + "name": "conda", + "class": "org.apache.zeppelin.python.PythonCondaInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false, + "completionSupport": false + } + }, + { + "name": "docker", + "class": "org.apache.zeppelin.python.PythonDockerInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "sap": { + "id": "sap", + "name": "sap", + "group": "sap", + "properties": { + "universe.password": { + "name": "universe.password", + "value": "", + "type": "password" + }, + "universe.api.url": { + "name": "universe.api.url", + "value": "http://localhost:6405/biprws", + "type": "url" + }, + "universe.authType": { + "name": "universe.authType", + "value": "secEnterprise", + "type": "string" + }, + "universe.user": { + "name": "universe.user", + "value": "", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "universe", + "class": "org.apache.zeppelin.sap.UniverseInterpreter", + "defaultInterpreter": true, + "editor": { + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "jdbc": { + "id": "jdbc", + "name": "jdbc", + "group": "jdbc", + "properties": { + "default.url": { + "name": "default.url", + "value": "jdbc:postgresql://localhost:5432/", + "type": "string" + }, + "default.driver": { + "name": "default.driver", + "value": "org.postgresql.Driver", + "type": "string" + }, + "zeppelin.jdbc.principal": { + "name": "zeppelin.jdbc.principal", + "value": "", + "type": "string" + }, + "default.completer.ttlInSeconds": { + "name": "default.completer.ttlInSeconds", + "value": "120", + "type": "number" + }, + "default.password": { + "name": "default.password", + "value": "", + "type": "password" + }, + "default.completer.schemaFilters": { + "name": "default.completer.schemaFilters", + "value": "", + "type": "textarea" + }, + "default.splitQueries": { + "name": "default.splitQueries", + "value": false, + "type": "checkbox" + }, + "default.user": { + "name": "default.user", + "value": "gpadmin", + "type": "string" + }, + "zeppelin.jdbc.concurrent.max_connection": { + "name": "zeppelin.jdbc.concurrent.max_connection", + "value": "10", + "type": "number" + }, + "common.max_count": { + "name": "common.max_count", + "value": "1000", + "type": "number" + }, + "default.precode": { + "name": "default.precode", + "value": "", + "type": "textarea" + }, + "zeppelin.jdbc.auth.type": { + "name": "zeppelin.jdbc.auth.type", + "value": "", + "type": "string" + }, + "default.statementPrecode": { + "name": "default.statementPrecode", + "value": "", + "type": "string" + }, + "zeppelin.jdbc.concurrent.use": { + "name": "zeppelin.jdbc.concurrent.use", + "value": true, + "type": "checkbox" + }, + "zeppelin.jdbc.keytab.location": { + "name": "zeppelin.jdbc.keytab.location", + "value": "", + "type": "string" + }, + "zeppelin.jdbc.interpolation": { + "name": "zeppelin.jdbc.interpolation", + "value": false, + "type": "checkbox" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sql", + "class": "org.apache.zeppelin.jdbc.JDBCInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionSupport": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "lens": { + "id": "lens", + "name": "lens", + "group": "lens", + "properties": { + "lens.server.base.url": { + "name": "lens.server.base.url", + "value": "http://\u003chostname\u003e:\u003cport\u003e/lensapi", + "type": "url" + }, + "zeppelin.lens.maxThreads": { + "name": "zeppelin.lens.maxThreads", + "value": "10", + "type": "number" + }, + "zeppelin.lens.maxResults": { + "name": "zeppelin.lens.maxResults", + "value": "1000", + "type": "number" + }, + "lens.client.dbname": { + "name": "lens.client.dbname", + "value": "default", + "type": "string" + }, + "lens.query.enable.persistent.resultset": { + "name": "lens.query.enable.persistent.resultset", + "value": false, + "type": "checkbox" + }, + "zeppelin.lens.run.concurrent": { + "name": "zeppelin.lens.run.concurrent", + "value": true, + "type": "checkbox" + }, + "lens.session.cluster.user": { + "name": "lens.session.cluster.user", + "value": "default", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "lens", + "class": "org.apache.zeppelin.lens.LensInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "pig": { + "id": "pig", + "name": "pig", + "group": "pig", + "properties": { + "zeppelin.pig.execType": { + "name": "zeppelin.pig.execType", + "value": "mapreduce", + "type": "string" + }, + "SPARK_JAR": { + "name": "SPARK_JAR", + "value": "", + "type": "textarea" + }, + "zeppelin.pig.includeJobStats": { + "name": "zeppelin.pig.includeJobStats", + "value": false, + "type": "checkbox" + }, + "zeppelin.pig.maxResult": { + "name": "zeppelin.pig.maxResult", + "value": "1000", + "type": "number" + }, + "SPARK_MASTER": { + "name": "SPARK_MASTER", + "value": "local", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "script", + "class": "org.apache.zeppelin.pig.PigInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "pig", + "editOnDblClick": false + } + }, + { + "name": "query", + "class": "org.apache.zeppelin.pig.PigQueryInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "pig", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "flink": { + "id": "flink", + "name": "flink", + "group": "flink", + "properties": { + "port": { + "name": "port", + "value": "6123", + "type": "number" + }, + "host": { + "name": "host", + "value": "local", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "flink", + "class": "org.apache.zeppelin.flink.FlinkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "scala", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "angular": { + "id": "angular", + "name": "angular", + "group": "angular", + "properties": {}, + "status": "READY", + "interpreterGroup": [ + { + "name": "angular", + "class": "org.apache.zeppelin.angular.AngularInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": true, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "livy": { + "id": "livy", + "name": "livy", + "group": "livy", + "properties": { + "livy.spark.executor.instances": { + "name": "livy.spark.executor.instances", + "value": "", + "type": "number" + }, + "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": { + "name": "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout", + "value": "", + "type": "string" + }, + "zeppelin.livy.concurrentSQL": { + "name": "zeppelin.livy.concurrentSQL", + "value": false, + "type": "checkbox" + }, + "zeppelin.livy.url": { + "name": "zeppelin.livy.url", + "value": "http://localhost:8998", + "type": "url" + }, + "zeppelin.livy.pull_status.interval.millis": { + "name": "zeppelin.livy.pull_status.interval.millis", + "value": "1000", + "type": "number" + }, + "livy.spark.executor.memory": { + "name": "livy.spark.executor.memory", + "value": "", + "type": "string" + }, + "zeppelin.livy.restart_dead_session": { + "name": "zeppelin.livy.restart_dead_session", + "value": false, + "type": "checkbox" + }, + "livy.spark.dynamicAllocation.enabled": { + "name": "livy.spark.dynamicAllocation.enabled", + "value": false, + "type": "checkbox" + }, + "zeppelin.livy.maxLogLines": { + "name": "zeppelin.livy.maxLogLines", + "value": "1000", + "type": "number" + }, + "livy.spark.dynamicAllocation.minExecutors": { + "name": "livy.spark.dynamicAllocation.minExecutors", + "value": "", + "type": "number" + }, + "livy.spark.executor.cores": { + "name": "livy.spark.executor.cores", + "value": "", + "type": "number" + }, + "zeppelin.livy.session.create_timeout": { + "name": "zeppelin.livy.session.create_timeout", + "value": "120", + "type": "number" + }, + "zeppelin.livy.spark.sql.maxResult": { + "name": "zeppelin.livy.spark.sql.maxResult", + "value": "1000", + "type": "number" + }, + "livy.spark.driver.cores": { + "name": "livy.spark.driver.cores", + "value": "", + "type": "number" + }, + "livy.spark.jars.packages": { + "name": "livy.spark.jars.packages", + "value": "", + "type": "textarea" + }, + "zeppelin.livy.spark.sql.field.truncate": { + "name": "zeppelin.livy.spark.sql.field.truncate", + "value": true, + "type": "checkbox" + }, + "livy.spark.driver.memory": { + "name": "livy.spark.driver.memory", + "value": "", + "type": "string" + }, + "zeppelin.livy.displayAppInfo": { + "name": "zeppelin.livy.displayAppInfo", + "value": true, + "type": "checkbox" + }, + "zeppelin.livy.principal": { + "name": "zeppelin.livy.principal", + "value": "", + "type": "string" + }, + "zeppelin.livy.keytab": { + "name": "zeppelin.livy.keytab", + "value": "", + "type": "textarea" + }, + "livy.spark.dynamicAllocation.maxExecutors": { + "name": "livy.spark.dynamicAllocation.maxExecutors", + "value": "", + "type": "number" + }, + "livy.spark.dynamicAllocation.initialExecutors": { + "name": "livy.spark.dynamicAllocation.initialExecutors", + "value": "", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.livy.LivySparkInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "scala", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "pyspark3", + "class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "sparkr", + "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "r", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "shared", + "class": "org.apache.zeppelin.livy.LivySharedInterpreter", + "defaultInterpreter": false + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "scoped", + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "groovy": { + "id": "groovy", + "name": "groovy", + "group": "groovy", + "properties": { + "GROOVY_CLASSES": { + "name": "GROOVY_CLASSES", + "value": "", + "type": "textarea" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "groovy", + "class": "org.apache.zeppelin.groovy.GroovyInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "neo4j": { + "id": "neo4j", + "name": "neo4j", + "group": "neo4j", + "properties": { + "neo4j.auth.user": { + "name": "neo4j.auth.user", + "value": "", + "type": "string" + }, + "neo4j.auth.password": { + "name": "neo4j.auth.password", + "value": "", + "type": "string" + }, + "neo4j.auth.type": { + "name": "neo4j.auth.type", + "value": "BASIC", + "type": "string" + }, + "neo4j.max.concurrency": { + "name": "neo4j.max.concurrency", + "value": "50", + "type": "string" + }, + "neo4j.url": { + "name": "neo4j.url", + "value": "bolt://localhost:7687", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "neo4j", + "class": "org.apache.zeppelin.graph.neo4j.Neo4jCypherInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "file": { + "id": "file", + "name": "file", + "group": "file", + "properties": { + "hdfs.url": { + "name": "hdfs.url", + "value": "http://localhost:50070/webhdfs/v1/", + "type": "url" + }, + "hdfs.user": { + "name": "hdfs.user", + "value": "hdfs", + "type": "string" + }, + "hdfs.maxlength": { + "name": "hdfs.maxlength", + "value": "1000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "hdfs", + "class": "org.apache.zeppelin.file.HDFSFileInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "elasticsearch": { + "id": "elasticsearch", + "name": "elasticsearch", + "group": "elasticsearch", + "properties": { + "elasticsearch.cluster.name": { + "name": "elasticsearch.cluster.name", + "value": "elasticsearch", + "type": "string" + }, + "elasticsearch.basicauth.username": { + "name": "elasticsearch.basicauth.username", + "value": "", + "type": "string" + }, + "elasticsearch.client.type": { + "name": "elasticsearch.client.type", + "value": "transport", + "type": "string" + }, + "elasticsearch.result.size": { + "name": "elasticsearch.result.size", + "value": "10", + "type": "number" + }, + "elasticsearch.port": { + "name": "elasticsearch.port", + "value": "9300", + "type": "number" + }, + "elasticsearch.host": { + "name": "elasticsearch.host", + "value": "localhost", + "type": "string" + }, + "elasticsearch.basicauth.password": { + "name": "elasticsearch.basicauth.password", + "value": "", + "type": "password" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "elasticsearch", + "class": "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "sh": { + "id": "sh", + "name": "sh", + "group": "sh", + "properties": { + "zeppelin.shell.interpolation": { + "name": "zeppelin.shell.interpolation", + "value": false, + "type": "checkbox" + }, + "zeppelin.shell.principal": { + "name": "zeppelin.shell.principal", + "value": "", + "type": "string" + }, + "shell.working.directory.user.home": { + "name": "shell.working.directory.user.home", + "value": false, + "type": "checkbox" + }, + "zeppelin.shell.auth.type": { + "name": "zeppelin.shell.auth.type", + "value": "", + "type": "string" + }, + "zeppelin.shell.keytab.location": { + "name": "zeppelin.shell.keytab.location", + "value": "", + "type": "string" + }, + "shell.command.timeout.millisecs": { + "name": "shell.command.timeout.millisecs", + "value": "60000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sh", + "class": "org.apache.zeppelin.shell.ShellInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "spark": { + "id": "spark", + "name": "spark", + "group": "spark", + "properties": { + "spark.executor.memory": { + "name": "spark.executor.memory", + "value": "1g", + "type": "string" + }, + "zeppelin.spark.sql.interpolation": { + "name": "zeppelin.spark.sql.interpolation", + "value": false, + "type": "checkbox" + }, + "zeppelin.spark.concurrentSQL": { + "name": "zeppelin.spark.concurrentSQL", + "value": false, + "type": "checkbox" + }, + "zeppelin.R.knitr": { + "name": "zeppelin.R.knitr", + "value": true, + "type": "checkbox" + }, + "zeppelin.R.cmd": { + "name": "zeppelin.R.cmd", + "value": "R", + "type": "string" + }, + "spark.app.name": { + "name": "spark.app.name", + "value": "Zeppelin", + "type": "string" + }, + "zeppelin.R.image.width": { + "name": "zeppelin.R.image.width", + "value": "100%", + "type": "number" + }, + "zeppelin.spark.importImplicit": { + "name": "zeppelin.spark.importImplicit", + "value": true, + "type": "checkbox" + }, + "zeppelin.dep.additionalRemoteRepository": { + "name": "zeppelin.dep.additionalRemoteRepository", + "value": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;", + "type": "textarea" + }, + "zeppelin.spark.maxResult": { + "name": "zeppelin.spark.maxResult", + "value": "1000", + "type": "number" + }, + "master": { + "name": "master", + "value": "local[*]", + "type": "string" + }, + "zeppelin.pyspark.python": { + "name": "zeppelin.pyspark.python", + "value": "/usr/bin/python3", + "type": "string" + }, + "args": { + "name": "args", + "value": "", + "type": "textarea" + }, + "zeppelin.spark.enableSupportedVersionCheck": { + "name": "zeppelin.spark.enableSupportedVersionCheck", + "value": true, + "type": "checkbox" + }, + "zeppelin.spark.useNew": { + "name": "zeppelin.spark.useNew", + "value": true, + "type": "checkbox" + }, + "zeppelin.dep.localrepo": { + "name": "zeppelin.dep.localrepo", + "value": "local-repo", + "type": "string" + }, + "zeppelin.pyspark.useIPython": { + "name": "zeppelin.pyspark.useIPython", + "value": true, + "type": "checkbox" + }, + "zeppelin.spark.sql.stacktrace": { + "name": "zeppelin.spark.sql.stacktrace", + "value": false, + "type": "checkbox" + }, + "zeppelin.spark.useHiveContext": { + "name": "zeppelin.spark.useHiveContext", + "value": true, + "type": "checkbox" + }, + "zeppelin.spark.uiWebUrl": { + "name": "zeppelin.spark.uiWebUrl", + "value": "", + "type": "string" + }, + "zeppelin.R.render.options": { + "name": "zeppelin.R.render.options", + "value": "out.format \u003d \u0027html\u0027, comment \u003d NA, echo \u003d FALSE, results \u003d \u0027asis\u0027, message \u003d F, warning \u003d F, fig.retina \u003d 2", + "type": "textarea" + }, + "zeppelin.spark.printREPLOutput": { + "name": "zeppelin.spark.printREPLOutput", + "value": true, + "type": "checkbox" + }, + "spark.cores.max": { + "name": "spark.cores.max", + "value": "1", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.spark.SparkInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "scala", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.spark.SparkSqlInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "dep", + "class": "org.apache.zeppelin.spark.DepInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "scala", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.spark.PySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionKey": "TAB", + "completionSupport": true + } + }, + { + "name": "ipyspark", + "class": "org.apache.zeppelin.spark.IPySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false, + "completionSupport": true + } + }, + { + "name": "r", + "class": "org.apache.zeppelin.spark.SparkRInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "r", + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "cassandra": { + "id": "cassandra", + "name": "cassandra", + "group": "cassandra", + "properties": { + "cassandra.ssl.truststore.password": { + "name": "cassandra.ssl.truststore.password", + "value": "none", + "type": "password" + }, + "cassandra.pooling.new.connection.threshold.remote": { + "name": "cassandra.pooling.new.connection.threshold.remote", + "value": "100", + "type": "number" + }, + "cassandra.query.default.fetchSize": { + "name": "cassandra.query.default.fetchSize", + "value": "5000", + "type": "number" + }, + "cassandra.socket.tcp.no_delay": { + "name": "cassandra.socket.tcp.no_delay", + "value": true, + "type": "checkbox" + }, + "cassandra.ssl.enabled": { + "name": "cassandra.ssl.enabled", + "value": false, + "type": "checkbox" + }, + "cassandra.hosts": { + "name": "cassandra.hosts", + "value": "localhost", + "type": "textarea" + }, + "cassandra.credentials.username": { + "name": "cassandra.credentials.username", + "value": "none", + "type": "string" + }, + "cassandra.pooling.new.connection.threshold.local": { + "name": "cassandra.pooling.new.connection.threshold.local", + "value": "100", + "type": "number" + }, + "cassandra.socket.read.timeout.millisecs": { + "name": "cassandra.socket.read.timeout.millisecs", + "value": "12000", + "type": "number" + }, + "cassandra.load.balancing.policy": { + "name": "cassandra.load.balancing.policy", + "value": "DEFAULT", + "type": "string" + }, + "cassandra.pooling.max.request.per.connection.local": { + "name": "cassandra.pooling.max.request.per.connection.local", + "value": "1024", + "type": "number" + }, + "cassandra.cluster": { + "name": "cassandra.cluster", + "value": "Test Cluster", + "type": "string" + }, + "cassandra.pooling.heartbeat.interval.seconds": { + "name": "cassandra.pooling.heartbeat.interval.seconds", + "value": "30", + "type": "number" + }, + "cassandra.query.default.serial.consistency": { + "name": "cassandra.query.default.serial.consistency", + "value": "SERIAL", + "type": "string" + }, + "cassandra.retry.policy": { + "name": "cassandra.retry.policy", + "value": "DEFAULT", + "type": "string" + }, + "cassandra.native.port": { + "name": "cassandra.native.port", + "value": "9042", + "type": "number" + }, + "cassandra.interpreter.parallelism": { + "name": "cassandra.interpreter.parallelism", + "value": "10", + "type": "number" + }, + "cassandra.pooling.pool.timeout.millisecs": { + "name": "cassandra.pooling.pool.timeout.millisecs", + "value": "5000", + "type": "number" + }, + "cassandra.pooling.max.request.per.connection.remote": { + "name": "cassandra.pooling.max.request.per.connection.remote", + "value": "256", + "type": "number" + }, + "cassandra.compression.protocol": { + "name": "cassandra.compression.protocol", + "value": "NONE", + "type": "string" + }, + "cassandra.ssl.truststore.path": { + "name": "cassandra.ssl.truststore.path", + "value": "none", + "type": "string" + }, + "cassandra.socket.connection.timeout.millisecs": { + "name": "cassandra.socket.connection.timeout.millisecs", + "value": "5000", + "type": "number" + }, + "cassandra.query.default.consistency": { + "name": "cassandra.query.default.consistency", + "value": "ONE", + "type": "string" + }, + "cassandra.keyspace": { + "name": "cassandra.keyspace", + "value": "system", + "type": "string" + }, + "cassandra.reconnection.policy": { + "name": "cassandra.reconnection.policy", + "value": "DEFAULT", + "type": "string" + }, + "cassandra.pooling.max.connection.per.host.local": { + "name": "cassandra.pooling.max.connection.per.host.local", + "value": "8", + "type": "number" + }, + "cassandra.credentials.password": { + "name": "cassandra.credentials.password", + "value": "none", + "type": "password" + }, + "cassandra.protocol.version": { + "name": "cassandra.protocol.version", + "value": "4", + "type": "string" + }, + "cassandra.max.schema.agreement.wait.second": { + "name": "cassandra.max.schema.agreement.wait.second", + "value": "10", + "type": "number" + }, + "cassandra.pooling.core.connection.per.host.remote": { + "name": "cassandra.pooling.core.connection.per.host.remote", + "value": "1", + "type": "number" + }, + "cassandra.pooling.core.connection.per.host.local": { + "name": "cassandra.pooling.core.connection.per.host.local", + "value": "2", + "type": "number" + }, + "cassandra.pooling.max.connection.per.host.remote": { + "name": "cassandra.pooling.max.connection.per.host.remote", + "value": "2", + "type": "number" + }, + "cassandra.pooling.idle.timeout.seconds": { + "name": "cassandra.pooling.idle.timeout.seconds", + "value": "120", + "type": "number" + }, + "cassandra.speculative.execution.policy": { + "name": "cassandra.speculative.execution.policy", + "value": "DEFAULT", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "cassandra", + "class": "org.apache.zeppelin.cassandra.CassandraInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "md": { + "id": "md", + "name": "md", + "group": "md", + "properties": { + "markdown.parser.type": { + "name": "markdown.parser.type", + "value": "pegdown", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "md", + "class": "org.apache.zeppelin.markdown.Markdown", + "defaultInterpreter": false, + "editor": { + "language": "markdown", + "editOnDblClick": true, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "alluxio": { + "id": "alluxio", + "name": "alluxio", + "group": "alluxio", + "properties": { + "alluxio.master.port": { + "name": "alluxio.master.port", + "value": "19998", + "type": "number" + }, + "alluxio.master.hostname": { + "name": "alluxio.master.hostname", + "value": "localhost", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "alluxio", + "class": "org.apache.zeppelin.alluxio.AlluxioInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "bigquery": { + "id": "bigquery", + "name": "bigquery", + "group": "bigquery", + "properties": { + "zeppelin.bigquery.project_id": { + "name": "zeppelin.bigquery.project_id", + "value": " ", + "type": "string" + }, + "zeppelin.bigquery.sql_dialect": { + "name": "zeppelin.bigquery.sql_dialect", + "value": "", + "type": "string" + }, + "zeppelin.bigquery.max_no_of_rows": { + "name": "zeppelin.bigquery.max_no_of_rows", + "value": "100000", + "type": "number" + }, + "zeppelin.bigquery.wait_time": { + "name": "zeppelin.bigquery.wait_time", + "value": "5000", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sql", + "class": "org.apache.zeppelin.bigquery.BigQueryInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "hbase": { + "id": "hbase", + "name": "hbase", + "group": "hbase", + "properties": { + "hbase.home": { + "name": "hbase.home", + "value": "/usr/lib/hbase/", + "type": "string" + }, + "zeppelin.hbase.test.mode": { + "name": "zeppelin.hbase.test.mode", + "value": false, + "type": "checkbox" + }, + "hbase.ruby.sources": { + "name": "hbase.ruby.sources", + "value": "lib/ruby", + "type": "string" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "hbase", + "class": "org.apache.zeppelin.hbase.HbaseInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": false, + "completionSupport": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + }, + "kylin": { + "id": "kylin", + "name": "kylin", + "group": "kylin", + "properties": { + "kylin.query.project": { + "name": "kylin.query.project", + "value": "learn_kylin", + "type": "textarea" + }, + "kylin.api.user": { + "name": "kylin.api.user", + "value": "ADMIN", + "type": "string" + }, + "kylin.query.limit": { + "name": "kylin.query.limit", + "value": "5000", + "type": "number" + }, + "kylin.api.url": { + "name": "kylin.api.url", + "value": "http://localhost:7070/kylin/api/query", + "type": "url" + }, + "kylin.api.password": { + "name": "kylin.api.password", + "value": "KYLIN", + "type": "password" + }, + "kylin.query.ispartial": { + "name": "kylin.query.ispartial", + "value": true, + "type": "checkbox" + }, + "kylin.query.offset": { + "name": "kylin.query.offset", + "value": "0", + "type": "number" + } + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "kylin", + "class": "org.apache.zeppelin.kylin.KylinInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false, + "completionSupport": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "isExistingProcess": false, + "setPermission": false, + "owners": [], + "isUserImpersonate": false + } + } + }, + "interpreterBindings": { + "2DKXYM99T": [ + "spark", + "md", + "angular", + "sh", + "livy", + "alluxio", + "file", + "flink", + "python", + "ignite", + "lens", + "cassandra", + "kylin", + "elasticsearch", + "jdbc", + "hbase", + "bigquery", + "pig", + "groovy", + "neo4j", + "sap" + ], + "2C2AUG798": [ + "spark", + "md", + "angular", + "sh", + "livy", + "alluxio", + "file", + "flink", + "python", + "ignite", + "lens", + "cassandra", + "kylin", + "elasticsearch", + "jdbc", + "hbase", + "bigquery", + "pig", + "groovy", + "neo4j", + "sap" + ] + }, + "interpreterRepositories": [ + { + "id": "central", + "type": "default", + "url": "http://repo1.maven.org/maven2/", + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + }, + { + "id": "local", + "type": "default", + "url": "file:///opt/helk/spark/.m2/repository", + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + } + ] +} \ No newline at end of file diff --git a/helk-jupyter/spark/spark-defaults.conf b/docker/helk-zeppelin/spark-defaults.conf similarity index 94% rename from helk-jupyter/spark/spark-defaults.conf rename to docker/helk-zeppelin/spark-defaults.conf index 95c2de14..24dd6cb0 100644 --- a/helk-jupyter/spark/spark-defaults.conf +++ b/docker/helk-zeppelin/spark-defaults.conf @@ -33,6 +33,6 @@ # https://spark.apache.org/docs/latest/sql-programming-guide.html#pyspark-usage-guide-for-pandas-with-apache-arrow spark.master spark://helk-spark-master:7077 -spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.3.1.jar +spark.jars /opt/helk/zeppelin/spark-cypher-0.1.6-cluster.jar,/opt/helk/es-hadoop/elasticsearch-hadoop-6.3.1.jar spark.jars.packages graphframes:graphframes:0.5.0-spark2.1-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0,databricks:spark-sklearn:0.2.3 spark.sql.execution.arrow.enabled true \ No newline at end of file diff --git a/docker/helk-zeppelin/zeppelin-env.sh.template b/docker/helk-zeppelin/zeppelin-env.sh.template new file mode 100644 index 00000000..7c4c3583 --- /dev/null +++ b/docker/helk-zeppelin/zeppelin-env.sh.template @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# export JAVA_HOME= +# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode. +# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" +# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m +# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default -Xms1024m -Xmx1024m -XX:MaxPermSize=512m +# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. +# export ZEPPELIN_SSL_PORT # ssl port (used when ssl environment variable is set to true) + +# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default. +# export ZEPPELIN_PID_DIR # The pid files are stored. ${ZEPPELIN_HOME}/run by default. +# export ZEPPELIN_WAR_TEMPDIR # The location of jetty temporary directory. +# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved +# export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z +# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false" +# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved +# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket +# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json +# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID +# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region +# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. +# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. +# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading +# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading +# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader +# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). +# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? +# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise + +#### Spark interpreter configuration #### + +## Use provided spark installation ## +## defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit +## +# export SPARK_HOME # (required) When it is defined, load it instead of Zeppelin embedded Spark libraries +# export SPARK_SUBMIT_OPTIONS # (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G". +# export SPARK_APP_NAME # (optional) The name of spark application. + +## Use embedded spark binaries ## +## without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries. +## however, it is not encouraged when you can define SPARK_HOME +## +# Options read in YARN client mode +# export HADOOP_CONF_DIR # yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. +# Pyspark (supported with Spark 1.2.1 and above) +# To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI +# export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers. +# export PYTHONPATH + +## Spark interpreter options ## +## +# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default. +# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default. +# export ZEPPELIN_SPARK_IMPORTIMPLICIT # Import implicits, UDF collection, and sql if set true. true by default. +# export ZEPPELIN_SPARK_MAXRESULT # Max number of Spark SQL result to display. 1000 by default. +# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000 + + +#### HBase interpreter configuration #### + +## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set + +# export HBASE_HOME= # (require) Under which HBase scripts and configuration should be +# export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml + +#### ZeppelinHub connection configuration #### +# export ZEPPELINHUB_API_ADDRESS # Refers to the address of the ZeppelinHub service in use +# export ZEPPELINHUB_API_TOKEN # Refers to the Zeppelin instance token of the user +# export ZEPPELINHUB_USER_KEY # Optional, when using Zeppelin with authentication. + +#### Zeppelin impersonation configuration +# export ZEPPELIN_IMPERSONATE_CMD # Optional, when user want to run interpreter as end web user. eg) 'sudo -H -u ${ZEPPELIN_IMPERSONATE_USER} bash -c ' +# export ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER #Optional, by default is true; can be set to false if you don't want to use --proxy-user option with Spark interpreter when impersonation enabled \ No newline at end of file diff --git a/helk-zookeeper/Dockerfile b/docker/helk-zookeeper/Dockerfile similarity index 92% rename from helk-zookeeper/Dockerfile rename to docker/helk-zookeeper/Dockerfile index abc4e091..603122ff 100644 --- a/helk-zookeeper/Dockerfile +++ b/docker/helk-zookeeper/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-kafka-base:1.1.0 +FROM cyb3rward0g/helk-kafka-base:1.1.1 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kafka Zookeeper." @@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND noninteractive # *********** Creating the right directories *************** RUN bash -c 'mkdir -pv /opt/helk/zookeeper' -# *********** Install Zookeeper *************** +# *********** ConfigureZookeeper *************** ENV ZOO_CONF_DIR=/opt/helk/zookeeper/conf \ ZOO_DATA_DIR=/opt/helk/zookeeper/data \ ZOO_DATA_LOG_DIR=/opt/helk/zookeeper/datalog \ diff --git a/helk-zookeeper/scripts/zookeeper-entrypoint.sh b/docker/helk-zookeeper/scripts/zookeeper-entrypoint.sh similarity index 100% rename from helk-zookeeper/scripts/zookeeper-entrypoint.sh rename to docker/helk-zookeeper/scripts/zookeeper-entrypoint.sh diff --git a/helk_install.sh b/docker/helk_install.sh similarity index 90% rename from helk_install.sh rename to docker/helk_install.sh index 4ce0826d..1aa57d8d 100755 --- a/helk_install.sh +++ b/docker/helk_install.sh @@ -18,6 +18,9 @@ echoerror() { printf "${RC} * ERROR${EC}: $@\n" 1>&2; } +# ********* Commention Out CDROM ********************** +sed -i "s/\(^deb cdrom.*$\)/\#/g" /etc/apt/sources.list + # ********** Check Minimum Requirements ************** check_min_requirements(){ # *********** Check System Kernel Name *************** @@ -25,7 +28,6 @@ check_min_requirements(){ echo "[HELK-INSTALLATION-INFO] HELK being hosted on a $systemKernel box" if [ "$systemKernel" == "Linux" ]; then AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo) - ES_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024/2}' /proc/meminfo) AVAILABLE_DISK=$(df -m | awk '$NF=="/"{printf "%.f\t\t", $4 / 1024}') if [ "${AVAILABLE_MEMORY}" -ge "12" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then echo "[HELK-INSTALLATION-INFO] Available Memory: $AVAILABLE_MEMORY" @@ -45,15 +47,6 @@ check_min_requirements(){ fi } -# *********** Getting Jupyter Token *************** -get_jupyter_token(){ - echo "[HELK-INSTALLATION-INFO] Waiting for HELK services and Jupyter Server to start.." - until curl -s localhost:8880 -o /dev/null; do - sleep 1 - done - jupyter_token="$(docker exec -ti helk-jupyter jupyter notebook list | grep -oP '(?<=token=).*(?= ::)' | awk '{$1=$1};1')" >> $LOGFILE 2>&1 -} - # ********** Install Curl ******************** install_curl(){ echo "[HELK-INSTALLATION-INFO] Checking if curl is installed first" @@ -260,9 +253,6 @@ prepare_helk(){ # ****** Setting KAFKA ADVERTISED_LISTENER environment variable *********** sed -i "s/ADVERTISED_LISTENER=HOSTIP/ADVERTISED_LISTENER=$host_ip/g" docker-compose-elk-${license_choice}.yml - echo "[HELK-INSTALLATION-INFO] Setting ES_JAVA_OPTS value..." - # ****** Setting ES JAVA OPTS environment variable *********** - sed -i "s/ES_JAVA_OPTS\=\-Xms6g \-Xmx6g/ES_JAVA_OPTS\=\-Xms${ES_MEMORY}g \-Xmx${ES_MEMORY}g/g" docker-compose-elk-${license_choice}.yml } show_banner(){ @@ -272,8 +262,8 @@ show_banner(){ echo "** HELK - THE HUNTING ELK **" echo "** **" echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **" - echo "** HELK build version: v0.1.1-alpha07062018 **" - echo "** HELK ELK version: 6.3.1 **" + echo "** HELK build version: v0.1.2-alpha08032018 **" + echo "** HELK ELK version: 6.3.2 **" echo "** License: GPL-3.0 **" echo "**********************************************" echo " " @@ -287,15 +277,13 @@ show_final_information(){ echo "** [HELK-INSTALLATION-INFO] USE THE FOLLOWING SETTINGS TO INTERACT WITH THE HELK **" echo "***********************************************************************************" echo " " - echo "HELK KIBANA URL: http://${host_ip}" - echo "HELK KIBANA & ELASTICSEARCH USER: helk" - echo "HELK KIBANA & ELASTICSEARCH PASSWORD: hunting" - echo "HELK JUPYTER CURRENT TOKEN: ${jupyter_token}" - echo "HELK JUPYTER LAB URL: http://${host_ip}:8880/lab" - echo "HELK SPARK Pyspark UI: http://${host_ip}:4040" - echo "HELK SPARK Cluster Master UI: http://${host_ip}:8080" - echo "HELK SPARK Cluster Worker1 UI: http://${host_ip}:8081" - echo "HELK SPARK Cluster Worker2 UI: http://${host_ip}:8082" + echo "HELK KIBANA URL: https://${host_ip}" + echo "HELK KIBANA USER: helk" + echo "HELK KIBANA PASSWORD: hunting" + echo "HELK JUPYTERHUB URL: http://${host_ip}/jupyter" + echo "HELK JUPYTERHUB USER:PWD : hunter1:hunter1@P@ssw0rd!" + echo "HELK JUPYTERHUB USER:PWD : hunter2:hunter2@P@ssw0rd!" + echo "HELK SPARK MASTER UI: http://${host_ip}:8080" echo " " echo "IT IS HUNTING SEASON!!!!!" echo " " @@ -311,7 +299,6 @@ manual_install(){ set_helk_license prepare_helk install_helk - get_jupyter_token sleep 180 show_final_information } @@ -321,7 +308,6 @@ automatic_install(){ check_min_requirements prepare_helk install_helk - get_jupyter_token sleep 180 show_final_information } diff --git a/helk-jupyter/kernels/pyspark_kernel.json b/helk-jupyter/kernels/pyspark_kernel.json deleted file mode 100644 index 90fdcd02..00000000 --- a/helk-jupyter/kernels/pyspark_kernel.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "argv": [ - "python3", - "-m", - "ipykernel_launcher", - "-f", - "{connection_file}" - ], - "display_name": "PySpark", - "language": "python" -} \ No newline at end of file diff --git a/helk-jupyter/scripts/jupyter-entrypoint.sh b/helk-jupyter/scripts/jupyter-entrypoint.sh deleted file mode 100755 index 757cf3a9..00000000 --- a/helk-jupyter/scripts/jupyter-entrypoint.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -# HELK script: jupyter-entryppoint.sh -# HELK script description: Restart HELK Jupyter Services -# HELK build Stage: Alpha -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -ln -sf /dev/stdout $JUPYTER_CONSOLE_LOG - -echo "[HELK-DOCKER-INSTALLATION-INFO] Starting jupyter services.." -exec $JUPYTER_EXEC >> $JUPYTER_CONSOLE_LOG 2>&1 diff --git a/helk-logstash/trial/pipeline/10-winevent-powershell-filter.conf b/helk-logstash/trial/pipeline/10-winevent-powershell-filter.conf deleted file mode 100644 index bb97857f..00000000 --- a/helk-logstash/trial/pipeline/10-winevent-powershell-filter.conf +++ /dev/null @@ -1,136 +0,0 @@ -# HELK powershell filter conf file -# HELK build Stage: Alpha -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -# Reference: -# Nate Guagenti (@neu5ron) https://gist.github.com/neu5ron/450289373db61d5c8d7378e79455ef07#file-511-windows-event-powershell-operational-conf - -filter { - if [source_name] == "Microsoft-Windows-PowerShell" or [source_name] == "PowerShell"{ - if [event_id] == 4103 { - mutate { - add_field => [ "PayloadInvocation", "%{[event_data][Payload]}" ] - add_field => [ "PayloadParams", "%{[event_data][Payload]}" ] - gsub => [ - "[event_data][ContextInfo]", " ", "", - "[event_data][ContextInfo]", " = ", "=" - ] - } - mutate { - gsub => [ - "PayloadInvocation", "CommandInvocation\(.*\)", "commandinvocation", - "PayloadInvocation", "ParameterBinding.*\r\n", "", - "PayloadParams", "parameterbinding\(.*\)", "parameterbinding", - "PayloadParams", "CommandInvocation.*\r\n", "", - "[event_data][Payload]", "CommandInvocation.*\r\n", "", - "[event_data][Payload]", "ParameterBinding.*\r\n", "" - ] - rename => { "[event_load][Payload]" => "[powershell][payload]" } - } - kv { - source => "PayloadInvocation" - field_split => "\n" - value_split => ":" - allow_duplicate_values => false - target => "[powershell]" - include_keys => [ "commandinvocation" ] - } - kv { - source => "PayloadParams" - value_split => "=" - allow_duplicate_values => false - target => "[powershell][param]" - include_keys => [ "name", "value" ] - } - kv { - source => "[event_data][ContextInfo]" - field_split => "\r\n" - value_split => "=" - remove_char_key => " " - allow_duplicate_values => false - include_keys => [ "Severity", "HostName", "HostVersion", "HostID", "HostApplication", "EngineVersion", "RunspaceID", "PipelineID", "CommandName", "CommandType", "ScriptName", "CommandPath", "SequenceNumber", "ConnectedUser", "ShellID" ] - } - mutate { - rename => { - "CommandName" => "[powershell][command][name]" - "CommandPath" => "[powershell][command][path]" - "CommandType" => "[powershell][command][type]" - "ConnectedUser" => "[powershell][connected][user]" - "EngineVersion" => "[powershell][engine][version]" - "HostApplication" => "[powershell][host][application]" - "HostID" => "[powershell][host][id]" - "HostName" => "[powershell][host][name]" - "HostVersion" => "[powershell][host][version]" - "PipelineID" => "[powershell][pipeline][id]" - "RunspaceID" => "[powershell][runspace][id]" - "Scriptname" => "[powershell][script][name]" - "SequenceNumber" => "[powershell][sequence][number]" - "ShellID" => "[powershell][shell][id]" - } - remove_field => [ - "Severity", - "EventType", - "Keywords", - "message", - "Opcode", - "PayloadInvocation", - "PayloadParams", - "[event_data][Payload]", - "[event_data][ContextInfo]" - ] - convert => { "[powershell][pipeline][id]" => "integer" } - convert => { "[powershell][sequence][number]" => "integer" } - } - } - if [event_id] == 4104 { - mutate { - rename => { - "[event_data][MessageNumber]" => "[powershell][message][number]" - "[event_data][MessageTotal]" => "[powershell][message][total]" - "[event_data][ScriptBlockId]" => "[powershell][scriptblock][id]" - "[event_data][ScriptBlockText]" => "[powershell][scriptblock][text]" - "[event_data][Path]" => "[powershell][script][path]" - } - remove_field => [ "message" ] - convert => { "[powershell][message][number]" => "integer" } - convert => { "[powershell][message][total]" => "integer" } - convert => { "[powershell][scriptblock][id]" => "integer" } - } - } - if [event_id] == 400 or [event_id] == 600 { - kv { - source => "[event_data][param3]" - field_split => "\n" - value_split => "=" - trim_key => "\t" - allow_duplicate_values => false - } - mutate { - rename => { - "ProviderName" => "[powershell][providername]" - "NewProviderState" => "[powershell][newproviderstate]" - "SequenceNumber" => "[powershell][sequencenumber" - "HostName" => "[powershell][host][name]" - "HostVersion" => "[powershell][host][version]" - "HostId" => "[powershell][host][id]" - "HostApplication" => "[powershell][host][application]" - "EngineVersion" => "[powershell][engine][version]" - "RunspaceId" => "[powershell][runspace][id]" - "PipelineId" => "[powershell][pipeline][id]" - "CommandName" => "[powershell][command][name]" - "CommandType" => "[powershell][command][type]" - "ScriptName" => "[powershell][script][name]" - "CommandPath" => "[powershell][command][path]" - "CommandLine" => "[powershell][command][line]" - "NewEngineState" => "[powershell][newengine][state]" - "PreviousEngineState" => "[powershell][previousengine][state]" - } - remove_field => [ "message" ] - remove_field => "[event_data][param1]" - remove_field => "[event_data][param2]" - remove_field => "[event_data][param3]" - } - } - } -} diff --git a/helk-logstash/trial/pipeline/11-winevent-sysmon-filter.conf b/helk-logstash/trial/pipeline/11-winevent-sysmon-filter.conf deleted file mode 100644 index bc7c8baf..00000000 --- a/helk-logstash/trial/pipeline/11-winevent-sysmon-filter.conf +++ /dev/null @@ -1,315 +0,0 @@ -# HELK sysmon filter conf file -# HELK build Stage: Alpha -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -filter { - if [log_name] == "Microsoft-Windows-Sysmon/Operational"{ - mutate { - rename => { - "[user][domain]" => "user_reporter_domain" - "[user][identifier]" => "user_reporter_sid" - "[user][name]" => "user_reporter_name" - "[user][type]" => "user_reporter_type" - "computer_name" => "host_name" - } - } - if [provider_guid] { - mutate { - gsub => ["provider_guid","[{}]",""] - } - } - if [event_data][RuleName] { - kv { - source => "[event_data][RuleName]" - field_split => "," - value_split => "=" - prefix => "mitre_" - transform_key => "lowercase" - } - } - if [event_data][Image] { - if [event_data][Image] =~ /^(\w*$)|^(\w*\..*$)/ { - mutate { - copy => {"[event_data][Image]" => "process_name"} - } - } - else { - grok { - match => { "[event_data][Image]" => ".*\\%{GREEDYDATA:process_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - } - mutate { - rename => { - "[event_data][Image]" => "process_path" - "[event_data][ProcessGuid]" => "process_guid" - "[event_data][ProcessId]" => "process_id" - } - gsub => ["process_guid","[{}]",""] - } - } - if [event_data][ParentImage] { - grok { - match => { "[event_data][ParentImage]" => ".*\\%{GREEDYDATA:process_parent_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - } - if [event_data][Hashes] { - kv { - source => "[event_data][Hashes]" - field_split => "," - value_split => "=" - prefix => "hash_" - transform_key => "lowercase" - } - } - if [event_data][TargetImage] { - grok { - match => { "[event_data][SourceImage]" => ".*\\%{GREEDYDATA:process_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - grok { - match => { "[event_data][TargetImage]" => ".*\\%{GREEDYDATA:target_process_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - mutate { - rename => { - "[event_data][SourceImage]" => "process_path" - "[event_data][SourceProcessGUID]" => "process_guid" - "[event_data][SourceProcessId]" => "process_id" - "[event_data][TargetImage]" => "target_process_path" - "[event_data][TargetProcessGUID]" => "target_process_guid" - "[event_data][TargetProcessId]" => "target_process_id" - } - gsub => ["process_guid","[{}]",""] - gsub => ["target_process_guid","[{}]",""] - } - } - if [event_data][User] { - grok { - match => { "[event_data][User]" => "%{GREEDYDATA:user_domain}\\%{GREEDYDATA:user_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - } - if [event_data][LogonId] { - mutate { rename => { "[event_data][LogonId]" => "user_logon_id" }} - mutate { gsub => [ "user_logon_id", "0x", "" ]} - ruby { - code => " - userlogonid = event.get('user_logon_id') - userlogonid = userlogonid.hex - event.set('user_logon_id', userlogonid) - " - tag_on_exception => "_0591_rubyexception" - } - } - if [event_id] == 1 { - mutate { - add_field => { "action" => "processcreate" } - rename => { - "[event_data][CommandLine]" => "process_command_line" - "[event_data][CurrentDirectory]" => "process_current_directory" - "[event_data][ParentImage]" => "process_parent_path" - "[event_data][ParentCommandLine]" => "process_parent_command_line" - "[event_data][IntegrityLevel]" => "process_integrity_level" - "[event_data][LogonGuid]" => "user_logon_guid" - "[event_data][ParentProcessGuid]" => "process_parent_guid" - "[event_data][ParentProcessId]" => "process_parent_id" - "[event_data][TerminalSessionId]" => "user_session_id" - "[event_data][FileVersion]" => "file_version" - "[event_data][Description]" => "file_description" - "[event_data][Product]" => "file_product" - "[event_data][Company]" => "file_company" - } - gsub => ["process_parent_guid","[{}]",""] - gsub => ["user_logon_guid","[{}]",""] - } - } - if [event_id] == 2 { - mutate { - add_field => { "action" => "filecreatetime" } - rename => { - "[event_data][TargetFileName]" => "file_name" - } - } - } - if [event_id] == 3 { - mutate { - add_field => { "action" => "networkconnect" } - rename => { - "[event_data][DestinationHostname]" => "dst_host_name" - "[event_data][DestinationIp]" => "dst_ip" - "[event_data][DestinationIsIpv6]" => "dst_is_ipv6" - "[event_data][DestinationPort]" => "dst_port" - "[event_data][DestinationPortName]" => "dst_port_name" - "[event_data][Initiated]" => "network_initiated" - "[event_data][Protocol]" => "network_protocol" - "[event_data][SourceHostname]" => "src_host_name" - "[event_data][SourceIp]" => "src_ip" - "[event_data][SourceIsIpv6]" => "src_is_ipv6" - "[event_data][SourcePort]" => "src_port" - "[event_data][SourcePortName]" => "src_port_name" - } - } - geoip { - source => "dst_ip" - remove_field => "[geoip][ip]" - } - } - if [event_id] == 4 { - mutate { - add_field => { "action" => "sysmonservicestatechanged" } - rename => { - "[event_data][State]" => "service_state" - "[event_data][Version]" => "sysmon_version" - "[event_data][SchemaVersion]" => "sysmon_schema_version" - } - } - } - if [event_id] == 6 { - mutate { - add_field => { "action" => "driverload" } - rename => { - "[event_data][ImageLoaded]" => "driver_loaded" - "[event_data][Signature]" => "driver_signature" - "[event_data][SignatureStatus]" => "driver_signature_status" - "[event_data][Signed]" => "driver_signed" - } - } - } - if [event_id] == 7 { - mutate { - add_field => { "action" => "imageload" } - rename => { - "[event_data][ImageLoaded]" => "module_loaded" - "[event_data][Signature]" => "module_signature" - "[event_data][SignatureStatus]" => "module_signature_status" - "[event_data][Signed]" => "module_signed" - } - } - } - if [event_id] == 8 { - mutate { - add_field => { "action" => "createremotethread" } - rename => { - "[event_data][NewThreadId]" => "thread_new_id" - "[event_data][StartAddress]" => "thread_start_address" - "[event_data][StartFunction]" => "thread_start_function" - "[event_data][StartModule]" => "thread_start_module" - } - } - } - if [event_id] == 9 { - mutate { - add_field => { "action" => "rawaccessread" } - rename => { - "[event_data][Device]" => "device_name" - } - } - } - if [event_id] == 10 { - mutate { - add_field => { "action" => "processaccess" } - rename => { - "[event_data][CallTrace]" => "process_call_trace" - "[event_data][GrantedAccess]" => "process_granted_access" - "[event_data][SourceThreadId]" => "thread_id" - } - } - } - if [event_id] == 11 { - mutate { - add_field => { "action" => "filecreate" } - rename => { - "[event_data][TargetFilename]" => "file_name" - } - } - } - if [event_id] == 12 or [event_id] == 13 or [event_id] == 14 { - mutate { - add_field => { "action" => "registryevent" } - rename => { - "[event_data][EventType]" => "event_type" - "[event_data][TargetObject]" => "registry_key_path" - "[event_data][Details]" => "registry_key_value" - } - } - } - if [event_id] == 15 { - mutate { - add_field => { "action" => "filecreatestreamhash" } - rename => { - "[event_data][TargetFilename]" => "file_name" - "[event_data][Hash]" => "hash" - } - } - } - if [event_id] == 16 { - kv { - source => "[event_data][ConfigurationFileHash]" - value_split => "=" - prefix => "sysmon_config_hash_" - transform_key => "lowercase" - } - mutate { - add_field => { "action" => "sysmonconfigstatechanged" } - rename => { - "[event_data][State]" => "sysmon_configuration_state" - "[event_data][Configuration]" => "sysmon_configuration" - } - } - } - if [event_id] == 18 or [event_id] == 17 { - mutate { - add_field => { "action" => "pipeevent" } - rename => { - "[event_data][PipeName]" => "pipe_name" - } - } - } - if [event_id] == 19 or [event_id] == 20 or [event_id] == 21 { - mutate { - add_field => { "action" => "wmievent" } - rename => { - "[event_data][EventType]" => "wmi_event_type" - "[event_data][Operation]" => "wmi_operation" - "[event_data][EventNamespace]" => "wmi_namespace" - "[event_data][Name]" => "wmi_name" - "[event_data][Query]" => "wmi_query" - "[event_data][Type]" => "wmi_type" - "[event_data][Destination]" => "wmi_destination" - } - } - } - date { - timezone => "UTC" - match => [ "[event_data][UtcTime]", "YYYY-MM-dd HH:mm:ss.SSS" ] - tag_on_failure => [ "_sysmon_datefailure", "_dateparsefailure" ] - } - date { - timezone => "UTC" - match => [ "[event_data][CreationUtcTime]", "YYYY-MM-dd HH:mm:ss.SSS" ] - target => "@date_creation" - tag_on_failure => [ "_sysmon_datefailure", "_dateparsefailure" ] - } - date { - timezone => "UTC" - match => [ "[event_data][PreviousCreationUtcTime]", "YYYY-MM-dd HH:mm:ss.SSS" ] - target => "@date_creation_previous" - tag_on_failure => [ "_sysmon_datefailure", "_dateparsefailure" ] - } - mutate { - remove_field => "[event_data]" - remove_field => "message" - remove_field => "[event_data][Hashes]" - remove_field => "[event_data][ConfigurationFileHash]" - remove_field => "[event_data][UtcTime]" - remove_field => "[event_data][CreationUtcTime]" - remove_field => "[event_data][PreviousCreationUtcTime]" - remove_field => "[user]" - rename => { "[event_data][User]" => "user_account"} - } - } -} - diff --git a/helk-logstash/trial/pipeline/12-winevent-security-filter.conf b/helk-logstash/trial/pipeline/12-winevent-security-filter.conf deleted file mode 100644 index 0401e947..00000000 --- a/helk-logstash/trial/pipeline/12-winevent-security-filter.conf +++ /dev/null @@ -1,854 +0,0 @@ -# HELK winevent-security filter conf file -# HELK build Stage: Alpha -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -filter { - if [log_name] == "Security"{ - if [event_data][ProcessName] { - grok { - match => { - "[event_data][ProcessName]" => ".*\\%{GREEDYDATA:process_name}" - } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - } - if [event_id] == 4611 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4611.md - mutate { - rename => { - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][LogonProcessName]" => "logon_process_name" - } - } - } - if [event_id] == 4616 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4616.md - mutate { - rename => { - "[event_data][NewTime]" => "@date_new_time" - "[event_data][PreviousTime]" => "@date_previous_time" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - } - } - } - if [event_id] == 4624 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4624.md - mutate { - rename => { - "[event_data][SubjectUserSid]" => "user_reporter_sid" - "[event_data][SubjectUserName]" => "user_reporter_name" - "[event_data][SubjectDomainName]" => "user_reporter_domain" - "[event_data][SubjectLogonId]" => "reporter_logon_id" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][TargetLogonId]" => "user_logon_id" - "[event_data][LogonType]" => "logon_type" - "[event_data][RestrictedAdminMode]" => "logon_restricted_adminmode" - "[event_data][VirtualAccount]" => "logon_virtual_account" - "[event_data][ElevatedToken]" => "logon_elevated_token" - "[event_data][TargetUserName]" => "user_name" - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetLinkedLogonId]" => "user_linked_logon_id" - "[event_data][TargetOutboundDomainName]" => "user_network_account_domain" - "[event_data][TargetOutboundUserName]" => "user_network_account_name" - "[event_data][TargetUserSid]" => "user_sid" - "[event_data][ImpersonationLevel]" => "impersonation_level" - "[event_data][LogonGuid]" => "user_logon_guid" - "[event_data][WorkstationName]" => "src_host_name" - "[event_data][IpAddress]" => "src_ip" - "[event_data][IpPort]" => "src_port" - "[event_data][LogonProcessName]" => "logon_process_name" - "[event_data][AuthenticationPackageName]" => "logon_authentication_package_name" - "[event_data][TransmittedServices]" => "logon_transmitted_services" - "[event_data][LmPackageName]" => "logon_package_name" - "[event_data][KeyLength]" => "logon_key_length" - } - } - if "logon_elevated_token" == "Yes"{ - mutate { - add_tag => ["elevated_logon"] - } - } - } - if [event_id] == 4625 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4625.md - mutate { - rename => { - "[event_data][SubjectUserSid]" => "user_reporter_sid" - "[event_data][SubjectUserName]" => "user_reporter_name" - "[event_data][SubjectDomainName]" => "user_reporter_domain" - "[event_data][SubjectLogonId]" => "reporter_logon_id" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][LogonType]" => "logon_type" - "[event_data][TargetUserName]" => "user_name" - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetUserSid]" => "user_sid" - "[event_data][WorkstationName]" => "src_host" - "[event_data][IpAddress]" => "src_ip" - "[event_data][IpPort]" => "src_port" - "[event_data][LogonProcessName]" => "logon_process_name" - "[event_data][AuthenticationPackageName]" => "logon_authentication_package_name" - "[event_data][TransmittedServices]" => "logon_transmitted_services" - "[event_data][LmPackageName]" => "logon_package_name" - "[event_data][KeyLength]" => "logon_key_length" - "[event_data][FailureReason]" => "logon_failure_reason" - "[event_data][Status]" => "logon_failure_status" - "[event_data][SubStatus]" => "logon_failure_substatus" - } - } - if "logon_elevated_token" == "Yes"{ - mutate { - add_tag => ["elevated_logon"] - } - } - } - if [event_id] == 4627 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4627.md - mutate { - rename => { - "[event_data][SubjectUserSid]" => "user_reporter_sid" - "[event_data][SubjectUserName]" => "user_reporter_name" - "[event_data][SubjectDomainName]" => "user_reporter_domain" - "[event_data][SubjectLogonId]" => "reporter_logon_id" - "[event_data][TargetLogonId]" => "user_logon_id" - "[event_data][LogonType]" => "logon_type" - "[event_data][TargetUserName]" => "user_name" - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetUserSid]" => "user_sid" - "[event_data][GroupMembership]" => "user_group_membership" - } - remove_field => "[event_data][EventCountTotal]" - remove_field => "[event_data][EventIdx]" - } - } - if [event_id] == 4634 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4634.md - mutate { - rename => { - "[event_data][LogonType]" => "logon_type" - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetLogonId]" => "user_logon_id" - "[event_data][TargetUserName" => "user_name" - "[event_data][TargetUserSid]" => "user_sid" - } - } - } - if [event_id] == 4647 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4647.md - mutate { - rename => { - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetLogonId]" => "user_logon_id" - "[event_data][TargetUserName" => "user_name" - "[event_data][TargetUserSid]" => "user_sid" - } - } - } - if [event_id] == 4648 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4648.md - mutate { - rename => { - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][LogonGuid]" => "user_logon_guid" - "[event_data][TargetUserName]" => "target_user_name" - "[event_data][TargetDomainName]" => "target_user_domain" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][TargetLogonId]" => "target_user_logon_id" - "[event_data][TargetServerName]" => "service_host_name" - "[event_data][TargetInfo]" => "service_host_info" - "[event_data][TargetLogonGuid]" => "target_user_logon_guid" - "[event_data][IpAddress]" => "src_ip" - "[event_data][IpPort]" => "src_port" - } - } - } - if [event_id] == 4656 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4656.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][AccessList]" => "object_access_list_requested" - "[event_data][AccessMask]" => "object_access_mask_requested" - "[event_data][AccessReason]" => "object_access_reason" - "[event_data][ObjectName]" => "object_name" - "[event_data][ObjectServer]" => "object_server" - "[event_data][ObjectType]" => "object_type" - "[event_data][HandleId]" => "object_access_handle_id" - "[event_data][PrivilegeList]" => "object_privilege_list" - "[event_data][TransactionId" => "object_access_transaction_id" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][ResourceAttributes]" => "object_resource_attributes" - "event_data][RestrictedSidCount]" => "object_restricted_sid_count" - } - } - } - if [event_id] == 4657 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4657.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ObjectName]" => "object_name" - "[event_data][ObjectValueName]" => "object_value_name" - "[event_data][HandleId]" => "object_access_handle_id" - "[event_data][OperationType]" => "object_operation_type" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][OldValueType]" => "object_value_old_type" - "[event_data][OldValue]" => "object_value_old" - "[event_data][NewValueType]" => "object_value_new_type" - "[event_data][NewValue]" => "object_value_new" - } - } - } - if [event_id] == 4658 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4658.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][ObjectServer]" => "object_server" - "[event_data][HandleId]" => "object_access_handle_id" - } - } - } - if [event_id] == 4661 or [event_id] == 4662 or [event_id] == 4663 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4661.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4662.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4663.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][Properties]" => "object_properties" - "[event_data][AccessMask]" => "object_access_mask_requested" - "[event_data][HandleId]" => "object_access_handle_id" - "[event_data][ObjectName]" => "object_name" - "[event_data][ObjectServer]" => "object_server" - "[event_data][ObjectType]" => "object_type" - "[event_data][AdditionalInfo2]" => "object_additional_info2" - "[event_data][OperationType]" => "object_operation_type" - "[event_data][AdditionalInfo]" => "object_additional_info" - "[event_data][AccessList]" => "object_access_list_requested" - "[event_data][ResourceAttributes]" => "object_resource_attributes" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][AccessReason]" => "object_access_reason" - "[event_data][PrivilegeList]" => "object_privilege_list" - "[event_data][TransactionId" => "object_access_transaction_id" - "event_data][RestrictedSidCount]" => "object_restricted_sid_count" - } - } - } - if [event_id] == 4670 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4670.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][HandleId]" => "object_access_handle_id" - "[event_data][NewSd]" => "object_new_sddl" - "[event_data][ObjectName]" => "object_name" - "[event_data][ObjectServer]" => "object_server" - "[event_data][ObjectType]" => "object_type" - "[event_data][OldSd]" => "object_old_sddl" - } - } - } - if [event_id] == 4672 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4672.md - mutate { - rename => { - "[event_data][PrivilegeList]" => "logon_privileges_assigned" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - } - } - } - if [event_id] == 4673 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4673.md - mutate { - rename => { - "[event_data][PrivilegeList]" => "service_privilege_list" - "[event_data][ObjectServer]" => "object_server" - "[event_data][Service]" => "service_name" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - } - } - } - if [event_id] == 4674 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4674.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][AccessMask]" => "object_access_mask_requested" - "[event_data][HandleId]" => "object_access_handle_id" - "[event_data][ObjectName]" => "object_name" - "[event_data][ObjectServer]" => "object_server" - "[event_data][ObjectType]" => "object_type" - "[event_data][PrivilegeList]" => "object_privilege_list" - } - } - } - if [event_id] == 4688 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4688.md - grok { - match => { "[event_data][NewProcessName]" => ".*\\%{GREEDYDATA:process_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - grok { - match => { "[event_data][ParentProcessName]" => ".*\\%{GREEDYDATA:process_parent_name}" } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - mutate { - rename => { - "[event_data][NewProcessId]" => "process_id" - "[event_data][NewProcessName]" => "process_path" - "[event_data][CommandLine]" => "command_line" - "[event_data][ParentProcessName]" => "process_parent_path" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "user_principal_domain" - "[event_data][TargetUserSid]" => "user_principal_sid" - "[event_data][TargetUserName]" => "user_principal_name" - "[event_data][TargetLogonId]" => "user_principal_id" - "[event_data][MandatoryLabel]" => "process_mandatory_level" - "[event_data][ProcessId]" => "process_parent_id" - "[event_data][TokenElevationType]" => "process_token_elevation_type" - } - } - } - if [event_id] == 4689 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4689.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][Status]" => "process_status" - } - } - } - if [event_id] == 4690 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4690.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][SourceHandleId]" => "process_handle_id" - "[event_data][SourceProcessId]" => "process_id" - "[event_data][TargetHandleId]" => "target_process_handle_id" - "[event_data][TargetProcessId]" => "target_process_id" - } - } - } - if [event_id] == 4697 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4697.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ServiceAccount]" => "service_account_name" - "[event_data][ServiceFileName]" => "service_image_path" - "[event_data][ServiceName]" => "service_name" - "[event_data][ServiceStartType]" => "service_start_type" - "[event_data][ServiceType]" => "service_type" - } - } - } - if [event_id] == 4701 or [event_id] == 4702 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4701.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4702.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TaskContentNew]" => "task_new_content" - "[event_data][TaskName]" => "task_name" - } - } - } - if [event_id] == 4703 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4703.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "user_target_domain" - "[event_data][TargetLogonId]" => "user_target_logon_id" - "[event_data][TargetUserSid]" => "user_target_sid" - "[event_data][TargetUserName]" => "user_target_name" - "[event_data][DisabledPrivilegeList]" => "user_target_disabled_privilegelist" - "[event_data][EnabledPrivilegeList]" => "user_target_enabled_privilegelist" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - } - } - } - if [event_id] == 4719 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4719.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][AuditPolicyChanges]" => "policy_changes" - "[event_data][CategoryId]" => "policy_category_id" - "[event_data][SubcategoryGuid]" => "policy_subcategory_guid" - "[event_data][SubcategoryId]" => "policy_subcategory_id" - } - } - } - if [event_id] == 4724 or [event_id] == 4725 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4724.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4725.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "user_target_domain" - "[event_data][TargetSid]" => "user_target_sid" - "[event_data][TargetUserName]" => "user_target_name" - } - } - } - if [event_id] == 4726 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4726.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "user_target_domain" - "[event_data][TargetSid]" => "user_target_sid" - "[event_data][TargetUserName]" => "user_target_name" - "[event_data][PrivilegeList]" => "user_privilege_list" - } - } - } - if [event_id] == 4728 or [event_id] == 4729 { - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "group_domain" - "[event_data][TargetSid]" => "group_sid" - "[event_data][TargetUserName]" => "group_name" - "[event_data][MemberName]" => "group_member_name" - "[event_data][MemberSid]" => "group_member_sid" - "[event_data][PrivilegeList]" => "group_privilege_list" - } - } - } - if [event_id] == 4732 or [event_id] == 4733 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4732.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4733.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "group_domain" - "[event_data][TargetSid]" => "group_sid" - "[event_data][TargetUserName]" => "group_name" - "[event_data][MemberName]" => "group_member_name" - "[event_data][MemberSid]" => "group_member_sid" - "[event_data][PrivilegeList]" => "group_privilege_list" - } - } - } - if [event_id] == 4738 or [event_id] == 4720 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4738.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4720.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "user_target_domain" - "[event_data][TargetSid]" => "user_target_sid" - "[event_data][TargetUserName]" => "user_target_name" - "[event_data][AccountExpires]" => "user_attribute_account_expires" - "[event_data][AllowedToDelegateTo]" => "user_attribute_allowed_todelegate" - "[event_data][DisplayName]" => "user_attribute_display_name" - "[event_data][Dummy]" => "user_attribute_dummy" - "[event_data][HomeDirectory]" => "user_attribute_home_directory" - "[event_data][HomePath]" => "user_attribute_home_path" - "[event_data][LogonHours]" => "user_attribute_logon_hours" - "[event_data][NewUacValue]" => "user_attribute_new_uacvalue" - "[event_data][OldUacValue]" => "user_attribute_old_uacvalue" - "[event_data][PasswordLastSet]" => "user_attribute_password_lastset" - "[event_data][PrimaryGroupId]" => "[user_attribute_primary_group_id" - "[event_data][PrivilegeList]" => "user_attribute_privilege_list" - "[event_data][ProfilePath]" => "user_attribute_profile_path" - "[event_data][SamAccountName]" => "user_attribute_samaccount_name" - "[event_data][ScriptPath]" => "user_attribute_script_path" - "[event_data][SidHistory]" => "user_attribute_sid_history" - } - } - } - if [event_id] == 4768 or [event_id] == 4769 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4768.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4769.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4771.md - mutate { - rename => { - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetUserName]" => "user_name" - "[event_data][IpAddress]" => "src_ip" - "[event_data][IpPort]" => "src_port" - "[event_data][PreAuthType]" => "service_ticket_preauthtype" - "[event_data][LogonGuid]" => "user_logon_guid" - "[event_data][ServiceName]" => "service_ticket_name" - "[event_data][ServiceSid]" => "service_ticket_id" - "[event_data][Status]" => "service_ticket_status" - "[event_data][TicketEncryptionType]" => "ticket_encryption_type" - "[event_data][TicketOptions]" => "ticket_options" - "[event_data][FailureCode]" => "ticket_failure_code" - "[event_data][TransmittedServices]" => "service_ticket_requested" - "[event_data][TargetSid]" => "user_sid" - } - } - } - if [event_id] == 4797 { - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "service_enumerated_domain" - "[event_data][TargetUserName]" => "service_enumerated_name" - "[event_data][Workstation]" => "host_name" - } - } - } - if [event_id] == 4798 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4798.md - mutate { - rename => { - "[event_data][CallerProcessId]" => "process_id" - "[event_data][CallerProcessName]" => "process_path" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "user_domain_enumerated" - "[event_data][TargetSid]" => "user_sid_enumerated" - "[event_data][TargetUserName]" => "user_name_enumerated" - } - } - } - if [event_id] == 4799 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4799.md - mutate { - rename => { - "[event_data][CallerProcessId]" => "process_id" - "[event_data][CallerProcessName]" => "process_path" - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][TargetDomainName]" => "group_domain_enumerated" - "[event_data][TargetSid]" => "group_sid_enumerated" - "[event_data][TargetUserName]" => "group_name_enumerated" - } - } - } - if [event_id] == 4800 or [event_id] == 4801 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4800.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4801.md - mutate { - rename => { - - "[event_data][TargetDomainName]" => "user_domain" - "[event_data][TargetUserSid]" => "user_sid" - "[event_data][TargetUserName]" => "user_name" - "[event_data][TargetLogonId]" => "user_logon_id" - "[event_data][SessionId]" => "user_session_id" - } - } - } - if [event_id] == 4907 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4907.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProcessName]" => "process_path" - "[event_data][HandleId]" => "object_access_handle_id" - "[event_data][NewSd]" => "object_new_sddl" - "[event_data][ObjectName]" => "object_name" - "[event_data][ObjectServer]" => "object_server" - "[event_data][ObjectType]" => "object_type" - "[event_data][OldSd]" => "object_old_sddl" - } - } - } - if [event_id] == 4957 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-4957.md - mutate { - rename => { - "[event_data][RuleAttr]" => "firewall_rule_attr" - "[event_data][RuleId]" => "firewall_rule_id" - "[event_data][RuleName]" => "firewall_rule_name" - } - } - } - if [event_id] == 5058 or [event_id] == 5059 or [event_id] == 5061 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5058.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5059.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5061.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][AlgorithmName]" => "key_algorithm_name" - "[event_data][KeyName]" => "key_name" - "[event_data][KeyType]" => "key_type" - "[event_data][Operation]" => "key_operation" - "[event_data][ProviderName" => "key_provider_name" - "[event_data][ReturnCode]" => "key_return_code" - } - } - } - if [event_id] == 5136 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5136.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][OpCorrelationID]" => "dsoperation_correlation_id" - "[event_data][AppCorrelationID]" => "dsoperation_app_correlation_id" - "[event_data][DSName]" => "dsobject_domain" - "[event_data][DSType]" => "dsobject_domain_type" - "[event_data][ObjectDN]" => "dsobject_dn" - "[event_data][ObjectGUID]" => "dsobject_guid" - "[event_data][ObjectClass]" => "dsobject_class" - "[event_data][AttributeLDAPDisplayName]" => "dsobject_attribute_name" - "[event_data][AttributeSyntaxOID]" => "dsobject_attribute_type" - "[event_data][AttributeValue]" => "dsobject_attribute_value" - "[event_data][OperationType]" => "dsoperation_type" - } - } - } - if [event_id] == 5140 or [event_id] == 5145 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5140.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5145.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][AccessList]" => "object_access_list_requested" - "[event_data][AccessMask]" => "object_access_mask_requested" - "[event_data][AccessReason]" => "user_access_reason" - "[event_data][IpAddress]" => "src_ip" - "[event_data][IpPort]" => "src_port" - "[event_data][ObjectType]" => "object_type" - "[event_data][RelativeTargetName]" => "share_relative_target_name" - "[event_data][ShareLocalPath]" => "share_local_path" - "[event_data][ShareName]" => "share_name" - } - } - } - if [event_id] == 5152 or [event_id] == 5154 or [event_id] == 5156 or [event_id] == 5158 or [event_id] == 5157 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5152.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5154.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5156.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5157.md - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5158.md - grok { - match => { - "[event_data][Application]" => ".*\\%{GREEDYDATA:process_name}" - } - tag_on_failure => [ "_grokparsefailure", "_parsefailure" ] - } - mutate { - rename => { - "[event_data][Application]" => "process_path" - "[event_data][DestAddress]" => "dst_ip" - "[event_data][DestPort]" => "dst_port" - "[event_data][Direction]" => "network_direction" - "[event_data][FilterRTID]" => "network_filter_rtid" - "[event_data][LayerName]" => "network_layer_name" - "[event_data][LayerRTID]" => "network_layer_rtid" - "[event_data][ProcessID]" => "process_id" - "[event_data][Protocol]" => "network_protocol" - "[event_data][RemoteMachineID]" => "dst_host_name_id" - "[event_data][RemoteUserID]" => "dst_user_id" - "[event_data][SourceAddress]" => "src_ip" - "[event_data][SourcePort]" => "src_port" - "[event_data][ProcessId]" => "process_id" - } - } - } - if [event_id] == 5447 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-5447.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][AccessList]" => "object_access_list_requested" - "[event_data][AccessMask]" => "object_access_mask_requested" - "[event_data][AccessReason]" => "user_access_reason" - "[event_data][IpAddress]" => "src_ip" - "[event_data][IpPort]" => "src_port" - "[event_data][ObjectType]" => "object_type" - "[event_data][RelativeTargetName]" => "share_relative_target_name" - "[event_data][ShareLocalPath]" => "share_local_path" - "[event_data][ShareName]" => "share_name" - "[event_data][Action]" => "filtering_action" - "[event_data][CalloutKey]" => "filtering_callout_key" - "[event_data][CalloutName]" => "filtering_callout_name" - "[event_data][ChangeType]" => "filtering_change_type" - "[event_data][Conditions]" => "filtering_conditions" - "[event_data][FilterId]" => "filtering_id" - "[event_data][FilterKey]" => "filtering_key" - "[event_data][FilterName]" => "filtering_name" - "[event_data][FilterType]" => "filtering_type" - "[event_data][LayerId]" => "filtering_layer_id" - "[event_data][LayerKey]" => "filtering_layer_key" - "[event_data][LayerName]" => "filtering_layer_name" - "[event_data][ProcessId]" => "process_id" - "[event_data][ProviderKey]" => "filtering_provider_key" - "[event_data][ProviderName" => "filtering_provider_name" - "[event_data][UserName]" => "user_name" - "[event_data][UserSid]" => "user_sid" - "[event_data][Weight]" => "filtering_weight" - } - } - } - if [event_id] == 6416 { - # https://github.com/MicrosoftDocs/windows-itpro-docs/blob/master/windows/security/threat-protection/auditing/event-6416.md - mutate { - rename => { - "[event_data][SubjectDomainName]" => "user_domain" - "[event_data][SubjectLogonId]" => "user_logon_id" - "[event_data][SubjectUserName]" => "user_name" - "[event_data][SubjectUserSid]" => "user_sid" - "[event_data][ClassId]" => "device_class_id" - "[event_data][ClassName]" => "device_class_name" - "[event_data][CompatibleIds]" => "device_compatible_ids" - "[event_data][DeviceDescription]" => "device_description" - "[event_data][DeviceId]" => "device_id" - "[event_data][LocationInformation]" => "device_location_information" - "[event_data][VendorIds]" => "device_vendor_ids" - } - } - } - mutate { - convert => { - "src_port" => "integer" - "dst_port" => "integer" - } - rename => { - "computer_name" => "host_name" - } - gsub => ["user_logon_guid","[{}]",""] - gsub => ["target_user_logon_guid","[{}]",""] - } - if [user_logon_id] { - mutate { gsub => [ "user_logon_id", "0x", "" ]} - ruby { - code => "event.set('user_logon_id', event.get('user_logon_id').to_s.hex)" - tag_on_exception => "_0591_rubyexception" - } - } - if [process_id] { - mutate { gsub => [ "process_id", "0x", "" ]} - ruby { - code => "event.set('process_id', event.get('process_id').to_s.hex)" - tag_on_exception => "_0591_rubyexception" - } - } - if [process_parent_id] { - mutate { gsub => [ "process_parent_id", "0x", "" ]} - ruby { - code => "event.set('process_parent_id', event.get('process_parent_id').to_s.hex)" - tag_on_exception => "_0591_rubyexception" - } - } - if [target_process_id] { - mutate { gsub => [ "target_process_id", "0x", "" ]} - ruby { - code => "event.set('target_process_id', event.get('target_process_id').to_s.hex)" - tag_on_exception => "_0591_rubyexception" - } - } - } -} - diff --git a/helk-nginx/trial/default b/helk-nginx/trial/default deleted file mode 100644 index fc65d080..00000000 --- a/helk-nginx/trial/default +++ /dev/null @@ -1,18 +0,0 @@ -server { - proxy_connect_timeout 900; - proxy_send_timeout 600; - proxy_read_timeout 600; - - listen 80; - - server_name 127.0.0.1; - - location / { - proxy_pass http://helk-kibana:5601; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - } -} diff --git a/helk-spark-master/scripts/spark-master-entrypoint.sh b/helk-spark-master/scripts/spark-master-entrypoint.sh deleted file mode 100755 index 7ff33ec0..00000000 --- a/helk-spark-master/scripts/spark-master-entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -# HELK script: spark-master-entrypoint.sh -# HELK script description: Starts Spark Master Service -# HELK build Stage: Alpha -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -ln -sf /dev/stdout $SPARK_LOGS/spark-master.out - -echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Spark Master Service.." -exec $SPARK_HOME/bin/spark-class org.apache.spark.deploy.master.Master \ - --host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_LOGS/spark-master.out \ No newline at end of file diff --git a/helk-spark-worker/scripts/spark-worker-entrypoint.sh b/helk-spark-worker/scripts/spark-worker-entrypoint.sh deleted file mode 100755 index a8a3740c..00000000 --- a/helk-spark-worker/scripts/spark-worker-entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -# HELK script: spark-worker-entrypoint.sh -# HELK script description: Starts Spark Worker Service -# HELK build Stage: Alpha -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -ln -sf /dev/stdout $SPARK_LOGS/spark-worker.out - -echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Spark Worker Service.." -exec /$SPARK_HOME/bin/spark-class org.apache.spark.deploy.worker.Worker \ - --webui-port $SPARK_WORKER_WEBUI_PORT $SPARK_MASTER >> $SPARK_LOGS/spark-worker.out \ No newline at end of file diff --git a/resources/images/JUPYTERHUB-Login.png b/resources/images/JUPYTERHUB-Login.png new file mode 100644 index 00000000..ca7e85fa Binary files /dev/null and b/resources/images/JUPYTERHUB-Login.png differ diff --git a/resources/images/JUPYTERHUB-Pyspark-Kernel.png b/resources/images/JUPYTERHUB-Pyspark-Kernel.png new file mode 100644 index 00000000..39b6a1f9 Binary files /dev/null and b/resources/images/JUPYTERHUB-Pyspark-Kernel.png differ diff --git a/resources/images/JUPYTERHUB-SparkContext.png b/resources/images/JUPYTERHUB-SparkContext.png new file mode 100644 index 00000000..311dad06 Binary files /dev/null and b/resources/images/JUPYTERHUB-SparkContext.png differ diff --git a/resources/images/JUPYTERHUB-basic-trial.png b/resources/images/JUPYTERHUB-basic-trial.png new file mode 100644 index 00000000..5a10d03f Binary files /dev/null and b/resources/images/JUPYTERHUB-basic-trial.png differ diff --git a/resources/images/JUPYTERHUB-hunter2.png b/resources/images/JUPYTERHUB-hunter2.png new file mode 100644 index 00000000..65f03583 Binary files /dev/null and b/resources/images/JUPYTERHUB-hunter2.png differ