diff --git a/README.md b/README.md index f1891408..c268d36d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ It also allows system administrators to: ## Installation -- The MHN server is supported on Ubuntu 18.04, Ubuntu 16.04, and Centos 6.9. +- The MHN server is supported on Ubuntu 18.04, Ubuntu 16.04, Centos 6.9 and Centos 7. - Other versions of Linux may work but are generally not tested or supported. Note: if you run into trouble during the install, please checkout the [troubleshooting guide](https://github.com/Pwnlandia/MHN/wiki/MHN-Troubleshooting-Guide) on the wiki. If you only want to experiment with MHN on some virtual machines, please check out the [Getting up and Running with Vagrant](https://github.com/Pwnlandia/mhn/wiki/Getting-up-and-running-using-Vagrant) guide on the wiki. diff --git a/install.sh b/install.sh index 14914132..ea3e313e 100755 --- a/install.sh +++ b/install.sh @@ -67,72 +67,74 @@ echo "[`date`] ========= Installing MHN Server =========" echo "[`date`] ========= MHN Server Install Finished =========" echo "" -while true; -do - echo -n "Would you like to integrate with Splunk? (y/n) " - read SPLUNK - if [ "$SPLUNK" == "y" -o "$SPLUNK" == "Y" ] - then - echo -n "Splunk Forwarder Host: " - read SPLUNK_HOST - echo -n "Splunk Forwarder Port: " - read SPLUNK_PORT - echo "The Splunk Universal Forwarder will send all MHN logs to $SPLUNK_HOST:$SPLUNK_PORT" - ./install_splunk_universalforwarder.sh "$SPLUNK_HOST" "$SPLUNK_PORT" - ./install_hpfeeds-logger-splunk.sh - break - elif [ "$SPLUNK" == "n" -o "$SPLUNK" == "N" ] - then - echo "Skipping Splunk integration" - echo "The splunk integration can be completed at a later time by running this:" - echo " cd /opt/mhn/scripts/" - echo " sudo ./install_splunk_universalforwarder.sh " - echo " sudo ./install_hpfeeds-logger-splunk.sh" - break - fi -done - - -while true; -do - echo -n "ELK Script will only work on Debian Based systems like Ubuntu" - echo -n "Would you like to install ELK? (y/n) " - read ELK - if [ "$ELK" == "y" -o "$ELK" == "Y" ] - then - ./install_elk.sh - break - elif [ "$ELK" == "n" -o "$ELK" == "N" ] - then - echo "Skipping ELK installation" - echo "The ELK installation can be completed at a later time by running this:" - echo " cd /opt/mhn/scripts/" - echo " sudo ./install_elk.sh" - break - fi -done - - -while true; -do - echo -n "A properly configured firewall is highly encouraged while running MHN." - echo -n "This script can enable and configure UFW for use with MHN." - echo -n "Would you like to add MHN rules to UFW? (y/n) " - read UFW - if [ "$UFW" == "y" -o "$UFW" == "Y" ] - then - ./enable_ufw.sh - break - elif [ "$UFW" == "n" -o "$UFW" == "N" ] - then - echo "Skipping UFW configuration" - echo "The UFW configuration can be completed at a later time by running this:" - echo " cd /opt/mhn/scripts/" - echo " sudo ./enable_ufw.sh" - break - fi -done +# These scripts only work on debian-like systems +if [ -f /etc/debian_version ]; then + while true; + do + echo -n "Would you like to integrate with Splunk? (y/n) " + read SPLUNK + if [ "$SPLUNK" == "y" -o "$SPLUNK" == "Y" ] + then + echo -n "Splunk Forwarder Host: " + read SPLUNK_HOST + echo -n "Splunk Forwarder Port: " + read SPLUNK_PORT + echo "The Splunk Universal Forwarder will send all MHN logs to $SPLUNK_HOST:$SPLUNK_PORT" + ./install_splunk_universalforwarder.sh "$SPLUNK_HOST" "$SPLUNK_PORT" + ./install_hpfeeds-logger-splunk.sh + break + elif [ "$SPLUNK" == "n" -o "$SPLUNK" == "N" ] + then + echo "Skipping Splunk integration" + echo "The splunk integration can be completed at a later time by running this:" + echo " cd /opt/mhn/scripts/" + echo " sudo ./install_splunk_universalforwarder.sh " + echo " sudo ./install_hpfeeds-logger-splunk.sh" + break + fi + done + + while true; + do + echo -n "ELK Script will only work on Debian Based systems like Ubuntu" + echo -n "Would you like to install ELK? (y/n) " + read ELK + if [ "$ELK" == "y" -o "$ELK" == "Y" ] + then + ./install_elk.sh + break + elif [ "$ELK" == "n" -o "$ELK" == "N" ] + then + echo "Skipping ELK installation" + echo "The ELK installation can be completed at a later time by running this:" + echo " cd /opt/mhn/scripts/" + echo " sudo ./install_elk.sh" + break + fi + done + + + while true; + do + echo -n "A properly configured firewall is highly encouraged while running MHN." + echo -n "This script can enable and configure UFW for use with MHN." + echo -n "Would you like to add MHN rules to UFW? (y/n) " + read UFW + if [ "$UFW" == "y" -o "$UFW" == "Y" ] + then + ./enable_ufw.sh + break + elif [ "$UFW" == "n" -o "$UFW" == "N" ] + then + echo "Skipping UFW configuration" + echo "The UFW configuration can be completed at a later time by running this:" + echo " cd /opt/mhn/scripts/" + echo " sudo ./enable_ufw.sh" + break + fi + done +fi chown $WWW_OWNER /var/log/mhn/mhn.log chown $WWW_OWNER /var/log/mhn/mhn.log diff --git a/scripts/install_elk.sh b/scripts/install_elk.sh index bfd2a1cf..839ed4c1 100755 --- a/scripts/install_elk.sh +++ b/scripts/install_elk.sh @@ -6,55 +6,62 @@ set -e DIR=`dirname "$0"` $DIR/install_hpfeeds-logger-json.sh -# install Java -apt-get install -y python-software-properties -add-apt-repository -y ppa:webupd8team/java -apt-get update -apt-get -y install oracle-java8-installer +###### +### Install ELK (https://www.elastic.co) +# +# Make sure the system has enought RAM (2GB was not enought for basic stuff) and disk space, otherwise ES can suddently stop. +# Recommended: 4GB RAM, 15 GB Disk. +# +# Known Issue: ES can fail to start after booting, no idea why. Restart the service with sudo systemctl restart elasticsearch.service +# +### ElasticSearch - https://www.elastic.co/guide/en/elasticsearch/reference/7.5/deb.html#deb-repo +# +# Runs on localhost:9200. Config file: /etc/elasticsearch/elasticsearch.yml +# Status: sudo systemctl status elasticsearch.service +# If exposed to the internet (not recommended), make sure to add FW rules to only allow trusted sources +# +### Kibana - https://www.elastic.co/guide/en/kibana/7.5/deb.html#deb-repo +## https://www.elastic.co/guide/en/kibana/current/access.html +## https://www.elastic.co/guide/en/kibana/current/kibana-authentication.html +# +# Runs on localhost:5601. Config file: /etc/kibana/kibana.yml +# Status: sudo systemctl status kibana.service +# +### Logstash - https://www.elastic.co/guide/en/logstash/7.5/installing-logstash.html#_apt +# +# Runs on localhost:9600-9700. Config file /etc/logstash/logstash.yml & /etc/logstash/conf.d/logstash.conf +# Status: sudo systemctl status logstash.service +# List plugins: /usr/share/logstash/bin/logstash-plugin list +# +###### + +# Install Java (required by ES) +apt update +apt install -y software-properties-common openjdk-8-jdk # Install ES -wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - -echo 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' | tee /etc/apt/sources.list.d/elasticsearch.list -apt-get update -apt-get -y install elasticsearch=1.4.4 -sed -i '/network.host/c\network.host\:\ localhost' /etc/elasticsearch/elasticsearch.yml -service elasticsearch restart -update-rc.d elasticsearch defaults 95 10 +apt install -y apt-transport-https +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - +echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-7.x.list +apt update +apt install -y elasticsearch +systemctl enable elasticsearch.service +systemctl start elasticsearch.service # Install Kibana -mkdir /tmp/kibana -cd /tmp/kibana ; -wget https://download.elasticsearch.org/kibana/kibana/kibana-4.0.1-linux-x64.tar.gz -tar xvf kibana-4.0.1-linux-x64.tar.gz -sed -i '/0.0.0.0/c\host\:\ localhost' /etc/elasticsearch/elasticsearch.yml -mkdir -p /opt/kibana -cp -R /tmp/kibana/kibana-4*/* /opt/kibana/ -rm -rf /tmp/kibana/kibana-4* - -cat > /etc/supervisor/conf.d/kibana.conf < /opt/logstash/mhn.conf < /etc/logstash/conf.d/mhn.conf < "/var/log/mhn/mhn-json.log" - start_position => "end" } } @@ -64,391 +71,203 @@ filter { } geoip { - source => "src_ip" - target => "src_ip_geo" - database => "/opt/GeoLite2-City.mmdb" - add_field => [ "[src_ip_geo][coordinates]", "%{[src_ip_geo][longitude]}" ] - add_field => [ "[src_ip_geo][coordinates]", "%{[src_ip_geo][latitude]}" ] - } - mutate { - convert => [ "[src_ip_geo][coordinates]", "float"] + source => "src_ip" + target => "src_ip_geo" + database => "/opt/GeoLite2-City.mmdb" } geoip { - source => "dst_ip" - target => "dst_ip_geo" + source => "dest_ip" + target => "dest_ip_geo" database => "/opt/GeoLite2-City.mmdb" - add_field => [ "[dst_ip_geo][coordinates]", "%{[dst_ip_geo][longitude]}" ] - add_field => [ "[dst_ip_geo][coordinates]", "%{[dst_ip_geo][latitude]}" ] } - - mutate { - convert => [ "[dst_ip_geo][coordinates]", "float"] - } } output { elasticsearch { - host => "127.0.0.1" - port => 9200 - protocol => "http" + hosts => ["http://localhost:9200"] index => "mhn-%{+YYYYMMddHH00}" - index_type => "event" template_name => "mhn_event" - template => "/opt/logstash/mhn-template.json" + template => "/etc/logstash/conf.d/mhn-template.json" template_overwrite => true manage_template => true } } - EOF -cat > /opt/logstash/mhn-template.json < /etc/logstash/conf.d/mhn-template.json < /etc/supervisor/conf.d/logstash.conf <> /etc/supervisor/conf.d/hpfeeds-broker.conf < $NGINXCONFIG < /etc/supervisor/conf.d/mhn-uwsgi.conf < /etc/supervisor/conf.d/mhn-uwsgi.conf < /etc/supervisor/conf.d/mhn-celery-worker.conf < /etc/supervisor/conf.d/mhn-celery-worker.conf < /etc/supervisor/conf.d/mhn-celery-beat.conf < /etc/supervisor/conf.d/mhn-celery-beat.conf < $MHN_HOME/server/collector.json < /etc/supervisor/conf.d/mhn-collector.conf < /etc/supervisor/conf.d/mhn-collector.conf < /tmp/nginx.conf + \mv /tmp/nginx.conf /etc/nginx/nginx.conf + systemctl enable nginx + systemctl start nginx +else + /etc/init.d/nginx restart +fi diff --git a/scripts/install_mongo.sh b/scripts/install_mongo.sh index 68c268cc..e53f9aa9 100755 --- a/scripts/install_mongo.sh +++ b/scripts/install_mongo.sh @@ -1,7 +1,7 @@ #!/bin/bash # Install MongoDB for the appropriate OS and version. -# Supports Ubuntu 14, 16, 18 and RHEL/CentOS 6.9 +# Supports Ubuntu 14, 16, 18 and RHEL/CentOS set -e set -x @@ -20,7 +20,7 @@ if [ -f /etc/debian_version ]; then fi elif [ -f /etc/redhat-release ]; then - ./install_mongodb_rhel69.sh + ./install_mongodb_rhel.sh else echo -e "ERROR: Unknown OS\nExiting!" exit -1 diff --git a/scripts/install_mongodb_rhel69.sh b/scripts/install_mongodb_rhel.sh similarity index 57% rename from scripts/install_mongodb_rhel69.sh rename to scripts/install_mongodb_rhel.sh index 42932c29..e1e7f4db 100755 --- a/scripts/install_mongodb_rhel69.sh +++ b/scripts/install_mongodb_rhel.sh @@ -7,10 +7,10 @@ set -x OS=RHEL export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:$PATH -cat >> /etc/yum.repos.d/mongodb.repo < /etc/yum.repos.d/mongodb.repo < /etc/supervisord.conf -cat >> /etc/supervisord.conf <> /etc/supervisord.conf < /usr/lib/systemd/system/supervisord.service <