elasticsearch cloud deploy for 8.x

.gitignore

@@ -7,4 +7,5 @@ terraform.tfvars
 .gcp*
 cluster_bootstrap_state
 gcp-account.json
-*.iml
+*.iml
+id_rsa.pub

assets/scripts/bootstrap.sh

@@ -10,9 +10,7 @@ set +e
 /opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh
 
 cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
-node.master: true
-node.data: false
-node.ingest: false
+node.roles: [ master ]
 EOF
 
 # add bootstrap.password to the keystore, so that config-cluster scripts can run
@@ -26,20 +24,14 @@ systemctl daemon-reload
 systemctl enable elasticsearch.service
 systemctl start elasticsearch.service
 
+set -e
 /opt/cloud-deploy-scripts/common/config-cluster.sh
-/opt/cloud-deploy-scripts/$cloud_provider/config-cluster.sh
+set +e
 
-while true
-do
-    HEALTH="$(curl $CURL_AUTH --silent -k "$ES_HOST/_cluster/health" | jq -r '.status')"
-    if [ "$HEALTH" == "green" ]; then
-        break
-    fi
-    sleep 5
-done
+/opt/cloud-deploy-scripts/$cloud_provider/config-cluster.sh
 
 if [ "$cloud_provider" == "aws" ]; then
 	shutdown -h now
 elif [ "$cloud_provider" == "gcp" ]; then
 	gcloud compute instances delete $HOSTNAME --zone $GCP_ZONE --quiet
-fi
+fi

assets/scripts/client.sh

@@ -15,14 +15,12 @@ fi
 /opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh
 
 cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
-node.master: false
-node.data: false
-node.ingest: false
+node.roles: []
 EOF
 
 # Start Elasticsearch
 systemctl daemon-reload
 systemctl enable elasticsearch.service
 systemctl start elasticsearch.service
 
-/opt/cloud-deploy-scripts/common/config-clients.sh
+/opt/cloud-deploy-scripts/common/config-clients.sh

assets/scripts/common/config-clients.sh

@@ -6,63 +6,9 @@
 # - BIND_TO_ALL
 # - ES_HOST
 # - CURL_AUTH
-
-
-function setup_grafana_dashboard() {
-    GRAFANA_BASIC_AUTH=""
-    if [ "$security_enabled" == "true" ]; then
-        GRAFANA_BASIC_AUTH=" --user $client_user:$client_pwd "
-    fi
-
-    while true; do
-        echo "Waiting for grafana to become available..."
-        if curl $GRAFANA_BASIC_AUTH --output /dev/null --fail http://localhost:3000; then break; fi
-        sleep 5
-    done
-
-    cat <<EOF >>/tmp/grafana-datasource.json
-{
-    "name": "Elasticsearch Monitor",
-    "type": "elasticsearch",
-    "typeLogoUrl": "public/app/plugins/datasource/elasticsearch/img/elasticsearch.svg",
-    "access": "proxy",
-    "url": "$ES_HOST",
-    "password": "",
-    "user": "",
-    "database": "[.monitoring-es-*-]YYYY.MM.DD",
-    "isDefault": true,
-    "jsonData": {
-      "esVersion": 70,
-      "interval": "Daily",
-      "logLevelField": "",
-      "logMessageField": "",
-      "maxConcurrentShardRequests": 5,
-      "timeField": "timestamp"
-    },
-    "readOnly": false,
-EOF
-
-    if [ "$security_enabled" == "true" ]; then
-        cat <<EOF >>/tmp/grafana-datasource.json
-    "basicAuth": true,
-    "basicAuthUser": "$client_user",
-    "secureJsonData": { "basicAuthPassword": "$client_pwd" }
-}
-EOF
-    else
-        echo '"basicAuth": false }' >> /tmp/grafana-datasource.json;
-    fi
-
-    curl $GRAFANA_BASIC_AUTH -XPOST -H 'Content-Type: application/json' localhost:3000/api/datasources -d @/tmp/grafana-datasource.json
-    rm /tmp/grafana-datasource.json
-
-    if [ -f /opt/grafana-dashboard.json ]; then
-        echo '{ "meta": {"isStarred": true}, "dashboard":' > /tmp/grafana-dashboard.json
-        cat /opt/grafana-dashboard.json | jq -r 'del(.uid) | del(.id)' >> /tmp/grafana-dashboard.json
-        echo '}' >> /tmp/grafana-dashboard.json
-        curl $GRAFANA_BASIC_AUTH -XPOST -H 'Content-Type: application/json' localhost:3000/api/dashboards/db -d @/tmp/grafana-dashboard.json
-    fi
-}
+# security_encryption_key
+# reporting_encryption_key
+# saved_objects_encryption_key
 
 # Setup x-pack security also on Kibana configs where applicable
 if [ -f "/etc/kibana/kibana.yml" ]; then
@@ -72,51 +18,26 @@ if [ -f "/etc/kibana/kibana.yml" ]; then
     else
         echo "server.host: $(hostname -i)" | sudo tee -a /etc/kibana/kibana.yml
     fi
+    echo "monitoring.enabled: $monitoring_enabled" | sudo tee -a /etc/kibana/kibana.yml
+    echo "monitoring.kibana.collection.enabled: $monitoring_enabled" | sudo tee -a /etc/kibana/kibana.yml
+
+    if [ ! -z "$security_encryption_key" ]; then
+        echo "$security_encryption_key" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.security.encryptionKey
+    fi
+    if [ ! -z "$reporting_encryption_key" ]; then
+        echo "$reporting_encryption_key" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.reporting.encryptionKey
+    fi
+    if [ ! -z "$saved_objects_encryption_key" ]; then
+        echo "$saved_objects_encryption_key" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.encryptedSavedObjects.encryptionKey
+    fi
 
-    echo "xpack.security.enabled: $security_enabled" | sudo tee -a /etc/kibana/kibana.yml
-    echo "xpack.monitoring.enabled: $monitoring_enabled" | sudo tee -a /etc/kibana/kibana.yml
 
     if [ "$security_enabled" == "true" ]; then
         echo "elasticsearch.username: \"kibana\"" | sudo tee -a /etc/kibana/kibana.yml
-        echo "elasticsearch.password: \"$client_pwd\"" | sudo tee -a /etc/kibana/kibana.yml
+        echo "${client_pwd}" | /usr/share/kibana/bin/kibana-keystore add --stdin elasticsearch.password
     fi
 
     systemctl daemon-reload
     systemctl enable kibana.service
     sudo service kibana restart
 fi
-
-if [ -f "/etc/grafana/grafana.ini" ]; then
-    sudo rm /etc/grafana/grafana.ini
-
-    if [ "$security_enabled" == "true" ]; then
-        cat <<EOF >>/etc/grafana/grafana.ini
-[security]
-admin_user = $client_user
-admin_password = $client_pwd
-EOF
-    else
-        cat <<EOF >>/etc/grafana/grafana.ini
-[auth.anonymous]
-enabled = true
-org_name = Main Org.
-org_role = Admin
-EOF
-    fi
-
-    sudo /bin/systemctl daemon-reload
-    sudo /bin/systemctl enable grafana-server.service
-    sudo service grafana-server start
-
-    setup_grafana_dashboard;
-fi
-
-if [ -d "/usr/share/cerebro/" ]; then
-    CEREBRO_CONFIG_PATH="$(echo /usr/share/cerebro/cerebro*/conf/application.conf)"
-    if [ "$security_enabled" == "true" ]; then
-        sudo sed -i "s/.{?BASIC_AUTH_USER}/$client_user/ig" $CEREBRO_CONFIG_PATH
-        sudo sed -i "s/.{?BASIC_AUTH_PWD}/$client_pwd/ig" $CEREBRO_CONFIG_PATH
-        sudo sed -i 's/.{?AUTH_TYPE}/"basic"/ig' $CEREBRO_CONFIG_PATH
-    fi
-    sudo systemctl restart cerebro        
-fi

assets/scripts/common/config-cluster.sh

@@ -3,34 +3,37 @@
 # - client_pwd
 # - ES_HOST
 # - CURL_AUTH
-
+i=1
 while true
 do
-    echo "Checking cluster health"
+    echo "Checking cluster health, attempt $i"
     HEALTH="$(curl $CURL_AUTH --silent -k "$ES_HOST/_cluster/health" | jq -r '.status')"
-    DATA_NODE_COUNT="$(curl $CURL_AUTH --silent -k "$ES_HOST/_cat/nodes?h=node.role" | grep 'd' | wc -l)"
+    DATA_NODE_COUNT="$(curl $CURL_AUTH --silent -k "$ES_HOST/_cat/nodes?h=node.role" | grep 'd\|h\|c' | wc -l)"
 
     if [ "$HEALTH" == "green" ] && [ "$DATA_NODE_COUNT" != "0" ]; then
         break
     fi
 
     sleep 5
+    i=$((i+1))
 done
 
+# if any of the below fail, bootstrap failed - exit on error
+set -e
 if [ "$security_enabled" == "true" ]; then
-    curl $CURL_AUTH \
-         -X PUT -H 'Content-Type: application/json' -k \
-         "$ES_HOST/_xpack/security/user/kibana/_password" -d '{ "password": "'"$client_pwd"'" }'
+  curl $CURL_AUTH \
+       -X PUT -H 'Content-Type: application/json' -k \
+       "$ES_HOST/_security/user/elastic/_password" -d '{ "password": "'"$client_pwd"'" }'
 
-    curl $CURL_AUTH \
-         -X PUT -H 'Content-Type: application/json' -k \
-         "$ES_HOST/_xpack/security/user/logstash_system/_password" -d '{ "password": "'"$client_pwd"'" }'
+  curl $CURL_AUTH \
+       -X PUT -H 'Content-Type: application/json' -k \
+       "$ES_HOST/_security/user/kibana/_password" -d '{ "password": "'"$client_pwd"'" }'
 
-    curl $CURL_AUTH \
-         -X PUT -H 'Content-Type: application/json' -k \
-         "$ES_HOST/_xpack/security/user/elastic/_password" -d '{ "password": "'"$client_pwd"'" }'
+  curl $CURL_AUTH \
+       -X PUT -H 'Content-Type: application/json' -k \
+       "$ES_HOST/_security/user/logstash_system/_password" -d '{ "password": "'"$client_pwd"'" }'
 
-    curl $CURL_AUTH \
-         -X PUT -H 'Content-Type: application/json' -k \
-         "$ES_HOST/_xpack/security/user/remote_monitoring_user/_password" -d '{ "password": "'"$client_pwd"'" }'
-fi
+  curl $CURL_AUTH \
+       -X PUT -H 'Content-Type: application/json' -k \
+       "$ES_HOST/_security/user/remote_monitoring_user/_password" -d '{ "password": "'"$client_pwd"'" }'  
+fi

assets/scripts/common/config-es.sh

@@ -14,7 +14,6 @@
 # Configure elasticsearch
 cat <<EOF >>/etc/elasticsearch/elasticsearch.yml
 cluster.name: $es_cluster
-xpack.monitoring.enabled: $monitoring_enabled
 xpack.monitoring.collection.enabled: $monitoring_enabled
 path.data: $elasticsearch_data_dir
 path.logs: $elasticsearch_logs_dir
@@ -33,13 +32,13 @@ if [ "$security_enabled" == "true" ]; then
     echo -n "$ca_cert" > /etc/elasticsearch/config/certs/ca.crt
     echo -n "$node_cert" > /etc/elasticsearch/config/certs/tls.crt
     echo -n "$node_key" > /etc/elasticsearch/config/certs/tls.key
-
     cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
 xpack.security.transport.ssl.enabled: true
 xpack.security.transport.ssl.verification_mode: "certificate"
 xpack.security.transport.ssl.key: "/etc/elasticsearch/config/certs/tls.key"
 xpack.security.transport.ssl.certificate: "/etc/elasticsearch/config/certs/tls.crt"
 xpack.security.transport.ssl.certificate_authorities: "/etc/elasticsearch/config/certs/ca.crt"
+
 EOF
 fi
 
@@ -51,6 +50,12 @@ xpack.monitoring.exporters.xpack_remote:
 EOF
 fi
 
+# Disable HTTP SSL. Configurations may vary for HTTP SSL - see here.
+#https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#security-http-tls-ssl-key-trusted-certificate-settings
+# When not using it, we need to outright disable it for the cluster to start.
+cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
+xpack.security.http.ssl.enabled: false
+EOF
 
 cat <<'EOF' >>/etc/security/limits.conf
 
@@ -68,7 +73,6 @@ RestartSec=10
 EOF
 
 # Setup heap size and memory locking
-sudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/init.d/elasticsearch
 sudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/default/elasticsearch
 
 # Set java heap size
@@ -101,6 +105,9 @@ if [ "$use_g1gc" = "true" ]; then
   sudo sed -i 's/[0-9]\+-:-XX:InitiatingHeapOccupancyPercent/10-:-XX:InitiatingHeapOccupancyPercent/ig' /etc/elasticsearch/jvm.options
 fi
 
+# Disable heap dumps
+echo "-XX:-HeapDumpOnOutOfMemoryError" | sudo tee -a /etc/elasticsearch/jvm.options
+
 # Create log and data dirs
 sudo mkdir -p $elasticsearch_logs_dir
 sudo mkdir -p $elasticsearch_data_dir

assets/scripts/data.sh

@@ -14,13 +14,11 @@ set +e
 if [ "$is_voting_only" == "true" ]
 then
   cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
-node.roles: [ master, data, voting_only, ingest ]
+node.roles: [ data_hot, data_content, ingest, transform, master, voting_only ]
 EOF
 else
   cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
-node.master: false
-node.data: true
-node.ingest: true
+node.roles: [ data_hot, data_content, ingest, transform ]
 EOF
 fi
 

assets/scripts/gcp/autoattach-disk.sh

@@ -6,7 +6,7 @@
 while true; do
     INSTANCE_ROLE="$(gcloud compute instances describe $HOSTNAME --zone $GCP_ZONE --format json | jq -r ".labels.role")"
     echo "INSTANCE_ROLE: $INSTANCE_ROLE"
-    UNATTACHED_VOLUME_ID="$(gcloud compute disks list --filter="zone:$GCP_ZONE AND labels.cluster-name:$es_cluster AND labels.auto-attach-group:$INSTANCE_ROLE" --format json | jq  -r '.[] | .name' | shuf -n 1)"
+    UNATTACHED_VOLUME_ID="$(gcloud compute disks list --filter="zone=$GCP_ZONE AND labels.cluster-name=$es_cluster AND labels.auto-attach-group=$INSTANCE_ROLE" --format json | jq  -r '.[] | .name' | shuf -n 1)"
     echo "UNATTACHED_VOLUME_ID: $UNATTACHED_VOLUME_ID"
 
     gcloud compute instances attach-disk $HOSTNAME --disk $UNATTACHED_VOLUME_ID --device-name "espersistent" --zone $GCP_ZONE

assets/scripts/gcp/config-bootstrap-node.sh

@@ -1,24 +1,25 @@
 # Required variables
 # - es_environment
 # - masters_count
-
+i=1
 while true
 do
     echo "Fetching masters..."
 
-    MASTER_INSTANCES="$(gcloud compute instances list --filter="labels.cluster:$es_environment AND labels.role:(master OR data-voters)" --format 'get(networkInterfaces[0].networkIP)' | sort)"
+    MASTER_INSTANCES="$(gcloud compute instances list --filter="labels.cluster=$es_environment AND labels.role=(master OR data-voters)" --format 'get(networkInterfaces[0].networkIP)' | sort)"
     COUNT=`echo "$MASTER_INSTANCES" | wc -l`
-    echo "Found $COUNT instances, expecting $masters_count"
+    echo "Found $COUNT instances, expecting $masters_count, attempt $i"
     if [ "$COUNT" -eq "$masters_count" ]; then
         echo "Masters count is correct... Rechecking in 60 sec"
         sleep 60
-        MASTER_INSTANCES_RECHECK="$(gcloud compute instances list --filter="labels.cluster:$es_environment AND labels.role:(master OR data-voters)" --format 'get(networkInterfaces[0].networkIP)' | sort)"
+        MASTER_INSTANCES_RECHECK="$(gcloud compute instances list --filter="labels.cluster=$es_environment AND labels.role=(master OR data-voters)" --format 'get(networkInterfaces[0].networkIP)' | sort)"
         if [ "$MASTER_INSTANCES" = "$MASTER_INSTANCES_RECHECK" ]; then
             break
         fi
     fi
 
     sleep 5
+    i=$((i+1))
 done
 
 echo "Fetched masters"

assets/scripts/gcp/config-es-discovery.sh

@@ -9,7 +9,8 @@ cloud.gce.project_id: ${gcp_project_id}
 cloud.gce.zone: ${gcp_zones}
 discovery.seed_providers: gce
 EOF
-
+# It is required to bind to all interfaces for discovery on GCP to work
+# echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml
 if [ "$BIND_TO_ALL" == "true" ]; then
 	echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml
 else

assets/scripts/master.sh

@@ -12,12 +12,10 @@ set +e
 /opt/cloud-deploy-scripts/$cloud_provider/config-es-discovery.sh
 
 cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
-node.master: true
-node.data: false
-node.ingest: false
+node.roles: [ master ]
 EOF
 
 # Start Elasticsearch
 systemctl daemon-reload
 systemctl enable elasticsearch.service
-systemctl start elasticsearch.service
+systemctl start elasticsearch.service