elasticsearch cloud deploy for 8.x

assets/scripts/aws/autoattach-disk.sh

@@ -28,7 +28,7 @@ echo 'Waiting for 30 seconds for the disk to become mountable...'
 sleep 30
 
 sudo mkdir -p $elasticsearch_data_dir
-export DEVICE_NAME=$(lsblk -ip | tail -n +2 | awk '{print $1 " " ($7? "MOUNTEDPART" : "") }' | sed ':a;N;$!ba;s/\n`/ /g' | grep -v MOUNTEDPART)
+export DEVICE_NAME=$(lsblk -ip | tail -n +2 | awk '{print $1 " " ($7? "MOUNTEDPART" : "") }' | sed ':a;N;$!ba;s/\n`/ /g' | sed ':a;N;$!ba;s/\n|-/ /g' | grep -v MOUNTEDPART)
 if sudo mount -o defaults -t ext4 $DEVICE_NAME $elasticsearch_data_dir; then
     echo 'Successfully mounted existing disk'
 else
@@ -37,4 +37,4 @@ else
     sudo mount -o defaults -t ext4 $DEVICE_NAME $elasticsearch_data_dir && echo 'Successfully mounted a fresh disk'
 fi
 echo "$DEVICE_NAME $elasticsearch_data_dir ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
-sudo chown -R elasticsearch:elasticsearch $elasticsearch_data_dir
+sudo chown -R elasticsearch:elasticsearch $elasticsearch_data_dir

assets/scripts/bootstrap.sh

@@ -46,4 +46,4 @@ then
   elif [ "$cloud_provider" == "gcp" ]; then
   	gcloud compute instances delete $HOSTNAME --zone $GCP_ZONE --quiet
   fi
-fi
+fi

assets/scripts/common/config-clients.sh

@@ -8,6 +8,7 @@
 # - CURL_AUTH
 # security_encryption_key
 # reporting_encryption_key
+# saved_objects_encryption_key
 
 # Setup x-pack security also on Kibana configs where applicable
 if [ -f "/etc/kibana/kibana.yml" ]; then
@@ -17,14 +18,19 @@ if [ -f "/etc/kibana/kibana.yml" ]; then
     else
         echo "server.host: $(hostname -i)" | sudo tee -a /etc/kibana/kibana.yml
     fi
+    echo "monitoring.enabled: $monitoring_enabled" | sudo tee -a /etc/kibana/kibana.yml
+    echo "monitoring.kibana.collection.enabled: $monitoring_enabled" | sudo tee -a /etc/kibana/kibana.yml
+
     if [ ! -z "$security_encryption_key" ]; then
         echo "$security_encryption_key" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.security.encryptionKey
     fi
     if [ ! -z "$reporting_encryption_key" ]; then
         echo "$reporting_encryption_key" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.reporting.encryptionKey
     fi
-    echo "xpack.security.enabled: $security_enabled" | sudo tee -a /etc/kibana/kibana.yml
-    echo "xpack.monitoring.enabled: $monitoring_enabled" | sudo tee -a /etc/kibana/kibana.yml
+    if [ ! -z "$saved_objects_encryption_key" ]; then
+        echo "$saved_objects_encryption_key" | /usr/share/kibana/bin/kibana-keystore add --stdin xpack.encryptedSavedObjects.encryptionKey
+    fi
+
 
     if [ "$security_enabled" == "true" ]; then
         echo "elasticsearch.username: \"kibana\"" | sudo tee -a /etc/kibana/kibana.yml

assets/scripts/common/config-es.sh

@@ -14,7 +14,6 @@
 # Configure elasticsearch
 cat <<EOF >>/etc/elasticsearch/elasticsearch.yml
 cluster.name: $es_cluster
-xpack.monitoring.enabled: $monitoring_enabled
 xpack.monitoring.collection.enabled: $monitoring_enabled
 path.data: $elasticsearch_data_dir
 path.logs: $elasticsearch_logs_dir
@@ -51,6 +50,12 @@ xpack.monitoring.exporters.xpack_remote:
 EOF
 fi
 
+# Disable HTTP SSL. Configurations may vary for HTTP SSL - see here.
+#https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#security-http-tls-ssl-key-trusted-certificate-settings
+# When not using it, we need to outright disable it for the cluster to start.
+cat <<'EOF' >>/etc/elasticsearch/elasticsearch.yml
+xpack.security.http.ssl.enabled: false
+EOF
 
 cat <<'EOF' >>/etc/security/limits.conf
 
@@ -68,7 +73,6 @@ RestartSec=10
 EOF
 
 # Setup heap size and memory locking
-sudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/init.d/elasticsearch
 sudo sed -i 's/#MAX_LOCKED_MEMORY=.*$/MAX_LOCKED_MEMORY=unlimited/' /etc/default/elasticsearch
 
 # Set java heap size
@@ -104,7 +108,6 @@ if [ "$use_g1gc" = "true" ]; then
   sudo sed -i 's/[0-9]\+-:-XX:InitiatingHeapOccupancyPercent/10-:-XX:InitiatingHeapOccupancyPercent/ig' /etc/elasticsearch/jvm.options
 fi
 
-
 # Create log and data dirs
 sudo mkdir -p $elasticsearch_logs_dir
 sudo mkdir -p $elasticsearch_data_dir

assets/scripts/gcp/config-es-discovery.sh

@@ -9,7 +9,8 @@ cloud.gce.project_id: ${gcp_project_id}
 cloud.gce.zone: ${gcp_zones}
 discovery.seed_providers: gce
 EOF
-
+# It is required to bind to all interfaces for discovery on GCP to work
+# echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml
 if [ "$BIND_TO_ALL" == "true" ]; then
 	echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml
 else

packer/README.md

@@ -4,8 +4,8 @@ This Packer configuration will generate Ubuntu images with Elasticsearch, Kibana
 
 The output of running Packer here would be two machine images, as below:
 
-* elasticsearch node image, containing latest Elasticsearch installed (latest version 7.x) and configured with best-practices.
-* kibana node image, based on the elasticsearch node image, and with Kibana (7.x, latest).
+* elasticsearch node image, containing latest Elasticsearch installed (latest version 8.x) and configured with best-practices.
+* kibana node image, based on the elasticsearch node image, and with Kibana (8.x, latest).
 
 ## On Amazon Web Services (AWS)
 
@@ -98,8 +98,8 @@ az account show --query "{ subscription_id: id }"
 Building the AMIs is done using the following commands:
 
 ```bash
-packer build -only=aws -var-file=variables.json elasticsearch7-node.packer.json
-packer build -only=aws -var-file=variables.json kibana7-node.packer.json
+packer build -only=aws -var-file=variables.json elasticsearch8-node.packer.json
+packer build -only=aws -var-file=variables.json kibana8-node.packer.json
 ```
 
 Replace the `-only` parameter to `azure` to build images for Azure instead of AWS.

packer/elasticsearch7-node.packer.json → packer/elasticsearch8-node.packer.json

@@ -4,7 +4,7 @@
     {
       "name": "aws",
       "type": "amazon-ebs",
-      "ami_name": "elasticsearch7-{{isotime | clean_resource_name}}",
+      "ami_name": "elasticsearch8-{{isotime | clean_resource_name}}",
       "availability_zone": "{{user `aws_az`}}",
       "iam_instance_profile": "packer",
       "instance_type": "t2.micro",
@@ -15,7 +15,7 @@
       "source_ami_filter": {
         "filters": {
           "virtualization-type": "hvm",
-          "name": "*ubuntu-jammy-22.04-amd64-server-*",
+          "name": "ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server*",
           "root-device-type": "ebs"
         },
         "owners": ["099720109477"],
@@ -24,7 +24,7 @@
       "ssh_timeout": "10m",
       "ssh_username": "ubuntu",
       "tags": {
-        "ImageType": "elasticsearch7-packer-image"
+        "ImageType": "elasticsearch8-packer-image"
       }
     },
     {
@@ -37,7 +37,7 @@
       "subscription_id": "{{user `azure_subscription_id`}}",
 
       "managed_image_resource_group_name": "{{user `azure_resource_group_name`}}",
-      "managed_image_name": "elasticsearch7-{{isotime \"2006-01-02T030405\"}}",
+      "managed_image_name": "elasticsearch8-{{isotime \"2006-01-02T030405\"}}",
 
       "os_type": "Linux",
       "image_publisher": "Canonical",
@@ -54,8 +54,8 @@
       "project_id": "{{user `gcp_project_id`}}",
       "source_image_family": "ubuntu-2204-lts",
       "zone": "{{user `gcp_zone`}}",
-      "image_family": "elasticsearch-7",
-      "image_name": "elasticsearch7-{{isotime | clean_resource_name}}",
+      "image_family": "elasticsearch-8",
+      "image_name": "elasticsearch8-{{isotime | clean_resource_name}}",
       "preemptible": true,
       "ssh_username": "ubuntu"
     }
@@ -88,7 +88,7 @@
     },
     {
       "type": "shell",
-      "script": "install-elasticsearch7.sh",
+      "script": "install-elasticsearch8.sh",
       "environment_vars": [ "ES_VERSION={{user `elasticsearch_version`}}" ],
       "execute_command": "echo '' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'"
     },

packer/install-cloud-plugin.sh

@@ -5,10 +5,6 @@ cd /usr/share/elasticsearch/
 
 if [[ $PACKER_BUILD_NAME == "aws" ]]; then
   sudo bin/elasticsearch-plugin install --batch discovery-ec2
-  sudo bin/elasticsearch-plugin install --batch repository-s3
-elif [[ $PACKER_BUILD_NAME == "azure" ]]; then
-  sudo bin/elasticsearch-plugin install --batch repository-azure
 elif [[ $PACKER_BUILD_NAME == "gcp" ]]; then
   sudo bin/elasticsearch-plugin install --batch discovery-gce
-  sudo bin/elasticsearch-plugin install --batch repository-gcs
 fi

packer/install-elasticsearch7.sh → packer/install-elasticsearch8.sh

@@ -3,7 +3,7 @@ set -e
 
 # Get the PGP Key
 wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-7.x.list
+echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-8.x.list
 
 apt-get update
 if [ -z "$ES_VERSION" ]; then
@@ -20,4 +20,5 @@ chown elasticsearch:elasticsearch /usr/share/elasticsearch/logs
 chown elasticsearch:elasticsearch /usr/share/elasticsearch/data
 
 mv elasticsearch.yml /etc/elasticsearch/elasticsearch.yml
-chown elasticsearch:elasticsearch /etc/elasticsearch/elasticsearch.yml
+chown elasticsearch:elasticsearch /etc/elasticsearch/elasticsearch.yml
+systemctl disable elasticsearch

packer/kibana7-node.packer.json → packer/kibana8-node.packer.json

@@ -5,7 +5,7 @@
       "name": "aws",
       "type": "amazon-ebs",
 
-      "ami_name": "kibana7-{{isotime | clean_resource_name}}",
+      "ami_name": "kibana8-{{isotime | clean_resource_name}}",
       "availability_zone": "{{user `aws_az`}}",
       "iam_instance_profile": "packer",
       "instance_type": "t2.medium",
@@ -16,7 +16,7 @@
       "source_ami_filter": {
         "filters": {
           "virtualization-type": "hvm",
-          "name": "elasticsearch7-*",
+          "name": "elasticsearch8-*",
           "root-device-type": "ebs"
         },
         "owners": [
@@ -27,7 +27,7 @@
       "ssh_timeout": "10m",
       "ssh_username": "ubuntu",
       "tags": {
-        "ImageType": "kibana7-packer-image"
+        "ImageType": "kibana8-packer-image"
       }
     },
     {
@@ -40,7 +40,7 @@
       "subscription_id": "{{user `azure_subscription_id`}}",
 
       "managed_image_resource_group_name": "{{user `azure_resource_group_name`}}",
-      "managed_image_name": "kibana7-{{isotime \"2006-01-02T030405\"}}",
+      "managed_image_name": "kibana8-{{isotime \"2006-01-02T030405\"}}",
 
       "os_type": "Linux",
       "custom_managed_image_name": "{{user `azure_elasticsearch_image_name`}}",
@@ -54,18 +54,18 @@
       "type": "googlecompute",
       "account_file": "{{user `gcp_account_file`}}",
       "project_id": "{{user `gcp_project_id`}}",
-      "source_image_family": "elasticsearch-7",
+      "source_image_family": "elasticsearch-8",
       "zone": "{{user `gcp_zone`}}",
-      "image_family": "kibana-7",
-      "image_name": "kibana7-{{isotime | clean_resource_name}}",
+      "image_family": "kibana-8",
+      "image_name": "kibana8-{{isotime | clean_resource_name}}",
       "preemptible": true,
       "ssh_username": "ubuntu"
     }
   ],
   "provisioners": [
     {
       "type": "shell",
-      "script": "install-kibana7.sh",
+      "script": "install-kibana8.sh",
       "environment_vars": [ "ES_VERSION={{user `elasticsearch_version`}}" ],
       "execute_command": "echo '' | {{ .Vars }} sudo -E -S sh '{{ .Path }}'"
     }

templates/aws_user_data.sh

@@ -35,6 +35,7 @@ export log_level="${log_level}"
 export log_size="${log_size}"
 export security_encryption_key="${security_encryption_key}"
 export reporting_encryption_key="${reporting_encryption_key}"
+export saved_objects_encryption_key="${saved_objects_encryption_key}"
 export auto_shut_down_bootstrap_node="${auto_shut_down_bootstrap_node}"
 
-/opt/cloud-deploy-scripts/${startup_script}
+/opt/cloud-deploy-scripts/${startup_script}

templates/gcp_user_data.sh

@@ -36,5 +36,6 @@ export log_level="${log_level}"
 export log_size="${log_size}"
 export security_encryption_key="${security_encryption_key}"
 export reporting_encryption_key="${reporting_encryption_key}"
+export saved_objects_encryption_key="${saved_objects_encryption_key}"
 export auto_shut_down_bootstrap_node="${auto_shut_down_bootstrap_node}"
 /opt/cloud-deploy-scripts/${startup_script}

terraform-aws/ami.tf

@@ -25,4 +25,3 @@ data "aws_ami" "kibana_client" {
   most_recent = true
   owners      = ["self"]
 }
-

terraform-aws/client.tf

@@ -1,16 +1,11 @@
-data "template_file" "client_userdata_script" {
-  template = file("${path.module}/../templates/aws_user_data.sh")
-  vars = merge(local.user_data_common, {
-    startup_script = "client.sh",
-    heap_size = var.client_heap_size
-  })
-}
-
 resource "aws_launch_template" "client" {
   name_prefix   = "elasticsearch-${var.es_cluster}-client-nodes"
   image_id      = data.aws_ami.kibana_client.id
   instance_type = var.master_instance_type
-  user_data     = base64encode(data.template_file.client_userdata_script.rendered)
+  user_data     = base64encode(templatefile("${path.module}/../templates/aws_user_data.sh",merge(local.user_data_common, {
+    startup_script = "client.sh",
+    heap_size = var.client_heap_size
+  })))
   key_name      = var.key_name
 
   iam_instance_profile {

terraform-aws/datas-voters.tf

@@ -1,17 +1,12 @@
-data "template_file" "data_voters_userdata_script" {
-  template = file("${path.module}/../templates/aws_user_data.sh")
-  vars = merge(local.user_data_common, {
-    heap_size = var.data_heap_size
-    is_voting_only      = "true"
-    startup_script = "data.sh"
-  })
-}
-
 resource "aws_launch_template" "data_voters" {
   name_prefix   = "elasticsearch-${var.es_cluster}-data-voters-nodes"
   image_id      = data.aws_ami.elasticsearch.id
   instance_type = var.data_instance_type
-  user_data     = base64encode(data.template_file.data_voters_userdata_script.rendered)
+  user_data     = base64encode(templatefile("${path.module}/../templates/aws_user_data.sh",merge(local.user_data_common, {
+    heap_size = var.data_heap_size
+    is_voting_only      = "true"
+    startup_script = "data.sh"
+  })))
   key_name      = var.key_name
 
   ebs_optimized = var.ebs_optimized

terraform-aws/datas.tf

@@ -1,16 +1,11 @@
-data "template_file" "data_userdata_script" {
-  template = file("${path.module}/../templates/aws_user_data.sh")
-  vars = merge(local.user_data_common, {
-    startup_script = "data.sh",
-    heap_size = var.data_heap_size
-  })
-}
-
 resource "aws_launch_template" "data" {
   name_prefix   = "elasticsearch-${var.es_cluster}-data-nodes"
   image_id      = data.aws_ami.elasticsearch.id
   instance_type = var.data_instance_type
-  user_data     = base64encode(data.template_file.data_userdata_script.rendered)
+  user_data     = base64encode(templatefile("${path.module}/../templates/aws_user_data.sh",merge(local.user_data_common, {
+    startup_script = "data.sh",
+    heap_size = var.data_heap_size
+  })))
   key_name      = var.key_name
 
   ebs_optimized = var.ebs_optimized

terraform-aws/disks.tf

@@ -23,7 +23,7 @@ resource "aws_ebs_volume" "master" {
 
   availability_zone = jsondecode(each.value)["az"]
   size              = 10
-  type              = "gp2"
+  type              = var.disk_type
   encrypted         = var.volume_encryption
 
   tags = {
@@ -39,7 +39,7 @@ resource "aws_ebs_volume" "data" {
 
   availability_zone = jsondecode(each.value)["az"]
   size              = var.elasticsearch_volume_size
-  type              = "gp2"
+  type              = var.disk_type
   encrypted         = var.volume_encryption
 
   tags = {
@@ -55,7 +55,7 @@ resource "aws_ebs_volume" "data-voter" {
 
   availability_zone = jsondecode(each.value)["az"]
   size              = var.elasticsearch_volume_size
-  type              = "gp2"
+  type              = var.disk_type
   encrypted         = var.volume_encryption
 
   tags = {
@@ -72,7 +72,7 @@ resource "aws_ebs_volume" "singlenode" {
 
   availability_zone = var.singlenode_az
   size              = var.elasticsearch_volume_size
-  type              = "gp2"
+  type              = var.disk_type
   encrypted         = var.volume_encryption
 
   tags = {