update the code to work with secure elasticsearch cluster connection

tools/maintenance_scripts/backup_elasticsearch_data.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
-source image_name.txt
+source vars.txt
 echo  $SEARCHENGINE_IMAGE
 sudo docker run   --name searchenginecach  --rm  -v /searchengine_backup/:/searchengine_backup/ -v /data/searchengine/searchengine/:/etc/searchengine/ --network=searchengine-net  $SEARCHENGINE_IMAGE backup_elasticsearch_data

tools/maintenance_scripts/check_cluster_health.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-curl 127.0.0.1:9203/_cluster/health?pretty
+curl -k -u elastic https://127.0.0.1:9201/_cluster/health?pretty

tools/maintenance_scripts/check_searchengine_health.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
-source image_name.txt
+source vars.txt
 sudo rm /data/searchengine/searchengine/check_report.txt
 sudo docker run  --rm --name searchengine_validator  -v /data/searchengine/searchengine/:/etc/searchengine/  -v /data/searchengine/searchengine/logs/:/opt/app-root/src/logs/  --network=searchengine-net  $SEARCHENGINE_IMAGE test_indexing_search_query

tools/maintenance_scripts/index_data.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-source image_name.txt
+source vars.txt
 echo  $SEARCHENGINE_IMAGE
 sudo docker run -d  --name searchengine_index   -v /data/searchengine/searchengine/:/etc/searchengine/  -v /data/searchengine/searchengine/logs/:/opt/app-root/src/logs/  --network=searchengine-net $SEARCHENGINE_IMAGE get_index_data_from_database
-

tools/maintenance_scripts/restore_elasticsearch_data.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
-source image_name.txt
+source vars.txt
 echo  $SEARCHENGINE_IMAGE
 sudo docker run   --name searchenginecach  --rm  -v /searchengine_backup/:/searchengine_backup/ -v /data/searchengine/searchengine/:/etc/searchengine/ --network=searchengine-net  $SEARCHENGINE_IMAGE restore_elasticsearch_data

tools/maintenance_scripts/run_elasticsearch_node1.sh

@@ -1,3 +1,42 @@
 #!/bin/bash
-sudo docker run  -d --rm -v /searchengine_backup:/searchengine_backup  -v  /data/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch -v  /data/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch -p  9201:9200 -p 9301:9300 --network searchengine-net  -e "path.data=/var/lib/elasticsearch" -e "path.logs=/var/log/elasticsearch" -e "path.repo=/searchengine_backup" -e "ingest.geoip.downloader.enabled=false" -e  "network.host=0.0.0.0" -e  "cluster.name=searchengine-cluster" -e "discovery.seed_hosts=searchengine_elasticsearch_node3" -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" -e "node.name=searchengine_elasticsearch_node1"  -e "bootstrap.memory_lock=true"  -e "node.master=true" -e "node.data=true" --ulimit memlock=-1:-1 --name searchengine_elasticsearch_node1 docker.elastic.co/elasticsearch/elasticsearch:7.16.2
-
+source vars.txt
+sudo docker run  -d --rm -v /searchengine_backup:/searchengine_backup  \
+  -v  /data/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch \
+  -v  /data/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch \
+  -v  $elast_certs_folder:/usr/share/elasticsearch/config/certificates \
+  -p  9201:9200 -p 9301:9300 \
+  --network searchengine-net  \
+  --ip 10.11.0.2  \
+  -e "path.data=/var/lib/elasticsearch" \
+  -e "path.logs=/var/log/elasticsearch" \
+  -e "path.repo=/searchengine_backup" \
+  -e "ingest.geoip.downloader.enabled=false" \
+  -e  "network.host=0.0.0.0" \
+  -e  "cluster.name=searchengine-cluster" \
+  -e "discovery.seed_hosts=searchengine_elasticsearch_node1" \
+  -e "http.host=0.0.0.0" \
+  -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" \
+  -e "node.name=searchengine_elasticsearch_node1"  \
+  -e "cluster.initial_master_nodes=searchengine_elasticsearch_node1,searchengine_elasticsearch_node2,searchengine_elasticsearch_node3"  \
+  -e "discovery.seed_hosts= searchengine_elasticsearch_node2" \
+  -e "bootstrap.memory_lock=true"  \
+  -e  "es_api_basic_auth_username=elastic" \
+  -e "ELASTIC_PASSWORD=$elastic_password" \
+  -e "es_validate_certs=no" \
+  -e "es_enable_http_ssl=true" \
+  -e "xpack.security.http.ssl.enabled=true" \
+  -e "xpack.security.enabled=true" \
+  -e "xpack.security.authc.realms.file.file1.order=0" \
+  -e "xpack.security.authc.realms.native.native1.order=1" \
+  -e "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-ca.p12" \
+  -e "xpack.security.http.ssl.truststore.password=$elastic_ca_password" \
+  -e "xpack.security.http.ssl.keystore.password=$elastic_ca_password" \
+  -e "xpack.security.transport.ssl.enabled=true" \
+  -e "xpack.security.transport.ssl.verification_mode=certificate" \
+  -e "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12" \
+  -e "xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12" \
+  -e "xpack.security.transport.ssl.keystore.password=$keystore_password" \
+  -e  "xpack.security.transport.ssl.truststore.password=$keystore_password" \
+  --ulimit memlock=-1:-1 \
+  --name searchengine_elasticsearch_node1 \
+  $ELASTICSEARCH_IMAGE

tools/maintenance_scripts/run_elasticsearch_node2.sh

@@ -1,3 +1,44 @@
 #!/bin/bash
-sudo docker run  -d --rm  -v /searchengine_backup:/searchengine_backup -v  /data/searchengine/elasticsearch/node2/data:/var/lib/elasticsearch -v  /data/searchengine/elasticsearch/node2/logs:/var/log/elasticsearch -p  9202:9200 -p 9302:9300 --network searchengine-net  -e "path.data=/var/lib/elasticsearch" -e "path.logs=/var/log/elasticsearch" -e "path.repo=/searchengine_backup"  -e "ingest.geoip.downloader.enabled=false" -e  "network.host=0.0.0.0" -e  "cluster.name=searchengine-cluster" -e "discovery.seed_hosts=searchengine_elasticsearch_node1" -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" -e "node.name=searchengine_elasticsearch_node2"  -e "bootstrap.memory_lock=true"  -e "node.master=true" -e "node.data=true" --ulimit memlock=-1:-1 --name searchengine_elasticsearch_node2 docker.elastic.co/elasticsearch/elasticsearch:7.16.2
+source vars.txt
 
+sudo docker run  -d --rm \
+  -v /searchengine_backup:/searchengine_backup \
+  -v  /data/searchengine/elasticsearch/node2/data:/var/lib/elasticsearch \
+  -v  /data/searchengine/elasticsearch/node2/logs:/var/log/elasticsearch \
+   -v $elast_certs_folder:/usr/share/elasticsearch/config/certificates \
+  -p  9202:9200 \
+  -p 9302:9300 \
+  --network searchengine-net  \
+   --ip 10.11.0.3  \
+  -e "path.data=/var/lib/elasticsearch" \
+  -e "path.logs=/var/log/elasticsearch" \
+  -e "path.repo=/searchengine_backup"  \
+  -e "ingest.geoip.downloader.enabled=false" \
+  -e  "network.host=0.0.0.0" \
+  -e  "cluster.name=searchengine-cluster" \
+  -e "discovery.seed_hosts=searchengine_elasticsearch_node1" \
+  -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" \
+  -e "node.name=searchengine_elasticsearch_node2"  \
+  -e "bootstrap.memory_lock=true"  \
+  -e "discovery.seed_hosts=searchengine_elasticsearch_node1"
+  -e "cluster.initial_master_nodes=earchengine_elasticsearch_node1,searchengine_elasticsearch_node2,searchengine_elasticsearch_node3"
+  -e  "es_api_basic_auth_username=elastic" \
+  -e "ELASTIC_PASSWORD=$elastic_password" \
+  -e "es_validate_certs=no" \
+  -e "es_enable_http_ssl=true" \
+  -e "xpack.security.http.ssl.enabled=true" \
+  -e "xpack.security.enabled=true" \
+  -e "xpack.security.authc.realms.file.file1.order=0" \
+  -e "xpack.security.authc.realms.native.native1.order=1" \
+  -e "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-ca.p12" \
+  -e "xpack.security.http.ssl.truststore.password=$elastic_ca_password" \
+  -e "xpack.security.http.ssl.keystore.password=$elastic_ca_password" \
+  -e "xpack.security.transport.ssl.enabled=true" \
+  -e "xpack.security.transport.ssl.verification_mode=certificate" \
+  -e "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node2/searchengine_elasticsearch_node2.p12" \
+  -e "xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node2/searchengine_elasticsearch_node2.p12" \
+  -e "xpack.security.transport.ssl.keystore.password=$keystore_password" \
+  -e  "xpack.security.transport.ssl.truststore.password=$keystore_password" \
+  --ulimit memlock=-1:-1 \
+  --name searchengine_elasticsearch_node2 \
+  $ELASTICSEARCH_IMAGE

tools/maintenance_scripts/run_elasticsearch_node3.sh

@@ -1,3 +1,43 @@
 #!/bin/bash
-sudo docker run  -d --rm  -v /searchengine_backup:/searchengine_backup -v  /data/searchengine/elasticsearch/node3/data:/var/lib/elasticsearch -v  /data/searchengine/elasticsearch/node3/logs:/var/log/elasticsearch -p  9203:9200 -p 9303:9300 --network searchengine-net  -e "path.data=/var/lib/elasticsearch" -e "path.logs=/var/log/elasticsearch" -e "path.repo=/searchengine_backup"  -e "ingest.geoip.downloader.enabled=false" -e  "network.host=0.0.0.0" -e  "cluster.name=searchengine-cluster" -e "discovery.seed_hosts=searchengine_elasticsearch_node1" -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" -e "node.name=searchengine_elasticsearch_node3"  -e "bootstrap.memory_lock=true"  -e "node.master=true" -e "node.data=true" --ulimit memlock=-1:-1 --name searchengine_elasticsearch_node3 docker.elastic.co/elasticsearch/elasticsearch:7.16.2
+source vars.txt
 
+sudo docker run  -d --rm  -v /searchengine_backup:/searchengine_backup \
+  -v  /data/searchengine/elasticsearch/node3/data:/var/lib/elasticsearch \
+  -v  /data/searchengine/elasticsearch/node3/logs:/var/log/elasticsearch \
+   -v $elast_certs_folder:/usr/share/elasticsearch/config/certificates \
+  -p  9203:9200 -p 9303:9300 \
+  --network searchengine-net \
+  --ip 10.11.0.2  \
+  -e "path.data=/var/lib/elasticsearch" \
+  -e "path.logs=/var/log/elasticsearch" \
+  -e "path.repo=/searchengine_backup" \
+  -e "ingest.geoip.downloader.enabled=false" \
+  -e  "network.host=0.0.0.0" \
+  -e  "cluster.name=searchengine-cluster" \
+  -e "discovery.seed_hosts=searchengine_elasticsearch_node1" \
+  -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" \
+  -e "node.name=searchengine_elasticsearch_node3" \
+  -e "bootstrap.memory_lock=true" \
+  -e "discovery.seed_hosts=searchengine_elasticsearch_node1"
+  -e "cluster.initial_master_nodes=earchengine_elasticsearch_node1,searchengine_elasticsearch_node2,searchengine_elasticsearch_node3"
+  -e  "es_api_basic_auth_username=elastic" \
+  -e "ELASTIC_PASSWORD=$elastic_password" \
+  -e "es_validate_certs=no" \
+  -e "es_enable_http_ssl=true" \
+  -e "xpack.security.http.ssl.enabled=true" \
+  -e "xpack.security.enabled=true" \
+  -e "xpack.security.authc.realms.file.file1.order=0" \
+  -e "xpack.security.authc.realms.native.native1.order=1" \
+  -e "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-ca.p12" \
+  -e "xpack.security.http.ssl.truststore.password=$elastic_ca_password" \
+  -e "xpack.security.http.ssl.keystore.password=$elastic_ca_password" \
+  -e "xpack.security.transport.ssl.enabled=true" \
+  -e "xpack.security.transport.ssl.verification_mode=certificate" \
+  -e "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node3/searchengine_elasticsearch_node3.p12" \
+  -e "xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node3/searchengine_elasticsearch_node3.p12" \
+  -e "xpack.security.transport.ssl.keystore.password=$keystore_password" \
+  -e  "xpack.security.transport.ssl.truststore.password=$keystore_password" \
+  --ulimit memlock=-1:-1 \
+  --name searchengine_elasticsearch_node3 \
+  --ip 10.11.0.2 \
+  $ELASTICSEARCH_IMAGE

tools/maintenance_scripts/stop_all_containers.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-read -p "Do you want to stop all the containers? (y/n) " resp
+read -p "Do you want to stop all and delete the containers? (y/n) " resp
 if [ "$resp" = "y" ]; then
   sudo docker stop $(sudo docker ps -q)
   sudo docker rm  $(sudo docker ps -a  -q)

tools/maintenance_scripts/stop_elasticsearch_node.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 sudo docker stop searchengine_elasticsearch_node$1
-sudo docker rm searchengine_elasticsearch_node$1
+#sudo docker rm searchengine_elasticsearch_node$1

tools/maintenance_scripts/vars.txt

@@ -0,0 +1,6 @@
+SEARCHENGINE_IMAGE=openmicroscopy/omero-searchengine:latest
+ELASTICSEARCH_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:8.8.1
+elast_certs_folder=/data/searchengine/elasticsearch/certs
+elastic_ca_password=replace_with_ca_password
+keystore_password=replace_with_keystore_password
+elastic_password=replace_with_elastic_password