diff --git a/tools/maintenance_scripts/backup_elasticsearch_data.sh b/tools/maintenance_scripts/backup_elasticsearch_data.sh index 38b13aa3..ff1d9030 100644 --- a/tools/maintenance_scripts/backup_elasticsearch_data.sh +++ b/tools/maintenance_scripts/backup_elasticsearch_data.sh @@ -1,4 +1,4 @@ #!/bin/bash -source image_name.txt +source vars.txt echo $SEARCHENGINE_IMAGE sudo docker run --name searchenginecach --rm -v /searchengine_backup/:/searchengine_backup/ -v /data/searchengine/searchengine/:/etc/searchengine/ --network=searchengine-net $SEARCHENGINE_IMAGE backup_elasticsearch_data diff --git a/tools/maintenance_scripts/check_cluster_health.sh b/tools/maintenance_scripts/check_cluster_health.sh index 10d825e5..f958c70f 100644 --- a/tools/maintenance_scripts/check_cluster_health.sh +++ b/tools/maintenance_scripts/check_cluster_health.sh @@ -1,2 +1,2 @@ #!/bin/bash -curl 127.0.0.1:9203/_cluster/health?pretty +curl -k -u elastic https://127.0.0.1:9201/_cluster/health?pretty diff --git a/tools/maintenance_scripts/check_searchengine_health.sh b/tools/maintenance_scripts/check_searchengine_health.sh index b729932a..d8b556a7 100644 --- a/tools/maintenance_scripts/check_searchengine_health.sh +++ b/tools/maintenance_scripts/check_searchengine_health.sh @@ -1,4 +1,4 @@ #!/bin/bash -source image_name.txt +source vars.txt sudo rm /data/searchengine/searchengine/check_report.txt sudo docker run --rm --name searchengine_validator -v /data/searchengine/searchengine/:/etc/searchengine/ -v /data/searchengine/searchengine/logs/:/opt/app-root/src/logs/ --network=searchengine-net $SEARCHENGINE_IMAGE test_indexing_search_query diff --git a/tools/maintenance_scripts/image_name.txt b/tools/maintenance_scripts/image_name.txt deleted file mode 100644 index d415af0f..00000000 --- a/tools/maintenance_scripts/image_name.txt +++ /dev/null @@ -1 +0,0 @@ -SEARCHENGINE_IMAGE=openmicroscopy/omero-searchengine:latest diff --git a/tools/maintenance_scripts/index_data.sh b/tools/maintenance_scripts/index_data.sh index db6201c5..3e4b3e53 100644 --- a/tools/maintenance_scripts/index_data.sh +++ b/tools/maintenance_scripts/index_data.sh @@ -1,5 +1,4 @@ #!/bin/bash -source image_name.txt +source vars.txt echo $SEARCHENGINE_IMAGE sudo docker run -d --name searchengine_index -v /data/searchengine/searchengine/:/etc/searchengine/ -v /data/searchengine/searchengine/logs/:/opt/app-root/src/logs/ --network=searchengine-net $SEARCHENGINE_IMAGE get_index_data_from_database - diff --git a/tools/maintenance_scripts/restore_elasticsearch_data.sh b/tools/maintenance_scripts/restore_elasticsearch_data.sh index 654a4c40..a4dc20e6 100644 --- a/tools/maintenance_scripts/restore_elasticsearch_data.sh +++ b/tools/maintenance_scripts/restore_elasticsearch_data.sh @@ -1,4 +1,4 @@ #!/bin/bash -source image_name.txt +source vars.txt echo $SEARCHENGINE_IMAGE sudo docker run --name searchenginecach --rm -v /searchengine_backup/:/searchengine_backup/ -v /data/searchengine/searchengine/:/etc/searchengine/ --network=searchengine-net $SEARCHENGINE_IMAGE restore_elasticsearch_data diff --git a/tools/maintenance_scripts/run_elasticsearch_node1.sh b/tools/maintenance_scripts/run_elasticsearch_node1.sh index 88130d36..c54ef292 100644 --- a/tools/maintenance_scripts/run_elasticsearch_node1.sh +++ b/tools/maintenance_scripts/run_elasticsearch_node1.sh @@ -1,3 +1,42 @@ #!/bin/bash -sudo docker run -d --rm -v /searchengine_backup:/searchengine_backup -v /data/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch -v /data/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch -p 9201:9200 -p 9301:9300 --network searchengine-net -e "path.data=/var/lib/elasticsearch" -e "path.logs=/var/log/elasticsearch" -e "path.repo=/searchengine_backup" -e "ingest.geoip.downloader.enabled=false" -e "network.host=0.0.0.0" -e "cluster.name=searchengine-cluster" -e "discovery.seed_hosts=searchengine_elasticsearch_node3" -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" -e "node.name=searchengine_elasticsearch_node1" -e "bootstrap.memory_lock=true" -e "node.master=true" -e "node.data=true" --ulimit memlock=-1:-1 --name searchengine_elasticsearch_node1 docker.elastic.co/elasticsearch/elasticsearch:7.16.2 - +source vars.txt +sudo docker run -d --rm -v /searchengine_backup:/searchengine_backup \ + -v /data/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch \ + -v /data/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch \ + -v $elast_certs_folder:/usr/share/elasticsearch/config/certificates \ + -p 9201:9200 -p 9301:9300 \ + --network searchengine-net \ + --ip 10.11.0.2 \ + -e "path.data=/var/lib/elasticsearch" \ + -e "path.logs=/var/log/elasticsearch" \ + -e "path.repo=/searchengine_backup" \ + -e "ingest.geoip.downloader.enabled=false" \ + -e "network.host=0.0.0.0" \ + -e "cluster.name=searchengine-cluster" \ + -e "discovery.seed_hosts=searchengine_elasticsearch_node1" \ + -e "http.host=0.0.0.0" \ + -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" \ + -e "node.name=searchengine_elasticsearch_node1" \ + -e "cluster.initial_master_nodes=searchengine_elasticsearch_node1,searchengine_elasticsearch_node2,searchengine_elasticsearch_node3" \ + -e "discovery.seed_hosts= searchengine_elasticsearch_node2" \ + -e "bootstrap.memory_lock=true" \ + -e "es_api_basic_auth_username=elastic" \ + -e "ELASTIC_PASSWORD=$elastic_password" \ + -e "es_validate_certs=no" \ + -e "es_enable_http_ssl=true" \ + -e "xpack.security.http.ssl.enabled=true" \ + -e "xpack.security.enabled=true" \ + -e "xpack.security.authc.realms.file.file1.order=0" \ + -e "xpack.security.authc.realms.native.native1.order=1" \ + -e "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-ca.p12" \ + -e "xpack.security.http.ssl.truststore.password=$elastic_ca_password" \ + -e "xpack.security.http.ssl.keystore.password=$elastic_ca_password" \ + -e "xpack.security.transport.ssl.enabled=true" \ + -e "xpack.security.transport.ssl.verification_mode=certificate" \ + -e "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12" \ + -e "xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12" \ + -e "xpack.security.transport.ssl.keystore.password=$keystore_password" \ + -e "xpack.security.transport.ssl.truststore.password=$keystore_password" \ + --ulimit memlock=-1:-1 \ + --name searchengine_elasticsearch_node1 \ + $ELASTICSEARCH_IMAGE diff --git a/tools/maintenance_scripts/run_elasticsearch_node2.sh b/tools/maintenance_scripts/run_elasticsearch_node2.sh index 1567b5cd..05de79a1 100644 --- a/tools/maintenance_scripts/run_elasticsearch_node2.sh +++ b/tools/maintenance_scripts/run_elasticsearch_node2.sh @@ -1,3 +1,44 @@ #!/bin/bash -sudo docker run -d --rm -v /searchengine_backup:/searchengine_backup -v /data/searchengine/elasticsearch/node2/data:/var/lib/elasticsearch -v /data/searchengine/elasticsearch/node2/logs:/var/log/elasticsearch -p 9202:9200 -p 9302:9300 --network searchengine-net -e "path.data=/var/lib/elasticsearch" -e "path.logs=/var/log/elasticsearch" -e "path.repo=/searchengine_backup" -e "ingest.geoip.downloader.enabled=false" -e "network.host=0.0.0.0" -e "cluster.name=searchengine-cluster" -e "discovery.seed_hosts=searchengine_elasticsearch_node1" -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" -e "node.name=searchengine_elasticsearch_node2" -e "bootstrap.memory_lock=true" -e "node.master=true" -e "node.data=true" --ulimit memlock=-1:-1 --name searchengine_elasticsearch_node2 docker.elastic.co/elasticsearch/elasticsearch:7.16.2 +source vars.txt +sudo docker run -d --rm \ + -v /searchengine_backup:/searchengine_backup \ + -v /data/searchengine/elasticsearch/node2/data:/var/lib/elasticsearch \ + -v /data/searchengine/elasticsearch/node2/logs:/var/log/elasticsearch \ + -v $elast_certs_folder:/usr/share/elasticsearch/config/certificates \ + -p 9202:9200 \ + -p 9302:9300 \ + --network searchengine-net \ + --ip 10.11.0.3 \ + -e "path.data=/var/lib/elasticsearch" \ + -e "path.logs=/var/log/elasticsearch" \ + -e "path.repo=/searchengine_backup" \ + -e "ingest.geoip.downloader.enabled=false" \ + -e "network.host=0.0.0.0" \ + -e "cluster.name=searchengine-cluster" \ + -e "discovery.seed_hosts=searchengine_elasticsearch_node1" \ + -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" \ + -e "node.name=searchengine_elasticsearch_node2" \ + -e "bootstrap.memory_lock=true" \ + -e "discovery.seed_hosts=searchengine_elasticsearch_node1" + -e "cluster.initial_master_nodes=earchengine_elasticsearch_node1,searchengine_elasticsearch_node2,searchengine_elasticsearch_node3" + -e "es_api_basic_auth_username=elastic" \ + -e "ELASTIC_PASSWORD=$elastic_password" \ + -e "es_validate_certs=no" \ + -e "es_enable_http_ssl=true" \ + -e "xpack.security.http.ssl.enabled=true" \ + -e "xpack.security.enabled=true" \ + -e "xpack.security.authc.realms.file.file1.order=0" \ + -e "xpack.security.authc.realms.native.native1.order=1" \ + -e "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-ca.p12" \ + -e "xpack.security.http.ssl.truststore.password=$elastic_ca_password" \ + -e "xpack.security.http.ssl.keystore.password=$elastic_ca_password" \ + -e "xpack.security.transport.ssl.enabled=true" \ + -e "xpack.security.transport.ssl.verification_mode=certificate" \ + -e "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node2/searchengine_elasticsearch_node2.p12" \ + -e "xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node2/searchengine_elasticsearch_node2.p12" \ + -e "xpack.security.transport.ssl.keystore.password=$keystore_password" \ + -e "xpack.security.transport.ssl.truststore.password=$keystore_password" \ + --ulimit memlock=-1:-1 \ + --name searchengine_elasticsearch_node2 \ + $ELASTICSEARCH_IMAGE diff --git a/tools/maintenance_scripts/run_elasticsearch_node3.sh b/tools/maintenance_scripts/run_elasticsearch_node3.sh index 94ddb0b8..0bc4d265 100644 --- a/tools/maintenance_scripts/run_elasticsearch_node3.sh +++ b/tools/maintenance_scripts/run_elasticsearch_node3.sh @@ -1,3 +1,43 @@ #!/bin/bash -sudo docker run -d --rm -v /searchengine_backup:/searchengine_backup -v /data/searchengine/elasticsearch/node3/data:/var/lib/elasticsearch -v /data/searchengine/elasticsearch/node3/logs:/var/log/elasticsearch -p 9203:9200 -p 9303:9300 --network searchengine-net -e "path.data=/var/lib/elasticsearch" -e "path.logs=/var/log/elasticsearch" -e "path.repo=/searchengine_backup" -e "ingest.geoip.downloader.enabled=false" -e "network.host=0.0.0.0" -e "cluster.name=searchengine-cluster" -e "discovery.seed_hosts=searchengine_elasticsearch_node1" -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" -e "node.name=searchengine_elasticsearch_node3" -e "bootstrap.memory_lock=true" -e "node.master=true" -e "node.data=true" --ulimit memlock=-1:-1 --name searchengine_elasticsearch_node3 docker.elastic.co/elasticsearch/elasticsearch:7.16.2 +source vars.txt +sudo docker run -d --rm -v /searchengine_backup:/searchengine_backup \ + -v /data/searchengine/elasticsearch/node3/data:/var/lib/elasticsearch \ + -v /data/searchengine/elasticsearch/node3/logs:/var/log/elasticsearch \ + -v $elast_certs_folder:/usr/share/elasticsearch/config/certificates \ + -p 9203:9200 -p 9303:9300 \ + --network searchengine-net \ + --ip 10.11.0.2 \ + -e "path.data=/var/lib/elasticsearch" \ + -e "path.logs=/var/log/elasticsearch" \ + -e "path.repo=/searchengine_backup" \ + -e "ingest.geoip.downloader.enabled=false" \ + -e "network.host=0.0.0.0" \ + -e "cluster.name=searchengine-cluster" \ + -e "discovery.seed_hosts=searchengine_elasticsearch_node1" \ + -e "http.host=0.0.0.0" -e "ES_JAVA_OPTS=-Xms2g -Xmx2g" \ + -e "node.name=searchengine_elasticsearch_node3" \ + -e "bootstrap.memory_lock=true" \ + -e "discovery.seed_hosts=searchengine_elasticsearch_node1" + -e "cluster.initial_master_nodes=earchengine_elasticsearch_node1,searchengine_elasticsearch_node2,searchengine_elasticsearch_node3" + -e "es_api_basic_auth_username=elastic" \ + -e "ELASTIC_PASSWORD=$elastic_password" \ + -e "es_validate_certs=no" \ + -e "es_enable_http_ssl=true" \ + -e "xpack.security.http.ssl.enabled=true" \ + -e "xpack.security.enabled=true" \ + -e "xpack.security.authc.realms.file.file1.order=0" \ + -e "xpack.security.authc.realms.native.native1.order=1" \ + -e "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-ca.p12" \ + -e "xpack.security.http.ssl.truststore.password=$elastic_ca_password" \ + -e "xpack.security.http.ssl.keystore.password=$elastic_ca_password" \ + -e "xpack.security.transport.ssl.enabled=true" \ + -e "xpack.security.transport.ssl.verification_mode=certificate" \ + -e "xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node3/searchengine_elasticsearch_node3.p12" \ + -e "xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node3/searchengine_elasticsearch_node3.p12" \ + -e "xpack.security.transport.ssl.keystore.password=$keystore_password" \ + -e "xpack.security.transport.ssl.truststore.password=$keystore_password" \ + --ulimit memlock=-1:-1 \ + --name searchengine_elasticsearch_node3 \ + --ip 10.11.0.2 \ + $ELASTICSEARCH_IMAGE diff --git a/tools/maintenance_scripts/stop_all_containers.sh b/tools/maintenance_scripts/stop_all_containers.sh index 0f4229da..03de623a 100644 --- a/tools/maintenance_scripts/stop_all_containers.sh +++ b/tools/maintenance_scripts/stop_all_containers.sh @@ -1,5 +1,5 @@ #!/bin/bash -read -p "Do you want to stop all the containers? (y/n) " resp +read -p "Do you want to stop all and delete the containers? (y/n) " resp if [ "$resp" = "y" ]; then sudo docker stop $(sudo docker ps -q) sudo docker rm $(sudo docker ps -a -q) diff --git a/tools/maintenance_scripts/stop_elasticsearch_node.sh b/tools/maintenance_scripts/stop_elasticsearch_node.sh index 3856cbb1..0188852f 100644 --- a/tools/maintenance_scripts/stop_elasticsearch_node.sh +++ b/tools/maintenance_scripts/stop_elasticsearch_node.sh @@ -1,3 +1,3 @@ #!/bin/bash sudo docker stop searchengine_elasticsearch_node$1 -sudo docker rm searchengine_elasticsearch_node$1 +#sudo docker rm searchengine_elasticsearch_node$1 diff --git a/tools/maintenance_scripts/vars.txt b/tools/maintenance_scripts/vars.txt new file mode 100644 index 00000000..54d701d2 --- /dev/null +++ b/tools/maintenance_scripts/vars.txt @@ -0,0 +1,6 @@ +SEARCHENGINE_IMAGE=openmicroscopy/omero-searchengine:latest +ELASTICSEARCH_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:8.8.1 +elast_certs_folder=/data/searchengine/elasticsearch/certs +elastic_ca_password=replace_with_ca_password +keystore_password=replace_with_keystore_password +elastic_password=replace_with_elastic_password