From 224c2fb34025cc59f64a5f8aab2249c27f66e4fa Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:23:07 +0000 Subject: [PATCH 01/15] Fix elasticsearch workflow --- azure-pipeline-start-elasticsearch.yml | 59 ++++++++++++-------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index 4d860e07..7740d9b3 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -12,36 +12,29 @@ trigger: include: - master -variables: - timeoutInMinutes: 60 - agentPool: 'hmcts-cftptl-agent-pool' - service_connection: 'GA' - -stages: - stage: StartElasticsearchService - jobs: - job: StartElasticsearchService - displayName: Start Elasticsearch Service - pool: - name: $(agentPool) - - steps: - - checkout: self - clean: true - - - task: DownloadSecureFile@1 - name: elkprivatekey - inputs: - secureFile: 'elk_private_key' - - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch Staging' - - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh demo $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch Demo' - - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh perftest $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch Perftest' - - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh ithc $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch ITHC' +pool: + name: 'hmcts-cftptl-agent-pool' + +jobs: + job: StartElasticsearchService + displayName: Start Elasticsearch Service + steps: + - checkout: self + clean: true + + - task: DownloadSecureFile@1 + name: elkprivatekey + inputs: + secureFile: 'elk_private_key' + + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch Staging' + + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh demo $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch Demo' + + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh perftest $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch Perftest' + + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh ithc $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch ITHC' From 8cd8a17d40b095e34dedddbb78c2886081eeec12 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:24:36 +0000 Subject: [PATCH 02/15] Make pipeline run on PR to test --- azure-pipeline-start-elasticsearch.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index 7740d9b3..455a6ca5 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -12,6 +12,11 @@ trigger: include: - master +pr: + branches: + include: + - master + pool: name: 'hmcts-cftptl-agent-pool' From 4fd5bb86c02e2d0b16f01e4ea7e3929c7dea3b33 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:26:45 +0000 Subject: [PATCH 03/15] Fix yaml formatting --- azure-pipeline-start-elasticsearch.yml | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index 455a6ca5..f602e3d2 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -21,25 +21,25 @@ pool: name: 'hmcts-cftptl-agent-pool' jobs: - job: StartElasticsearchService - displayName: Start Elasticsearch Service - steps: - - checkout: self - clean: true + - job: StartElasticsearchService + displayName: Start Elasticsearch Service + steps: + - checkout: self + clean: true - - task: DownloadSecureFile@1 - name: elkprivatekey - inputs: - secureFile: 'elk_private_key' + - task: DownloadSecureFile@1 + name: elkprivatekey + inputs: + secureFile: 'elk_private_key' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch Staging' + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch Staging' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh demo $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch Demo' + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh demo $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch Demo' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh perftest $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch Perftest' + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh perftest $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch Perftest' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh ithc $(elkprivatekey.secureFilePath) - displayName: 'Start Elasticsearch ITHC' + - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh ithc $(elkprivatekey.secureFilePath) + displayName: 'Start Elasticsearch ITHC' From 28b93461424b19cbd616c26e11cb536f8cd7b964 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:29:24 +0000 Subject: [PATCH 04/15] Change script path --- azure-pipeline-start-elasticsearch.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index f602e3d2..e6c84bfb 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -32,14 +32,14 @@ jobs: inputs: secureFile: 'elk_private_key' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) + - script: ./scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) displayName: 'Start Elasticsearch Staging' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh demo $(elkprivatekey.secureFilePath) + - script: ./scripts/vm/restart-es-service.sh demo $(elkprivatekey.secureFilePath) displayName: 'Start Elasticsearch Demo' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh perftest $(elkprivatekey.secureFilePath) + - script: ./scripts/vm/restart-es-service.sh perftest $(elkprivatekey.secureFilePath) displayName: 'Start Elasticsearch Perftest' - - script: ./$(System.DefaultWorkingDirectory)/scripts/vm/restart-es-service.sh ithc $(elkprivatekey.secureFilePath) + - script: ./scripts/vm/restart-es-service.sh ithc $(elkprivatekey.secureFilePath) displayName: 'Start Elasticsearch ITHC' From 701d16ed533f43bdd2b76e63417d82b54df75578 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:32:05 +0000 Subject: [PATCH 05/15] Update script arguments --- scripts/vm/restart-es-service.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 8f4d7615..3587133a 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -14,8 +14,8 @@ PERFTEST_HOSTS=("10.112.153.7" "10.112.153.6" "10.112.153.9" "10.112.153.5") # c CHECK_COMMAND="sudo systemctl is-failed elasticsearch.service" RESTART_COMMAND="sudo systemctl restart elasticsearch.service" -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " exit 1 fi From 0a165d6d293e330551bce687cf89db417b24b886 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:33:39 +0000 Subject: [PATCH 06/15] Set perms on private key --- azure-pipeline-start-elasticsearch.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index e6c84bfb..012c9c2f 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -32,6 +32,9 @@ jobs: inputs: secureFile: 'elk_private_key' + - script: chmod 600 $(elkprivatekey.secureFilePath) + displayName: 'Set permissions on private key' + - script: ./scripts/vm/restart-es-service.sh staging $(elkprivatekey.secureFilePath) displayName: 'Start Elasticsearch Staging' From 1f1073b6e93182ce7d60057d7289d358f2522176 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:43:20 +0000 Subject: [PATCH 07/15] Handle timeouts --- scripts/vm/restart-es-service.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 3587133a..59e50957 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -41,10 +41,12 @@ esac for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do STATUS=$(ssh -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") - if [ "${STATUS}" != "active" ]; then + if [ "${STATUS}" == "active" ]; then + echo "Elasticsearch service on ${REMOTE_HOST} is active." + elif [[ "${STATUS}" == *"Connection timed out"* ]]; then + echo "Connection to ${REMOTE_HOST} timed out." + else echo "Restarting Elasticsearch service on ${REMOTE_HOST}" ssh -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${RESTART_COMMAND}" - else - echo "Elasticsearch service on ${REMOTE_HOST} is active." fi done \ No newline at end of file From 8af1f67cc90876ffe9f1b3ebf4414ac3af5fdbc9 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:46:53 +0000 Subject: [PATCH 08/15] Set connection timeout for ssh --- scripts/vm/restart-es-service.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 59e50957..8baf1a5a 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -40,13 +40,13 @@ case "${ENVIRONMENT}" in esac for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do - STATUS=$(ssh -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") - if [ "${STATUS}" == "active" ]; then + STATUS=$(ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") + if [[ "${STATUS}" == "active" ]]; then echo "Elasticsearch service on ${REMOTE_HOST} is active." elif [[ "${STATUS}" == *"Connection timed out"* ]]; then echo "Connection to ${REMOTE_HOST} timed out." else echo "Restarting Elasticsearch service on ${REMOTE_HOST}" - ssh -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${RESTART_COMMAND}" + ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${RESTART_COMMAND}" fi done \ No newline at end of file From 5c72cce314e3c93a11f4d31709c74d48a79e2e2d Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:49:08 +0000 Subject: [PATCH 09/15] Echo status to troubleshoot --- scripts/vm/restart-es-service.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 8baf1a5a..53d8410a 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -41,6 +41,7 @@ esac for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do STATUS=$(ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") + echo $STATUS if [[ "${STATUS}" == "active" ]]; then echo "Elasticsearch service on ${REMOTE_HOST} is active." elif [[ "${STATUS}" == *"Connection timed out"* ]]; then From 18629e92982541c84462076196450aa42f20e578 Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:52:37 +0000 Subject: [PATCH 10/15] Change check --- scripts/vm/restart-es-service.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 53d8410a..731b1d47 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -44,8 +44,8 @@ for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do echo $STATUS if [[ "${STATUS}" == "active" ]]; then echo "Elasticsearch service on ${REMOTE_HOST} is active." - elif [[ "${STATUS}" == *"Connection timed out"* ]]; then - echo "Connection to ${REMOTE_HOST} timed out." + elif [[ -z "${STATUS}" ]]; then + echo "Problem connecting to ${REMOTE_HOST}." else echo "Restarting Elasticsearch service on ${REMOTE_HOST}" ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${RESTART_COMMAND}" From 2a8911ca87cfc195d13fa4373a9c35ff05f14fcb Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:54:50 +0000 Subject: [PATCH 11/15] Update if condition --- scripts/vm/restart-es-service.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 731b1d47..0b173f7f 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -42,12 +42,14 @@ esac for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do STATUS=$(ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") echo $STATUS - if [[ "${STATUS}" == "active" ]]; then - echo "Elasticsearch service on ${REMOTE_HOST} is active." - elif [[ -z "${STATUS}" ]]; then + if [[ -z "${STATUS}" ]]; then echo "Problem connecting to ${REMOTE_HOST}." - else - echo "Restarting Elasticsearch service on ${REMOTE_HOST}" - ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${RESTART_COMMAND}" - fi + else + if [[ "${STATUS}" == "active" ]]; then + echo "Elasticsearch service on ${REMOTE_HOST} is active." + else + echo "Restarting Elasticsearch service on ${REMOTE_HOST}" + ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${RESTART_COMMAND}" + fi + fi done \ No newline at end of file From c5460bc2b06b5cf9eb32b9c0ea447baeabdd23da Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Wed, 11 Dec 2024 15:55:04 +0000 Subject: [PATCH 12/15] Remove echo --- scripts/vm/restart-es-service.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 0b173f7f..02081cb4 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -41,7 +41,6 @@ esac for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do STATUS=$(ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") - echo $STATUS if [[ -z "${STATUS}" ]]; then echo "Problem connecting to ${REMOTE_HOST}." else From 4f8c1bfd88c455c5fd7d59c822bc3df123a7975d Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Thu, 12 Dec 2024 08:26:46 +0000 Subject: [PATCH 13/15] Update message --- scripts/vm/restart-es-service.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/restart-es-service.sh b/scripts/vm/restart-es-service.sh index 02081cb4..75366b99 100755 --- a/scripts/vm/restart-es-service.sh +++ b/scripts/vm/restart-es-service.sh @@ -42,7 +42,7 @@ esac for REMOTE_HOST in "${REMOTE_HOSTS[@]}"; do STATUS=$(ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no -i "${PRIVATE_KEY}" ${REMOTE_USER}@${REMOTE_HOST} "${CHECK_COMMAND}") if [[ -z "${STATUS}" ]]; then - echo "Problem connecting to ${REMOTE_HOST}." + echo "Problem connecting to host ${REMOTE_HOST}." else if [[ "${STATUS}" == "active" ]]; then echo "Elasticsearch service on ${REMOTE_HOST} is active." From f85e4ba8ff5f1a047fcaf75c29d6b77e8c3dad0d Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Thu, 12 Dec 2024 08:28:00 +0000 Subject: [PATCH 14/15] Don't run on PR --- azure-pipeline-start-elasticsearch.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index 012c9c2f..61689498 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -12,11 +12,6 @@ trigger: include: - master -pr: - branches: - include: - - master - pool: name: 'hmcts-cftptl-agent-pool' From 05b3a54df968636ad4b3a55e0c0a2cab415cdb8c Mon Sep 17 00:00:00 2001 From: Ieuan Byers Date: Thu, 12 Dec 2024 08:28:31 +0000 Subject: [PATCH 15/15] Only run on cron --- azure-pipeline-start-elasticsearch.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipeline-start-elasticsearch.yml b/azure-pipeline-start-elasticsearch.yml index 61689498..d6c6f007 100644 --- a/azure-pipeline-start-elasticsearch.yml +++ b/azure-pipeline-start-elasticsearch.yml @@ -7,11 +7,6 @@ schedules: include: - master -trigger: - branches: - include: - - master - pool: name: 'hmcts-cftptl-agent-pool'