Skip to content

Commit

Permalink
Added notes on recent prod deploy
Browse files Browse the repository at this point in the history
  • Loading branch information
@stvoutsin
stvoutsin authored Sep 26, 2023
1 parent 6a5fdc9 commit d2b110e
Showing 1 changed file with 354 additions and 0 deletions.
354 changes: 354 additions & 0 deletions notes/stv/20230926-live-blue-deploy.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,354 @@
#
# <meta:header>
# <meta:licence>
# Copyright (c) 2022, ROE (http://www.roe.ac.uk/)
#
# This information is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This information is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# </meta:licence>
# </meta:header>
#



Target:

New live deploy on blue

Result:

Success.


# -----------------------------------------------------
# Stop the live service while we build the new one.
#[user@desktop]

ssh [email protected] \
'
date
hostname
zeppelin-daemon.sh stop
'

> Zeppelin stop



# -----------------------------------------------------
# Create a container to work with.
#[user@desktop]

#
# Live is green, selecting blue for the deployment.
#

source "${HOME:?}/aglais.env"

agcolour=blue
configname=zeppelin-54.86-spark-6.26.43

agproxymap=3000:3000
clientname=ansibler-${agcolour}
cloudname=iris-gaia-${agcolour}

podman run \
--rm \
--tty \
--interactive \
--name "${clientname:?}" \
--hostname "${clientname:?}" \
--publish "${agproxymap:?}" \
--env "cloudname=${cloudname:?}" \
--env "configname=${configname:?}" \
--env "SSH_AUTH_SOCK=/mnt/ssh_auth_sock" \
--env "ANSIBLE_ANY_ERRORS_FATAL=true" \
--volume "${SSH_AUTH_SOCK:?}:/mnt/ssh_auth_sock:rw,z" \
--volume "${HOME:?}/clouds.yaml:/etc/openstack/clouds.yaml:ro,z" \
--volume "${AGLAIS_CODE:?}/deployments:/deployments:ro,z" \
ghcr.io/wfau/atolmis/ansible-client:2022.07.25 \
bash


# -----------------------------------------------------
# Deploy everything.
#[root@ansibler]

time \
source /deployments/hadoop-yarn/bin/deploy.sh

..

>

TASK [Run unittests to check connectivity] ****************************************************************************************************************************************************************
changed: [localhost]

TASK [Display Python test results] ************************************************************************************************************************************************************************
ok: [localhost] => {
"msg": [
"test_empty_string_edge_case (tests.test_decorators.TestDecorator) ... ok",
"test_http_happy_path (tests.test_decorators.TestDecorator) ... ok",
"test_https_happy_path (tests.test_decorators.TestDecorator) ... ok",
"test_special_characters_edge_case (tests.test_decorators.TestDecorator) ... ok",
"test_whitespace_string_edge_case (tests.test_decorators.TestDecorator) ... ok",
"test_wss_happy_path (tests.test_decorators.TestDecorator) ... ok",
"test_port_443_open (tests.test_port.TestPort)",
"Test that port 443 is open ... ok",
"test_port_80_open (tests.test_port.TestPort)",
"Test that port 80 is open ... ok",
"test_redirect_to_https (tests.test_port.TestPort)",
"Test that the service redirects correctly ... ok",
"test_redirect_to_https_with_path (tests.test_port.TestPort)",
"Test that the service redirects correctly with path ... ok",
"test_websocket_connection (tests.test_websockets.WebSocketTest)",
"Test a WSS connection to the domain ... ok",
"",
"----------------------------------------------------------------------",
"Ran 11 tests in 2.442s",
"",
"OK"
]
}

TASK [Run script to verify open ports] ********************************************************************************************************************************************************************
ok: [localhost]

TASK [Check script output] ********************************************************************************************************************************************************************************
ok: [localhost] => {
"msg": "Security group for server iris-gaia-blue-20230926-zeppelin is correctly configured"
}

PLAY RECAP ************************************************************************************************************************************************************************************************
localhost : ok=7 changed=4 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0

/
aglais:
status:
deployment:
type: hadoop-yarn
conf: zeppelin-54.86-spark-6.26.43
name: iris-gaia-blue-20230926
date: 20230926T083356
hostname: zeppelin.gaia-dmp.uk
spec:
openstack:
cloud:
base: arcus
name: iris-gaia-blue

real 80m9.771s
user 10m50.489s
sys 3m3.708s




# -----------------------------------------------------
# Import our live users.
#[root@ansibler]

source /deployments/zeppelin/bin/create-user-tools.sh

import-live-users

>
..

"notebooks":
{
"user": "PKuzma",
"debug": {
"script": "copy-notebooks.sh",
"result": "PASS",
"messages": ["SKIP: Notebooks skipped - null password"]
}
}
}
]}

> Done


# -----------------------------------------------------
# Run Tests
#[root@ansibler]

git clone https://github.com/stvoutsin/aglais-testing
pushd aglais-testing/
pip install -r pip-requirements
python3 setup.py install
popd


# -----------------------------------------------------
# Create user config (Use SVoutsinas acount)
#[root@ansibler]

nano /tmp/test-users.json
{
"users": [
{
"username": "",
"shirouser": {
"name": "",
"password": ""
}
}
]
}




pushd aglais-testing/gdmp_benchmark


python3 gdmp_benchmark.py --zeppelin_url https://iris-gaia-blue.gaia-dmp.uk --usercount 1 --notebook_config /deployments/zeppelin/test/config/quick.json --user_config /tmp/test-users.json --delay_start 0 --delay_notebook 0
{

"config": {
"endpoint": "https://iris-gaia-blue.gaia-dmp.uk",
"testconfig": "/deployments/zeppelin/test/config/quick.json",
"userconfig": "/tmp/test-users.json",
"usercount": "1",
"delaystart": "0",
"delaynotebook": "0"
},

}
---start---
[{
"name": "GaiaDMPSetup",
"result": "SUCCESS",
"outputs": {
"valid": true
},
"messages": [],
"time": {
"result": "FAST",
"elapsed": "49.00",
"percent": "-2.00",
"start": "2023-09-26T10:37:47.208322",
"finish": "2023-09-26T10:38:36.235771"
},
"logs": ""
},
{
"name": "Mean_proper_motions_over_the_sky",
"result": "SUCCESS",
"outputs": {
"valid": true
},
"messages": [],
"time": {
"result": "FAST",
"elapsed": "120.00",
"percent": "-4.00",
"start": "2023-09-26T10:38:36.235875",
"finish": "2023-09-26T10:40:37.066599"
},
"logs": ""
},
{
"name": "Source_counts_over_the_sky.json",
"result": "SUCCESS",
"outputs": {
"valid": true
},
"messages": [],
"time": {
"result": "FAST",
"elapsed": "49.00",
"percent": "-10.91",
"start": "2023-09-26T10:40:37.067249",
"finish": "2023-09-26T10:41:26.884195"
},
"logs": ""
},
{
"name": "Library_Validation.json",
"result": "SUCCESS",
"outputs": {
"valid": true
},
"messages": [],
"time": {
"result": "FAST",
"elapsed": "10.00",
"percent": "0.00",
"start": "2023-09-26T10:41:26.884289",
"finish": "2023-09-26T10:41:37.363749"
},
"logs": ""
}]
---end---


# -----------------------------------------------------
# Get the public IP address of our Zeppelin node.
#[root@ansibler]

deployname=$(
yq eval \
'.aglais.status.deployment.name' \
'/opt/aglais/aglais-status.yml'
)

zeppelinid=$(
openstack \
--os-cloud "${cloudname:?}" \
server list \
--format json \
| jq -r '.[] | select(.Name == "'${deployname:?}'-zeppelin") | .ID'
)

zeppelinip=$(
openstack \
--os-cloud "${cloudname:?}" \
server show \
--format json \
"${zeppelinid:?}" \
| jq -r ".addresses | .\"${deployname}-internal-network\" | .[1]"
)

cat << EOF
Zeppelin ID [${zeppelinid:?}]
Zeppelin IP [${zeppelinip:?}]
EOF
Zeppelin ID [2a30239d-8f7e-4510-af50-888b8f5ebcdb]
Zeppelin IP [128.232.226.225]





# -----------------------------------------------------
# Update the dns entry with new IP
# [root@ansibler]


source /deployments/zeppelin/bin/create-user-tools.sh
ducktoken=$(getsecret 'devops.duckdns.token')
duckname=aglais-live

curl "https://www.duckdns.org/update/${duckname:?}/${ducktoken:?}/${zeppelinip:?}"

> OK


# -----------------------------------------------------
# Manually check https://dmp.gaia.ac.uk/#/ to validate
# [root@ansibler]

> Success

0 comments on commit d2b110e

Please sign in to comment.