From a01873689c2c7adcf4d57e3f8e0cc0ffb0560ace Mon Sep 17 00:00:00 2001 From: hector-baez Date: Sun, 19 Nov 2023 01:25:24 -0600 Subject: [PATCH 1/5] add last object --- scheduler/supabase/benparser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scheduler/supabase/benparser.py b/scheduler/supabase/benparser.py index 102d9e7..b4bbb59 100644 --- a/scheduler/supabase/benparser.py +++ b/scheduler/supabase/benparser.py @@ -17,6 +17,7 @@ key = line.split(" = ")[0].strip() value = line.split(" = ")[1].strip() job[key] = value + jobs.append(job) # Filter out empty dictionaries jobs = [d for d in jobs if d] From 2d136c46fb5f83c1ee6beda1d1c276853b68ada4 Mon Sep 17 00:00:00 2001 From: hector-baez Date: Sun, 19 Nov 2023 01:25:41 -0600 Subject: [PATCH 2/5] change hostnames path --- scheduler/servers/ben-servers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler/servers/ben-servers.sh b/scheduler/servers/ben-servers.sh index 3f42d22..cf4b0da 100755 --- a/scheduler/servers/ben-servers.sh +++ b/scheduler/servers/ben-servers.sh @@ -4,7 +4,7 @@ declare -A SERVER_MAP USER="ubuntu" REMOTE_PATH=/etc/ben/ben -HOSTNAME="/home/ubuntu/crux/hostnames" +HOSTNAME="/home/ubuntu/t-rex-monitor/hostnames" SERVER_MAP=( ["/tmp/ben-ecopcr"]="/tmp/ben-blast" From 98120717040288e7d6cdc470e836ae9d64b4cac7 Mon Sep 17 00:00:00 2001 From: hector-baez Date: Tue, 21 Nov 2023 11:09:35 -0600 Subject: [PATCH 3/5] update json parse for volume id --- main/dismantle_instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/dismantle_instance.sh b/main/dismantle_instance.sh index a4eaa2f..0cdff7d 100755 --- a/main/dismantle_instance.sh +++ b/main/dismantle_instance.sh @@ -36,7 +36,7 @@ fi source ${JSCRED} # get volume id -volume_id=$(openstack server show $NAME -c volumes_attached -f json | jq -r '.volumes_attached[0].id') +volume_id=$(openstack server show blast00 -c volumes_attached -f json | jq -r '.volumes_attached | split("=")[1]') # get corresponding ip address ip_address=$(grep -A 5 $NAME $SSHCONFIG | grep "HostName" | awk '{print $2}') # remove IP from instance From 04312b957e506eaa0791706320e7fc34c111591e Mon Sep 17 00:00:00 2001 From: hector-baez Date: Tue, 21 Nov 2023 11:09:56 -0600 Subject: [PATCH 4/5] add network check on setup --- main/setup_instance.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/main/setup_instance.sh b/main/setup_instance.sh index e111bf4..29323ef 100755 --- a/main/setup_instance.sh +++ b/main/setup_instance.sh @@ -120,7 +120,7 @@ do elapsed=$(( current_time - start_time )) if [[ $elapsed -ge 300 ]]; then - echo "Timeout: SSH not ready after 5 minutes on $VMNAME$chunk." + echo "Timeout: SSH or Network not ready after 5 minutes on $VMNAME$chunk." echo "Deleting server $VMNAME$chunk..." ./dismantle_instance.sh -j $JSCRED -h hostnames -m $VMNAME$chunk -c $SSHCONFIG -d $DATASOURCE # del ip in case it wasn't attached @@ -130,7 +130,14 @@ do if ssh -i /home/$USER/.ssh/$PRIVATEKEY -o ConnectTimeout=5 -o StrictHostKeyChecking=no $USER@$ip_address echo "SSH is up" > /dev/null 2>&1; then echo "SSH is ready on $VMNAME$chunk." - break + # Network check: Ping an external server to verify network connectivity + if ssh -i /home/$USER/.ssh/$PRIVATEKEY -o ConnectTimeout=5 -o StrictHostKeyChecking=no $USER@$ip_address ping -c 3 google.com > /dev/null 2>&1; then + echo "Network is up on $VMNAME$chunk." + break + else + echo "Network check failed on $VMNAME$chunk. Retrying..." + sleep 5 + fi else echo "Still waiting for SSH on $VMNAME$chunk..." sleep 5 From d8ef6efbdbb82f5c2757f083f626dc830bbe5aee Mon Sep 17 00:00:00 2001 From: hector-baez Date: Tue, 21 Nov 2023 11:17:35 -0600 Subject: [PATCH 5/5] rm command key from json --- scheduler/supabase/benparser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scheduler/supabase/benparser.py b/scheduler/supabase/benparser.py index b4bbb59..d1c0604 100644 --- a/scheduler/supabase/benparser.py +++ b/scheduler/supabase/benparser.py @@ -16,6 +16,8 @@ else: key = line.split(" = ")[0].strip() value = line.split(" = ")[1].strip() + if key == "command": + continue job[key] = value jobs.append(job)