diff --git a/main/dismantle_instance.sh b/main/dismantle_instance.sh index a4eaa2f..0cdff7d 100755 --- a/main/dismantle_instance.sh +++ b/main/dismantle_instance.sh @@ -36,7 +36,7 @@ fi source ${JSCRED} # get volume id -volume_id=$(openstack server show $NAME -c volumes_attached -f json | jq -r '.volumes_attached[0].id') +volume_id=$(openstack server show blast00 -c volumes_attached -f json | jq -r '.volumes_attached | split("=")[1]') # get corresponding ip address ip_address=$(grep -A 5 $NAME $SSHCONFIG | grep "HostName" | awk '{print $2}') # remove IP from instance diff --git a/main/setup_instance.sh b/main/setup_instance.sh index e111bf4..29323ef 100755 --- a/main/setup_instance.sh +++ b/main/setup_instance.sh @@ -120,7 +120,7 @@ do elapsed=$(( current_time - start_time )) if [[ $elapsed -ge 300 ]]; then - echo "Timeout: SSH not ready after 5 minutes on $VMNAME$chunk." + echo "Timeout: SSH or Network not ready after 5 minutes on $VMNAME$chunk." echo "Deleting server $VMNAME$chunk..." ./dismantle_instance.sh -j $JSCRED -h hostnames -m $VMNAME$chunk -c $SSHCONFIG -d $DATASOURCE # del ip in case it wasn't attached @@ -130,7 +130,14 @@ do if ssh -i /home/$USER/.ssh/$PRIVATEKEY -o ConnectTimeout=5 -o StrictHostKeyChecking=no $USER@$ip_address echo "SSH is up" > /dev/null 2>&1; then echo "SSH is ready on $VMNAME$chunk." - break + # Network check: Ping an external server to verify network connectivity + if ssh -i /home/$USER/.ssh/$PRIVATEKEY -o ConnectTimeout=5 -o StrictHostKeyChecking=no $USER@$ip_address ping -c 3 google.com > /dev/null 2>&1; then + echo "Network is up on $VMNAME$chunk." + break + else + echo "Network check failed on $VMNAME$chunk. Retrying..." + sleep 5 + fi else echo "Still waiting for SSH on $VMNAME$chunk..." sleep 5 diff --git a/scheduler/servers/ben-servers.sh b/scheduler/servers/ben-servers.sh index 3f42d22..cf4b0da 100755 --- a/scheduler/servers/ben-servers.sh +++ b/scheduler/servers/ben-servers.sh @@ -4,7 +4,7 @@ declare -A SERVER_MAP USER="ubuntu" REMOTE_PATH=/etc/ben/ben -HOSTNAME="/home/ubuntu/crux/hostnames" +HOSTNAME="/home/ubuntu/t-rex-monitor/hostnames" SERVER_MAP=( ["/tmp/ben-ecopcr"]="/tmp/ben-blast" diff --git a/scheduler/supabase/benparser.py b/scheduler/supabase/benparser.py index 102d9e7..d1c0604 100644 --- a/scheduler/supabase/benparser.py +++ b/scheduler/supabase/benparser.py @@ -16,7 +16,10 @@ else: key = line.split(" = ")[0].strip() value = line.split(" = ")[1].strip() + if key == "command": + continue job[key] = value + jobs.append(job) # Filter out empty dictionaries jobs = [d for d in jobs if d]