Merge pull request #14 from CALeDNA/develop

vm setup/dismantle fixes
CALeDNA · Nov 21, 2023 · 83129dd · 83129dd
2 parents 115d3f2 + d8ef6ef
commit 83129dd
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 4 deletions.
diff --git a/main/dismantle_instance.sh b/main/dismantle_instance.sh
@@ -36,7 +36,7 @@ fi
 source ${JSCRED}
 
 # get volume id
-volume_id=$(openstack server show $NAME -c volumes_attached -f json | jq -r '.volumes_attached[0].id')
+volume_id=$(openstack server show blast00 -c volumes_attached -f json | jq -r '.volumes_attached | split("=")[1]')
 # get corresponding ip address
 ip_address=$(grep -A 5 $NAME $SSHCONFIG | grep "HostName" | awk '{print $2}')
 # remove IP from instance

diff --git a/main/setup_instance.sh b/main/setup_instance.sh
@@ -120,7 +120,7 @@ do
         elapsed=$(( current_time - start_time ))
 
         if [[ $elapsed -ge 300 ]]; then
-            echo "Timeout: SSH not ready after 5 minutes on $VMNAME$chunk."
+            echo "Timeout: SSH or Network not ready after 5 minutes on $VMNAME$chunk."
             echo "Deleting server $VMNAME$chunk..."
             ./dismantle_instance.sh -j $JSCRED -h hostnames -m $VMNAME$chunk -c $SSHCONFIG -d $DATASOURCE
             # del ip in case it wasn't attached
@@ -130,7 +130,14 @@ do
 
         if ssh -i /home/$USER/.ssh/$PRIVATEKEY -o ConnectTimeout=5 -o StrictHostKeyChecking=no $USER@$ip_address echo "SSH is up" > /dev/null 2>&1; then
             echo "SSH is ready on $VMNAME$chunk."
-            break
+            # Network check: Ping an external server to verify network connectivity
+            if ssh -i /home/$USER/.ssh/$PRIVATEKEY -o ConnectTimeout=5 -o StrictHostKeyChecking=no $USER@$ip_address ping -c 3 google.com > /dev/null 2>&1; then
+                echo "Network is up on $VMNAME$chunk."
+                break
+            else
+                echo "Network check failed on $VMNAME$chunk. Retrying..."
+                sleep 5
+            fi
         else
             echo "Still waiting for SSH on $VMNAME$chunk..."
             sleep 5

diff --git a/scheduler/servers/ben-servers.sh b/scheduler/servers/ben-servers.sh
@@ -4,7 +4,7 @@ declare -A SERVER_MAP
 
 USER="ubuntu"
 REMOTE_PATH=/etc/ben/ben
-HOSTNAME="/home/ubuntu/crux/hostnames"
+HOSTNAME="/home/ubuntu/t-rex-monitor/hostnames"
 
 SERVER_MAP=(
     ["/tmp/ben-ecopcr"]="/tmp/ben-blast"

diff --git a/scheduler/supabase/benparser.py b/scheduler/supabase/benparser.py
@@ -16,7 +16,10 @@
             else:
                 key = line.split(" = ")[0].strip()
                 value = line.split(" = ")[1].strip()
+                if key == "command":
+                    continue
                 job[key] = value
+        jobs.append(job)
 
     # Filter out empty dictionaries
     jobs = [d for d in jobs if d]