port_forwarding: Forward prometheus port for each replica

local-k8s.sh

@@ -105,7 +105,7 @@ function upgrade() {
     kubectl wait sts/weaviate -n weaviate --for jsonpath='{.status.readyReplicas}'=${REPLICAS} --timeout=${TIMEOUT}
     echo_green "upgrade # Waiting for rollout upgrade to be over"
     kubectl -n weaviate rollout status statefulset weaviate
-    port_forward_to_weaviate
+    port_forward_to_weaviate $REPLICAS
     wait_weaviate
     wait_for_other_services
 
@@ -180,7 +180,7 @@ EOF
     TIMEOUT=$(get_timeout)
     echo_green "setup # Waiting (with timeout=$TIMEOUT) for Weaviate $REPLICAS node cluster to be ready"
     kubectl wait sts/weaviate -n weaviate --for jsonpath='{.status.readyReplicas}'=${REPLICAS} --timeout=${TIMEOUT}
-    port_forward_to_weaviate
+    port_forward_to_weaviate $REPLICAS
     wait_weaviate
     wait_for_other_services
 

utilities/helpers.sh

@@ -153,6 +153,7 @@ function wait_for_raft_sync() {
 }
 
 function port_forward_to_weaviate() {
+    replicas=$1
     echo_green "Port-forwarding to Weaviate cluster"
     # Install kube-relay tool to perform port-forwarding
     # Check if kubectl-relay binary is available
@@ -197,6 +198,11 @@ function port_forward_to_weaviate() {
 
     /tmp/kubectl-relay sts/weaviate -n weaviate ${WEAVIATE_METRICS}:2112 &> /tmp/weaviate_metrics_frwd.log &
 
+    # Loop over replicas and port-forward to each node to port ${WEAVIATE_METRICS} + i
+    for i in $(seq 0 $((replicas-1))); do
+        /tmp/kubectl-relay pod/weaviate-$i -n weaviate $((WEAVIATE_METRICS+1+i)):2112 &> /tmp/weaviate_metrics_frwd_${i}.log &
+    done
+
     if [[ $OBSERVABILITY == "true" ]]; then
         /tmp/kubectl-relay svc/prometheus-grafana -n monitoring ${GRAFANA_PORT}:80 &> /tmp/grafana_frwd.log &