From e02b998436da41dfaeeec82299e061e45acb9c0b Mon Sep 17 00:00:00 2001
From: Rodrigo Lopez <rodrigo.lopez@weaviate.io>
Date: Thu, 31 Oct 2024 16:27:38 +0100
Subject: [PATCH] port_forwarding: Forward prometheus port for each replica

---
 local-k8s.sh         | 4 ++--
 utilities/helpers.sh | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/local-k8s.sh b/local-k8s.sh
index 87fc542..a3cddf5 100755
--- a/local-k8s.sh
+++ b/local-k8s.sh
@@ -105,7 +105,7 @@ function upgrade() {
     kubectl wait sts/weaviate -n weaviate --for jsonpath='{.status.readyReplicas}'=${REPLICAS} --timeout=${TIMEOUT}
     echo_green "upgrade # Waiting for rollout upgrade to be over"
     kubectl -n weaviate rollout status statefulset weaviate
-    port_forward_to_weaviate
+    port_forward_to_weaviate $REPLICAS
     wait_weaviate
     wait_for_other_services
 
@@ -180,7 +180,7 @@ EOF
     TIMEOUT=$(get_timeout)
     echo_green "setup # Waiting (with timeout=$TIMEOUT) for Weaviate $REPLICAS node cluster to be ready"
     kubectl wait sts/weaviate -n weaviate --for jsonpath='{.status.readyReplicas}'=${REPLICAS} --timeout=${TIMEOUT}
-    port_forward_to_weaviate
+    port_forward_to_weaviate $REPLICAS
     wait_weaviate
     wait_for_other_services
 
diff --git a/utilities/helpers.sh b/utilities/helpers.sh
index 4d747d9..90c4911 100644
--- a/utilities/helpers.sh
+++ b/utilities/helpers.sh
@@ -153,6 +153,7 @@ function wait_for_raft_sync() {
 }
 
 function port_forward_to_weaviate() {
+    replicas=$1
     echo_green "Port-forwarding to Weaviate cluster"
     # Install kube-relay tool to perform port-forwarding
     # Check if kubectl-relay binary is available
@@ -197,6 +198,11 @@ function port_forward_to_weaviate() {
 
     /tmp/kubectl-relay sts/weaviate -n weaviate ${WEAVIATE_METRICS}:2112 &> /tmp/weaviate_metrics_frwd.log &
 
+    # Loop over replicas and port-forward to each node to port ${WEAVIATE_METRICS} + i
+    for i in $(seq 0 $((replicas-1))); do
+        /tmp/kubectl-relay pod/weaviate-$i -n weaviate $((WEAVIATE_METRICS+1+i)):2112 &> /tmp/weaviate_metrics_frwd_${i}.log &
+    done
+
     if [[ $OBSERVABILITY == "true" ]]; then
         /tmp/kubectl-relay svc/prometheus-grafana -n monitoring ${GRAFANA_PORT}:80 &> /tmp/grafana_frwd.log &