Merge pull request #3953 from tssurya/ocpbugs-17841

OCPBUGS-17841: Ensure gcp-routes hack for internalLB hairpin traffic works for SGW
openshift · Oct 10, 2023 · 06a3363 · 06a3363
2 parents c74052c + 9facf37
commit 06a3363
Showing 1 changed file with 62 additions and 0 deletions.
diff --git a/templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml b/templates/master/00-master/gcp/files/opt-libexec-openshift-gcp-routes-sh.yaml
@@ -112,6 +112,27 @@ contents:
         done
     }
 
+    remove_stale_routes() {
+        ## find extra ovn routes
+        local ovnkContainerID=$(crictl ps --name ovnkube-controller | awk '{ print $1 }' | tail -n+2)
+        if [ -z "${ovnkContainerID}" ]; then
+            echo "Plugin is SDN, nothing to do.. exiting"
+            return
+        fi
+        echo "Found ovnkube-controller pod... ${ovnkContainerID}"
+        local routeVIPsV4=$(crictl exec -i ${ovnkContainerID} ovn-nbctl lr-policy-list ovn_cluster_router | grep "1010" | grep "ip4" | awk '$8{print $8}')
+        echo "Found v4route vips: ${routeVIPsV4}"
+        local host=$(hostname)
+        echo ${host}
+        for route_vip in ${routeVIPsV4}; do
+            if [[ ! -v vips[${route_vip}] ]] || [[ "${vips[${route_vip}]}" = down ]]; then
+                echo removing stale vip "${route_vip}" for local clients
+                echo "ovn-nbctl lr-policy-del ovn_cluster_router 1010 inport == rtos-${host} && ip4.dst == ${route_vip}"
+                crictl exec -i ${ovnkContainerID} ovn-nbctl lr-policy-del ovn_cluster_router 1010 "inport == \"rtos-${host}\" && ip4.dst == ${route_vip}"
+            fi
+        done
+    }
+
     add_rules() {
         for vip in "${!vips[@]}"; do
             echo "ensuring rule for ${vip} for external clients"
@@ -124,11 +145,49 @@ contents:
         done
     }
 
+    add_routes() {
+        local ovnkContainerID=$(crictl ps --name ovnkube-controller | awk '{ print $1 }' | tail -n+2)
+        if [ -z "${ovnkContainerID}" ]; then
+            echo "Plugin is SDN, nothing to do.. exiting"
+            return
+        fi
+        echo "Found ovnkube-controller pod... ${ovnkContainerID}"
+        local ovnK8sMp0v4=$(ip -brief address show ovn-k8s-mp0 | awk '{print $3}' | awk -F/ '{print $1}')
+        echo "Found ovn-k8s-mp0 interface IP ${ovnK8sMp0v4}"
+        local host=$(hostname)
+        echo ${host}
+        for vip in "${!vips[@]}"; do
+            if [[ "${vips[${vip}]}" != down ]]; then
+                echo "ensuring route for ${vip} for internal clients"
+                local routes=$(crictl exec -i ${ovnkContainerID} ovn-nbctl lr-policy-list ovn_cluster_router | grep "1010" | grep "${vip}" | grep "${ovnK8sMp0v4}")
+                echo "OVNK Routes on ovn-cluster-router at 1010 priority: $routes"
+                if [[ "${routes}" == *"${vip}"* ]]; then
+                    echo "Route exists"
+                else
+                    echo "Route does not exist; creating it..."
+                    echo "ovn-nbctl lr-policy-add ovn_cluster_router 1010 inport == rtos-${host} && ip4.dst == ${vip} reroute ${ovnK8sMp0v4}"
+                    crictl exec -i ${ovnkContainerID} ovn-nbctl lr-policy-add ovn_cluster_router 1010 "inport == \"rtos-${host}\" && ip4.dst == ${vip}" reroute "${ovnK8sMp0v4}"
+                fi
+            fi
+        done
+    }
+
     clear_rules() {
         iptables -t nat -F "${CHAIN_NAME}" || true
         iptables -t nat -F "${CHAIN_NAME}-local" || true
     }
 
+    clear_routes() {
+        local ovnkContainerID=$(crictl ps --name ovnkube-controller | awk '{ print $1 }' | tail -n+2)
+        if [ -z "${ovnkContainerID}" ]; then
+            echo "Plugin is SDN, nothing to do.. exiting"
+            return
+        fi
+        echo "Found ovnkube-controller pod... ${ovnkContainerID}"
+        echo "clearing all routes from ovn-cluster-router"
+        crictl exec -i ${ovnkContainerID} ovn-nbctl lr-policy-del ovn_cluster_router 1010
+    }
+
     # out paramater: vips
     list_lb_ips() {
         for k in "${!vips[@]}"; do
@@ -179,13 +238,16 @@ contents:
         while :; do
           list_lb_ips
           remove_stale
+          remove_stale_routes # needed for OVN-Kubernetes plugin's routingViaHost=false mode
           add_rules
+          add_routes # needed for OVN-Kubernetes plugin's routingViaHost=false mode
           echo "done applying vip rules"
           sleep_or_watch
         done
         ;;
       cleanup)
         clear_rules
+        clear_routes # needed for OVN-Kubernetes plugin's routingViaHost=false mode
         ;;
       *)
         echo $"Usage: $0 {start|cleanup}"