From 7f98a20a85039b6edb6027bf43f04a4ca40a7a3d Mon Sep 17 00:00:00 2001 From: huchen2021 <85480625+huchen2021@users.noreply.github.com> Date: Thu, 26 Oct 2023 00:22:47 +0800 Subject: [PATCH] [cluster launcher] [vSphere] Do not fetch runtime-info of vm from cached_nodes (#40655) Power-on-off status is runtime info of VM, should not fetch it from cached-nodes, which is probably dirty data. It should query by pyvmomi_sdk every time. Signed-off-by: Chen Hui --- .../ray/autoscaler/_private/vsphere/node_provider.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/ray/autoscaler/_private/vsphere/node_provider.py b/python/ray/autoscaler/_private/vsphere/node_provider.py index 7cb99f09f71f2..126dc7759d30a 100755 --- a/python/ray/autoscaler/_private/vsphere/node_provider.py +++ b/python/ray/autoscaler/_private/vsphere/node_provider.py @@ -176,12 +176,16 @@ def non_terminated_nodes(self, tag_filters): return nodes def is_running(self, node_id): - node = self._get_cached_node(node_id) - return node.power_state in {HardPower.State.POWERED_ON} + vm = self.pyvmomi_sdk_provider.get_pyvmomi_obj( + [vim.VirtualMachine], obj_id=node_id + ) + return vm.runtime.powerState != vim.VirtualMachinePowerState.poweredOn def is_terminated(self, node_id): - node = self._get_cached_node(node_id) - return node.power_state not in {HardPower.State.POWERED_ON} + vm = self.pyvmomi_sdk_provider.get_pyvmomi_obj( + [vim.VirtualMachine], obj_id=node_id + ) + return vm.runtime.powerState != vim.VirtualMachinePowerState.poweredOn def node_tags(self, node_id): with self.tag_cache_lock: