diff --git a/ci/L0_backend_vllm/metrics_test/test.sh b/ci/L0_backend_vllm/metrics_test/test.sh index 884be62..a9a4db9 100755 --- a/ci/L0_backend_vllm/metrics_test/test.sh +++ b/ci/L0_backend_vllm/metrics_test/test.sh @@ -75,11 +75,11 @@ run_test() { fi fi + set -e + # TODO: Non-graceful shutdown when metrics are enabled. kill $SERVER_PID wait $SERVER_PID - - set -e } RET=0 diff --git a/src/model.py b/src/model.py index 46c35a2..ad2a5c8 100644 --- a/src/model.py +++ b/src/model.py @@ -228,7 +228,7 @@ def _init_engine(self): # Run the engine in a separate thread running the AsyncIO event loop. self._llm_engine = None self._llm_engine_start_cv = threading.Condition() - self._llm_engine_shutdown_event = asyncio.Event() + self._llm_engine_shutdown_event = threading.Event() self._event_thread = threading.Thread( target=asyncio.run, args=(self._run_llm_engine(),) ) @@ -268,7 +268,8 @@ async def _run_llm_engine(self): self._llm_engine_start_cv.notify_all() # Wait for the engine shutdown signal. - await self._llm_engine_shutdown_event.wait() + while not self._llm_engine_shutdown_event.is_set(): + await asyncio.sleep(0.1) # Prevent busy-waiting # Wait for the ongoing requests to complete. while self._ongoing_request_count > 0: