Move decrement_ongoing_request_count to top to cover all exceptions

triton-inference-server · Jul 25, 2024 · 15d00ef · 15d00ef
1 parent 492c23e
commit 15d00ef
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/model.py b/src/model.py
@@ -343,6 +343,7 @@ async def generate(self, request):
         """
         response_sender = request.get_response_sender()
         self.ongoing_request_count += 1
+        decrement_ongoing_request_count = True
         try:
             request_id = random_uuid()
             prompt = pb_utils.get_input_tensor_by_name(
@@ -400,7 +401,6 @@ async def generate(self, request):
             response_iterator = await self.llm_engine.add_request(
                 request_id, prompt, sampling_params
             )
-            decrement_ongoing_request_count = True
 
             async for output in response_iterator:
                 if response_sender.is_cancelled():