From 29f4139fc6172d1b43953aa51199bccf4ba59775 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Thu, 5 Oct 2023 14:30:30 -0700
Subject: [PATCH] Rollback non-decoupled any response on cancel

---
 README.md      | 9 ++++-----
 src/pb_stub.cc | 5 +++++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 86abb10d..03285f69 100644
--- a/README.md
+++ b/README.md
@@ -508,10 +508,8 @@ Supported error codes:
 #### Request Cancellation Handling
 
 One or more requests may be cancelled by the client during execution. Starting
-from 23.10, `request.is_cancelled()` returns whether the request is cancelled.
-
-If a request is cancelled, the model may respond with any dummy object in place
-of the normal output tensors on the request. For example:
+from 23.10, `request.is_cancelled()` returns whether the request is cancelled or
+not. For example:
 
 ```python
 import triton_python_backend_utils as pb_utils
@@ -524,7 +522,8 @@ class TritonPythonModel:
 
         for request in requests:
             if request.is_cancelled():
-                responses.append(None)
+                responses.append(pb_utils.InferenceResponse(
+                    error=pb_utils.TritonError("Message", pb_utils.TritonError.CANCELLED)))
             else:
                 ...
 
diff --git a/src/pb_stub.cc b/src/pb_stub.cc
index c379998d..370df866 100644
--- a/src/pb_stub.cc
+++ b/src/pb_stub.cc
@@ -771,6 +771,11 @@ Stub::ProcessRequests(RequestBatch* request_batch_shm_ptr)
           std::to_string(response_size) + "\n";
       throw PythonBackendException(err);
     }
+    for (auto& response : responses) {
+      if (!py::isinstance<InferResponse>(response)) {
+        std::string str = py::str(response.get_type());
+      }
+    }
     for (size_t i = 0; i < response_size; i++) {
       // If the model has checked for cancellation and the request is cancelled,
       // replace returned type with a cancelled response.