#4073: Throw instead of continue on DPRINT WAIT hang

tenstorrent · Dec 11, 2023 · 6702969 · 6702969
1 parent 3bc5d4d
commit 6702969
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/tests/tt_metal/tt_metal/unit_tests_fast_dispatch/dprint/test_print_hanging.cpp b/tests/tt_metal/tt_metal/unit_tests_fast_dispatch/dprint/test_print_hanging.cpp
@@ -37,10 +37,17 @@ TEST_F(CommandQueueWithDPrintFixture, TestPrintHanging) {
         DataMovementConfig{.processor = DataMovementProcessor::RISCV_0, .noc = NOC::RISCV_0_default}
     );
 
-    // Run the program
+    // Run the program, we expect it to throw on waiting for CQ to finish
     EnqueueProgram(cq, program, false);
+try {
     Finish(cq);
     tt_await_debug_print_server();
+} catch (std::runtime_error& e) {
+    const string expected = "Command Queue could not finish: device hang due to unanswered DPRINT WAIT.";
+    const string error = string(e.what());
+    log_info(tt::LogTest, "Caught exception (one is expected in this test): {}", error);
+    EXPECT_TRUE(error.find(expected) != string::npos);
+}
 
     // Check the print log against golden output.
     EXPECT_TRUE(

diff --git a/tt_metal/impl/dispatch/command_queue.cpp b/tt_metal/impl/dispatch/command_queue.cpp
@@ -879,9 +879,9 @@ void CommandQueue::finish() {
         tt::Cluster::instance().read_sysmem(&finish, 4, HOST_CQ_FINISH_PTR, 0);
 
         // There's also a case where the device can be hung due to an unanswered DPRINT WAIT and
-        // a full print buffer. Poll the print server for this case and finish if it happens.
+        // a full print buffer. Poll the print server for this case and throw if it happens.
         if (tt_print_hang_detected()) {
-            break;
+            TT_THROW("Command Queue could not finish: device hang due to unanswered DPRINT WAIT.");
         }
     } while (finish != 1);