feat: output error to debug console

debugger/include/debugger.h

@@ -59,6 +59,7 @@ class Debugger {
   bool stop();
 
   void onLuaFileLoaded(lua_State* L, std::string_view path, bool is_entry);
+  void onError(std::string_view msg);
 
  private:
   void onClientConnected(const std::shared_ptr<dap::ReaderWriter>& rw);

debugger/src/debugger.cpp

@@ -63,6 +63,11 @@ void Debugger::onLuaFileLoaded(lua_State* L,
   debug_bridge_->onLuaFileLoaded(L, path, is_entry);
 }
 
+void Debugger::onError(std::string_view msg) {
+  if (session_ != nullptr)
+    session_->send(dap::OutputEvent{.output = std::string{msg}});
+}
+
 void Debugger::closeSession() {
   if (std::exchange(session_, nullptr) == nullptr)
     DEBUGGER_LOG_INFO("Debugger already stopped");

debugger/src/internal/debug_bridge.cpp

@@ -21,41 +21,26 @@
 
 namespace luau::debugger {
 
-DebugBridge::DebugBridge(bool stop_on_entry) : stop_on_entry_(stop_on_entry) {}
-
-DebugBridge::~DebugBridge() {
-  if (main_vm_ != nullptr)
-    lua_setthreaddata(main_vm_, nullptr);
-}
-
-DebugBridge* DebugBridge::getDebugBridge(lua_State* L) {
-  lua_State* main_vm = lua_mainthread(L);
-  auto* debugger = reinterpret_cast<Debugger*>(lua_getthreaddata(main_vm));
-  if (debugger == nullptr)
-    return nullptr;
-  return debugger->debug_bridge_.get();
-}
-
-void DebugBridge::initialize(lua_State* L) {
-  main_vm_ = L;
-  lua_Callbacks* cb = lua_callbacks(L);
-  cb->debugbreak = [](lua_State* L, lua_Debug* ar) {
+class LuaCallbacks {
+ public:
+  static void debugbreak(lua_State* L, lua_Debug* ar) {
     auto bridge = DebugBridge::getDebugBridge(L);
     if (bridge == nullptr)
       return;
     bridge->onDebugBreak(L, ar,
-                         bridge->isBreakOnEntry(L) ? BreakReason::Entry
-                                                   : BreakReason::BreakPoint);
+                         bridge->isBreakOnEntry(L)
+                             ? DebugBridge::BreakReason::Entry
+                             : DebugBridge::BreakReason::BreakPoint);
   };
 
-  cb->interrupt = [](lua_State* L, int gc) {
+  static void interrupt(lua_State* L, int gc) {
     auto bridge = DebugBridge::getDebugBridge(L);
     if (bridge == nullptr)
       return;
     bridge->processPendingMainThreadActions();
   };
 
-  cb->userthread = [](lua_State* LP, lua_State* L) {
+  static void userthread(lua_State* LP, lua_State* L) {
     auto bridge = DebugBridge::getDebugBridge(L);
     if (bridge == nullptr)
       return;
@@ -66,10 +51,46 @@ void DebugBridge::initialize(lua_State* L) {
       bridge->markAlive(L, LP);
   };
 
+  static void debugstep(lua_State* L, lua_Debug* ar) {
+    auto bridge = DebugBridge::getDebugBridge(L);
+    if (bridge == nullptr)
+      return;
+    if (bridge->single_step_processor_ != nullptr)
+      if ((bridge->single_step_processor_)(L, ar))
+        bridge->onDebugBreak(L, ar, DebugBridge::BreakReason::Step);
+  };
+};
+
+DebugBridge::DebugBridge(bool stop_on_entry) : stop_on_entry_(stop_on_entry) {}
+
+DebugBridge::~DebugBridge() {
+  if (main_vm_ != nullptr)
+    lua_setthreaddata(main_vm_, nullptr);
+}
+
+DebugBridge* DebugBridge::getDebugBridge(lua_State* L) {
+  lua_State* main_vm = lua_mainthread(L);
+  auto* debugger = reinterpret_cast<Debugger*>(lua_getthreaddata(main_vm));
+  if (debugger == nullptr)
+    return nullptr;
+  return debugger->debug_bridge_.get();
+}
+
+void DebugBridge::initialize(lua_State* L) {
+  main_vm_ = L;
+
+  initCallbacks();
   lua_singlestep(L, true);
   markAlive(L, nullptr);
 }
 
+void DebugBridge::initCallbacks() {
+  lua_Callbacks* cb = lua_callbacks(main_vm_);
+  cb->debugbreak = LuaCallbacks::debugbreak;
+  cb->interrupt = LuaCallbacks::interrupt;
+  cb->userthread = LuaCallbacks::userthread;
+}
+
 bool DebugBridge::isDebugBreak() {
   std::scoped_lock lock(break_mutex_);
   return !resume_;
@@ -408,14 +429,7 @@ void DebugBridge::enableDebugStep(bool enable) {
     return;
   }
 
-  callbacks->debugstep = [](lua_State* L, lua_Debug* ar) {
-    auto bridge = DebugBridge::getDebugBridge(L);
-    if (bridge == nullptr)
-      return;
-    if (bridge->single_step_processor_ != nullptr)
-      if ((bridge->single_step_processor_)(L, ar))
-        bridge->onDebugBreak(L, ar, BreakReason::Step);
-  };
+  callbacks->debugstep = LuaCallbacks::debugstep;
 }
 
 void DebugBridge::resumeInternal() {

debugger/src/internal/debug_bridge.h

@@ -28,6 +28,7 @@ struct BreakContext {
 
 using namespace dap;
 
+class LuaCallbacks;
 class DebugBridge final {
  public:
   static DebugBridge* getDebugBridge(lua_State* L);
@@ -101,6 +102,8 @@ class DebugBridge final {
   }
 
  private:
+  void initCallbacks();
+
   std::string normalizePath(std::string_view path) const;
   BreakContext getBreakContext(lua_State* L) const;
   int getStackDepth(lua_State* L) const;
@@ -135,6 +138,7 @@ class DebugBridge final {
   void markDead(lua_State* L);
 
  private:
+  friend class LuaCallbacks;
   bool stop_on_entry_ = false;
   std::string entry_path_;
 

luaud/luau_runtime.cpp

@@ -114,17 +114,23 @@ static int lua_collectgarbage(lua_State* L) {
 
 namespace luau {
 
+static Runtime* runtime_ = nullptr;
+
 Runtime::Runtime() {
   Luau::assertHandler() = [](const char* expr, const char* file, int line,
                              const char* function) {
-    error(std::format("{}({}): ASSERTION FAILED: {}\n", file, line, expr));
+    if (runtime_)
+      runtime_->onError(
+          std::format("{}({}): ASSERTION FAILED: {}\n", file, line, expr));
     return 1;
   };
+  runtime_ = this;
   vm_ = luaL_newstate();
 }
 
 Runtime::~Runtime() {
   lua_close(vm_);
+  runtime_ = nullptr;
 }
 
 void Runtime::installDebugger(debugger::Debugger* debugger) {
@@ -152,7 +158,7 @@ void Runtime::installLibrary() {
 bool Runtime::runFile(const char* name) {
   std::optional<std::string> source = file_utils::readFile(name);
   if (!source) {
-    error(std::format("Error opening {}\n", name));
+    onError(std::format("Error opening {}\n", name));
     return false;
   }
 
@@ -183,10 +189,23 @@ bool Runtime::runFile(const char* name) {
     msg += "\nstacktrace:\n";
     msg += lua_debugtrace(L);
 
-    error(msg);
+    onError(msg);
   }
 
   lua_pop(vm_, 1);
   return status == 0;
 }
+
+void Runtime::setErrorHandler(std::function<void(std::string_view)> handler) {
+  errorHandler_ = handler;
+}
+
+void Runtime::onError(std::string_view msg) {
+  auto with_color = std::format("\x1B[31m{}\033[0m\n", msg);
+  if (errorHandler_)
+    errorHandler_(with_color);
+  if (debugger_)
+    debugger_->onError(with_color);
+}
+
 }  // namespace luau

luaud/luau_runtime.h

@@ -19,20 +19,13 @@ class Runtime {
   void installLibrary();
   bool runFile(const char* name);
 
-  void setErrorHandler(std::function<void(std::string_view)> handler) {
-    errorHandler_ = handler;
-  }
-
- private:
-  static void error(std::string_view msg) {
-    if (errorHandler_)
-      errorHandler_(std::format("\x1B[31m{}\033[0m\n", msg));
-  }
+  void setErrorHandler(std::function<void(std::string_view)> handler);
+  void onError(std::string_view msg);
 
  private:
   lua_State* vm_ = nullptr;
   debugger::Debugger* debugger_ = nullptr;
-  inline static std::function<void(std::string_view)> errorHandler_ = nullptr;
+  std::function<void(std::string_view)> errorHandler_ = nullptr;
 };
 
 }  // namespace luau

luaud/main.cpp

@@ -33,8 +33,8 @@ int main(int argc, char** argv) {
   luau::debugger::Debugger debugger(true);
 
   runtime.setErrorHandler(error_handler);
-  runtime.installDebugger(&debugger);
   runtime.installLibrary();
+  runtime.installDebugger(&debugger);
 
   debugger.listen(std::atoi(argv[1]));
   int result = runtime.runFile(argv[2]);

tests/main.lua

@@ -97,11 +97,17 @@ local function test_math()
   print("mul", mul(a, b))
 end
 
+local function test_error()
+    -- error should output to debug console
+    -- abc.def()
+end
+
 local function main()
     test_variables()
     test_step()
     test_coroutine()
     test_math()
+    test_error()
 end
 
 main()