From a2003bc616d4927ea53328ae9694994bf7a4f3c0 Mon Sep 17 00:00:00 2001 From: Mohan_J <57227290+PMohanJ@users.noreply.github.com> Date: Fri, 3 May 2024 00:47:41 +0530 Subject: [PATCH] Fix the keyboard key held state occurring due to bad network conditions (#146) * Client sends key-repeat events to the server to indicate deliberate key held state to distinguish from key held state which might occur under some bad network conditions. The server releases those keys that are held due to bad network conditions by analysing the time interval of key-repeat events of the corresponding keys. * Minor edits --------- Co-authored-by: Seungmin Kim <8457324+ehfd@users.noreply.github.com> --- addons/gst-web/src/app.js | 3 +++ addons/gst-web/src/input.js | 35 +++++++++++++++++++++++++++ addons/gst-web/src/util.js | 16 ++++++++++++ src/selkies_gstreamer/__main__.py | 1 + src/selkies_gstreamer/webrtc_input.py | 33 ++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/addons/gst-web/src/app.js b/addons/gst-web/src/app.js index 0cc66d44..6edb3d40 100644 --- a/addons/gst-web/src/app.js +++ b/addons/gst-web/src/app.js @@ -578,6 +578,9 @@ window.addEventListener('focus', () => { window.addEventListener('blur', () => { // reset keyboard to avoid stuck keys. webrtc.sendDataChannelMessage("kr"); + + // Clear the key-repeat events on window blur + webrtc.input.keyRepeatQueue.clear(); }); webrtc.onclipboardcontent = (content) => { diff --git a/addons/gst-web/src/input.js b/addons/gst-web/src/input.js index e7d102bb..a2e7878d 100644 --- a/addons/gst-web/src/input.js +++ b/addons/gst-web/src/input.js @@ -129,6 +129,9 @@ class Input { // variable used to scale cursor speed this.cursorScaleFactor = null; + + // keys pressed list to send key repeat events to server + this.keyRepeatQueue = new Queue(); } /** @@ -610,12 +613,32 @@ class Input { this.keyboard = new Guacamole.Keyboard(window); this.keyboard.onkeydown = (keysym) => { this.send("kd," + keysym); + if (!this.keyRepeatQueue.find(keysym)) { + this.keyRepeatQueue.enqueue(keysym); + } }; this.keyboard.onkeyup = (keysym) => { this.send("ku," + keysym); + this.keyRepeatQueue.remove(keysym) }; this._windowMath(); + + this.keyRepeatRunning = true; + this._handleKeyRepeatEvents(); + } + + // A handler function to send key-repeat events for keys that are pressed and kept hold + async _handleKeyRepeatEvents(){ + while (this.keyRepeatRunning) { + var keysyms = this.keyRepeatQueue.toArray(); + + for(var keysym of keysyms){ + this.send("kt," + keysym); + } + + await this.sleep(200); + } } detach() { @@ -628,6 +651,10 @@ class Input { delete this.keyboard; this.send("kr"); } + + // Reset the key-repeat handler + this.keyRepeatQueue.clear(); + this.keyRepeatRunning = false } /** @@ -662,6 +689,14 @@ class Input { parseInt( (() => {var offsetRatioHeight = document.body.offsetHeight * window.devicePixelRatio; return offsetRatioHeight - offsetRatioHeight % 2})() ) ]; } + + async sleep(milliseconds) { + await new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, milliseconds); + }); + } } /** diff --git a/addons/gst-web/src/util.js b/addons/gst-web/src/util.js index 27360092..600eff80 100644 --- a/addons/gst-web/src/util.js +++ b/addons/gst-web/src/util.js @@ -29,4 +29,20 @@ class Queue { return this.items.length===0; } + toArray() { + return [...this.items] + } + + remove(element) { + var index = this.items.indexOf(element) + this.items.splice(index, 1) + } + + find(element) { + return this.items.indexOf(element) == -1 ? false: true; + } + + clear(){ + this.items.length = 0; + } } \ No newline at end of file diff --git a/src/selkies_gstreamer/__main__.py b/src/selkies_gstreamer/__main__.py index 00daa6d1..857c03ee 100644 --- a/src/selkies_gstreamer/__main__.py +++ b/src/selkies_gstreamer/__main__.py @@ -864,6 +864,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config): loop.run_in_executor(None, lambda: turn_rest_mon.start()) loop.run_in_executor(None, lambda: rtc_file_mon.start()) loop.run_in_executor(None, lambda: system_mon.start()) + loop.run_in_executor(None, lambda: webrtc_input.handle_key_repeat()) while True: if using_webrtc_csv: diff --git a/src/selkies_gstreamer/webrtc_input.py b/src/selkies_gstreamer/webrtc_input.py index f4a20acd..3607961b 100644 --- a/src/selkies_gstreamer/webrtc_input.py +++ b/src/selkies_gstreamer/webrtc_input.py @@ -114,6 +114,9 @@ def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipbo self.ping_start = None + # Stores key-repeat keys with arrival time + self.key_repeat_keys = {} + self.on_video_encoder_bit_rate = lambda bitrate: logger.warn( 'unhandled on_video_encoder_bit_rate') self.on_audio_encoder_bit_rate = lambda bitrate: logger.warn( @@ -313,7 +316,7 @@ def send_mouse(self, action, data): else: self.mouse.release(btn) - def send_x11_keypress(self, keysym, down=True): + def send_x11_keypress(self, keysym, down=True, key_repeat=False): """Sends keypress to X server The key sym is converted to a keycode using the X server library. @@ -330,11 +333,36 @@ def send_x11_keypress(self, keysym, down=True): # Although prevented in most cases, this fix may present issues in some keyboard layouts if keysym == 60 and self.keyboard._display.keysym_to_keycode(keysym) == 94: keysym = 44 + + if key_repeat: + # Set or update the timestamp of the key + self.key_repeat_keys[keysym] = time.monotonic() + return + keycode = pynput.keyboard.KeyCode(keysym) if down: self.keyboard.press(keycode) else: self.keyboard.release(keycode) + + if self.key_repeat_keys.get(keysym): + del self.key_repeat_keys[keysym] + + def handle_key_repeat(self): + """Handles key-repeat event keys by monitoring the pressed keys from the + dictionary object and releases those based on the elapsed time + """ + while True: + now = time.monotonic() + + # Iterating over a copy of the data + for key, timeout in tuple(self.key_repeat_keys.items()): + elapsed_time = now - timeout + + # Release the key if elapsed time is over 1.5s + if elapsed_time >= 1.5: + self.send_x11_keypress(key, down=False) + time.sleep(0.2) def send_x11_mouse(self, x, y, button_mask, scroll_magnitude, relative=False): """Sends mouse events to the X server. @@ -587,6 +615,9 @@ def on_message(self, msg): elif toks[0] == "kr": # Keyboard reset self.reset_keyboard() + elif toks[0] == "kt": + # key-repeat events for a key + self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True) elif toks[0] in ["m", "m2"]: # Mouse action # x,y,button_mask