diff --git a/addons/gst-web/src/app.js b/addons/gst-web/src/app.js index 0cc66d44..6edb3d40 100644 --- a/addons/gst-web/src/app.js +++ b/addons/gst-web/src/app.js @@ -578,6 +578,9 @@ window.addEventListener('focus', () => { window.addEventListener('blur', () => { // reset keyboard to avoid stuck keys. webrtc.sendDataChannelMessage("kr"); + + // Clear the key-repeat events on window blur + webrtc.input.keyRepeatQueue.clear(); }); webrtc.onclipboardcontent = (content) => { diff --git a/addons/gst-web/src/input.js b/addons/gst-web/src/input.js index e7d102bb..a2e7878d 100644 --- a/addons/gst-web/src/input.js +++ b/addons/gst-web/src/input.js @@ -129,6 +129,9 @@ class Input { // variable used to scale cursor speed this.cursorScaleFactor = null; + + // keys pressed list to send key repeat events to server + this.keyRepeatQueue = new Queue(); } /** @@ -610,12 +613,32 @@ class Input { this.keyboard = new Guacamole.Keyboard(window); this.keyboard.onkeydown = (keysym) => { this.send("kd," + keysym); + if (!this.keyRepeatQueue.find(keysym)) { + this.keyRepeatQueue.enqueue(keysym); + } }; this.keyboard.onkeyup = (keysym) => { this.send("ku," + keysym); + this.keyRepeatQueue.remove(keysym) }; this._windowMath(); + + this.keyRepeatRunning = true; + this._handleKeyRepeatEvents(); + } + + // A handler function to send key-repeat events for keys that are pressed and kept hold + async _handleKeyRepeatEvents(){ + while (this.keyRepeatRunning) { + var keysyms = this.keyRepeatQueue.toArray(); + + for(var keysym of keysyms){ + this.send("kt," + keysym); + } + + await this.sleep(200); + } } detach() { @@ -628,6 +651,10 @@ class Input { delete this.keyboard; this.send("kr"); } + + // Reset the key-repeat handler + this.keyRepeatQueue.clear(); + this.keyRepeatRunning = false } /** @@ -662,6 +689,14 @@ class Input { parseInt( (() => {var offsetRatioHeight = document.body.offsetHeight * window.devicePixelRatio; return offsetRatioHeight - offsetRatioHeight % 2})() ) ]; } + + async sleep(milliseconds) { + await new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, milliseconds); + }); + } } /** diff --git a/addons/gst-web/src/util.js b/addons/gst-web/src/util.js index 27360092..600eff80 100644 --- a/addons/gst-web/src/util.js +++ b/addons/gst-web/src/util.js @@ -29,4 +29,20 @@ class Queue { return this.items.length===0; } + toArray() { + return [...this.items] + } + + remove(element) { + var index = this.items.indexOf(element) + this.items.splice(index, 1) + } + + find(element) { + return this.items.indexOf(element) == -1 ? false: true; + } + + clear(){ + this.items.length = 0; + } } \ No newline at end of file diff --git a/src/selkies_gstreamer/__main__.py b/src/selkies_gstreamer/__main__.py index 00daa6d1..857c03ee 100644 --- a/src/selkies_gstreamer/__main__.py +++ b/src/selkies_gstreamer/__main__.py @@ -864,6 +864,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config): loop.run_in_executor(None, lambda: turn_rest_mon.start()) loop.run_in_executor(None, lambda: rtc_file_mon.start()) loop.run_in_executor(None, lambda: system_mon.start()) + loop.run_in_executor(None, lambda: webrtc_input.handle_key_repeat()) while True: if using_webrtc_csv: diff --git a/src/selkies_gstreamer/webrtc_input.py b/src/selkies_gstreamer/webrtc_input.py index f4a20acd..3607961b 100644 --- a/src/selkies_gstreamer/webrtc_input.py +++ b/src/selkies_gstreamer/webrtc_input.py @@ -114,6 +114,9 @@ def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipbo self.ping_start = None + # Stores key-repeat keys with arrival time + self.key_repeat_keys = {} + self.on_video_encoder_bit_rate = lambda bitrate: logger.warn( 'unhandled on_video_encoder_bit_rate') self.on_audio_encoder_bit_rate = lambda bitrate: logger.warn( @@ -313,7 +316,7 @@ def send_mouse(self, action, data): else: self.mouse.release(btn) - def send_x11_keypress(self, keysym, down=True): + def send_x11_keypress(self, keysym, down=True, key_repeat=False): """Sends keypress to X server The key sym is converted to a keycode using the X server library. @@ -330,11 +333,36 @@ def send_x11_keypress(self, keysym, down=True): # Although prevented in most cases, this fix may present issues in some keyboard layouts if keysym == 60 and self.keyboard._display.keysym_to_keycode(keysym) == 94: keysym = 44 + + if key_repeat: + # Set or update the timestamp of the key + self.key_repeat_keys[keysym] = time.monotonic() + return + keycode = pynput.keyboard.KeyCode(keysym) if down: self.keyboard.press(keycode) else: self.keyboard.release(keycode) + + if self.key_repeat_keys.get(keysym): + del self.key_repeat_keys[keysym] + + def handle_key_repeat(self): + """Handles key-repeat event keys by monitoring the pressed keys from the + dictionary object and releases those based on the elapsed time + """ + while True: + now = time.monotonic() + + # Iterating over a copy of the data + for key, timeout in tuple(self.key_repeat_keys.items()): + elapsed_time = now - timeout + + # Release the key if elapsed time is over 1.5s + if elapsed_time >= 1.5: + self.send_x11_keypress(key, down=False) + time.sleep(0.2) def send_x11_mouse(self, x, y, button_mask, scroll_magnitude, relative=False): """Sends mouse events to the X server. @@ -587,6 +615,9 @@ def on_message(self, msg): elif toks[0] == "kr": # Keyboard reset self.reset_keyboard() + elif toks[0] == "kt": + # key-repeat events for a key + self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True) elif toks[0] in ["m", "m2"]: # Mouse action # x,y,button_mask