Fix the keyboard key held state occurring due to bad network conditions

addons/gst-web/src/app.js

@@ -578,6 +578,9 @@ window.addEventListener('focus', () => {
 window.addEventListener('blur', () => {
     // reset keyboard to avoid stuck keys.
     webrtc.sendDataChannelMessage("kr");
+
+    // Clear the key-repeat events on window blur
+    webrtc.input.keyRepeatQueue.clear();
 });
 
 webrtc.onclipboardcontent = (content) => {

addons/gst-web/src/input.js

@@ -129,6 +129,9 @@ class Input {
 
         // variable used to scale cursor speed
         this.cursorScaleFactor = null;
+
+        // keys pressed list to send key repeat events to server
+        this.keyRepeatQueue = new Queue();
     }
 
     /**
@@ -610,12 +613,32 @@ class Input {
         this.keyboard = new Guacamole.Keyboard(window);
         this.keyboard.onkeydown = (keysym) => {
             this.send("kd," + keysym);
+            if (!this.keyRepeatQueue.find(keysym)) {
+                this.keyRepeatQueue.enqueue(keysym);
+            }
         };
         this.keyboard.onkeyup = (keysym) => {
             this.send("ku," + keysym);
+           this.keyRepeatQueue.remove(keysym)
         };
 
         this._windowMath();
+
+        this.keyRepeatRunning = true; 
+        this._handleKeyRepeatEvents();
+    }
+
+    // A handler function to send key-repeat events for keys that are pressed and kept hold
+    async _handleKeyRepeatEvents(){
+        while (this.keyRepeatRunning) {
+            var keysyms = this.keyRepeatQueue.toArray();
+
+            for(var keysym of keysyms){
+                this.send("kt," + keysym);
+            }
+
+            await this.sleep(200);
+        }
     }
 
     detach() {
@@ -628,6 +651,10 @@ class Input {
             delete this.keyboard;
             this.send("kr");
         }
+
+        // Reset the key-repeat handler
+        this.keyRepeatQueue.clear();
+        this.keyRepeatRunning = false
     }
 
     /**
@@ -662,6 +689,14 @@ class Input {
             parseInt( (() => {var offsetRatioHeight = document.body.offsetHeight * window.devicePixelRatio; return offsetRatioHeight - offsetRatioHeight % 2})() )
         ];
     }
+
+    async sleep(milliseconds) {
+        await new Promise((resolve, reject) => {
+            setTimeout(() => {
+                resolve();
+            }, milliseconds);
+        });
+    }
 }
 
 /**

addons/gst-web/src/util.js

@@ -29,4 +29,20 @@ class Queue {
         return this.items.length===0;
     }
 
+    toArray() {
+        return [...this.items]
+    }
+
+    remove(element) {
+        var index = this.items.indexOf(element)
+        this.items.splice(index, 1)
+    }
+
+    find(element) {
+        return this.items.indexOf(element) == -1 ? false: true;
+    }
+
+    clear(){
+        this.items.length = 0;
+    }
 }

src/selkies_gstreamer/__main__.py

@@ -864,6 +864,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config):
         loop.run_in_executor(None, lambda: turn_rest_mon.start())
         loop.run_in_executor(None, lambda: rtc_file_mon.start())
         loop.run_in_executor(None, lambda: system_mon.start())
+        loop.run_in_executor(None, lambda: webrtc_input.handle_key_repeat())
 
         while True:
             if using_webrtc_csv:

src/selkies_gstreamer/webrtc_input.py

@@ -114,6 +114,9 @@ def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipbo
 
         self.ping_start = None
 
+        # Stores key-repeat keys with arrival time
+        self.key_repeat_keys = {}
+
         self.on_video_encoder_bit_rate = lambda bitrate: logger.warn(
             'unhandled on_video_encoder_bit_rate')
         self.on_audio_encoder_bit_rate = lambda bitrate: logger.warn(
@@ -313,7 +316,7 @@ def send_mouse(self, action, data):
                 else:
                     self.mouse.release(btn)
 
-    def send_x11_keypress(self, keysym, down=True):
+    def send_x11_keypress(self, keysym, down=True, key_repeat=False):
         """Sends keypress to X server
 
         The key sym is converted to a keycode using the X server library.
@@ -330,11 +333,36 @@ def send_x11_keypress(self, keysym, down=True):
         # Although prevented in most cases, this fix may present issues in some keyboard layouts
         if keysym == 60 and self.keyboard._display.keysym_to_keycode(keysym) == 94:
             keysym = 44
+
+        if key_repeat:
+            # Set or update the timestamp of the key
+            self.key_repeat_keys[keysym] = time.monotonic()
+            return
+
         keycode = pynput.keyboard.KeyCode(keysym)
         if down:
             self.keyboard.press(keycode)
         else:
             self.keyboard.release(keycode)
+
+            if self.key_repeat_keys.get(keysym):
+                del self.key_repeat_keys[keysym]
+
+    def handle_key_repeat(self):
+        """Handles key-repeat event keys by monitoring the pressed keys from the 
+           dictionary object and releases those based on the elapsed time
+        """
+        while True:
+            now = time.monotonic()
+
+            # Iterating over a copy of the data
+            for key, timeout in tuple(self.key_repeat_keys.items()):
+                elapsed_time = now - timeout
+
+                # Release the key if elapsed time is over 1.5s 
+                if elapsed_time >= 1.5:
+                    self.send_x11_keypress(key, down=False)
+            time.sleep(0.2)
 
     def send_x11_mouse(self, x, y, button_mask, scroll_magnitude, relative=False):
         """Sends mouse events to the X server.
@@ -587,6 +615,9 @@ def on_message(self, msg):
         elif toks[0] == "kr":
             # Keyboard reset
             self.reset_keyboard()
+        elif toks[0] == "kt":
+            # key-repeat events for a key
+            self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True)
         elif toks[0] in ["m", "m2"]:
             # Mouse action
             # x,y,button_mask