Fix the keyboard key held state occurring due to bad network conditio…

…ns (#146)

* Client sends key-repeat events to the server to indicate deliberate key held state
to distinguish from key held state which might occur under some bad network conditions.

The server releases those keys that are held due to bad network conditions by analysing
the time interval of key-repeat events of the corresponding keys.

* Minor edits

---------

Co-authored-by: Seungmin Kim <[email protected]>

addons/gst-web/src/app.js

@@ -578,6 +578,9 @@ window.addEventListener('focus', () => {
 window.addEventListener('blur', () => {
     // reset keyboard to avoid stuck keys.
     webrtc.sendDataChannelMessage("kr");
+
+    // Clear the key-repeat events on window blur
+    webrtc.input.keyRepeatQueue.clear();
 });
 
 webrtc.onclipboardcontent = (content) => {

addons/gst-web/src/input.js

@@ -129,6 +129,9 @@ class Input {
 
         // variable used to scale cursor speed
         this.cursorScaleFactor = null;
+
+        // keys pressed list to send key repeat events to server
+        this.keyRepeatQueue = new Queue();
     }
 
     /**
@@ -610,12 +613,32 @@ class Input {
         this.keyboard = new Guacamole.Keyboard(window);
         this.keyboard.onkeydown = (keysym) => {
             this.send("kd," + keysym);
+            if (!this.keyRepeatQueue.find(keysym)) {
+                this.keyRepeatQueue.enqueue(keysym);
+            }
         };
         this.keyboard.onkeyup = (keysym) => {
             this.send("ku," + keysym);
+           this.keyRepeatQueue.remove(keysym)
         };
 
         this._windowMath();
+
+        this.keyRepeatRunning = true; 
+        this._handleKeyRepeatEvents();
+    }
+
+    // A handler function to send key-repeat events for keys that are pressed and kept hold
+    async _handleKeyRepeatEvents(){
+        while (this.keyRepeatRunning) {
+            var keysyms = this.keyRepeatQueue.toArray();
+
+            for(var keysym of keysyms){
+                this.send("kt," + keysym);
+            }
+
+            await this.sleep(200);
+        }
     }
 
     detach() {
@@ -628,6 +651,10 @@ class Input {
             delete this.keyboard;
             this.send("kr");
         }
+
+        // Reset the key-repeat handler
+        this.keyRepeatQueue.clear();
+        this.keyRepeatRunning = false
     }
 
     /**
@@ -662,6 +689,14 @@ class Input {
             parseInt( (() => {var offsetRatioHeight = document.body.offsetHeight * window.devicePixelRatio; return offsetRatioHeight - offsetRatioHeight % 2})() )
         ];
     }
+
+    async sleep(milliseconds) {
+        await new Promise((resolve, reject) => {
+            setTimeout(() => {
+                resolve();
+            }, milliseconds);
+        });
+    }
 }
 
 /**

addons/gst-web/src/util.js

@@ -29,4 +29,20 @@ class Queue {
         return this.items.length===0;
     }
 
+    toArray() {
+        return [...this.items]
+    }
+
+    remove(element) {
+        var index = this.items.indexOf(element)
+        this.items.splice(index, 1)
+    }
+
+    find(element) {
+        return this.items.indexOf(element) == -1 ? false: true;
+    }
+
+    clear(){
+        this.items.length = 0;
+    }
 }

src/selkies_gstreamer/__main__.py

@@ -864,6 +864,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config):
         loop.run_in_executor(None, lambda: turn_rest_mon.start())
         loop.run_in_executor(None, lambda: rtc_file_mon.start())
         loop.run_in_executor(None, lambda: system_mon.start())
+        loop.run_in_executor(None, lambda: webrtc_input.handle_key_repeat())
 
         while True:
             if using_webrtc_csv:

src/selkies_gstreamer/webrtc_input.py

@@ -114,6 +114,9 @@ def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipbo
 
         self.ping_start = None
 
+        # Stores key-repeat keys with arrival time
+        self.key_repeat_keys = {}
+
         self.on_video_encoder_bit_rate = lambda bitrate: logger.warn(
             'unhandled on_video_encoder_bit_rate')
         self.on_audio_encoder_bit_rate = lambda bitrate: logger.warn(
@@ -313,7 +316,7 @@ def send_mouse(self, action, data):
                 else:
                     self.mouse.release(btn)
 
-    def send_x11_keypress(self, keysym, down=True):
+    def send_x11_keypress(self, keysym, down=True, key_repeat=False):
         """Sends keypress to X server
 
         The key sym is converted to a keycode using the X server library.
@@ -330,11 +333,36 @@ def send_x11_keypress(self, keysym, down=True):
         # Although prevented in most cases, this fix may present issues in some keyboard layouts
         if keysym == 60 and self.keyboard._display.keysym_to_keycode(keysym) == 94:
             keysym = 44
+
+        if key_repeat:
+            # Set or update the timestamp of the key
+            self.key_repeat_keys[keysym] = time.monotonic()
+            return
+
         keycode = pynput.keyboard.KeyCode(keysym)
         if down:
             self.keyboard.press(keycode)
         else:
             self.keyboard.release(keycode)
+
+            if self.key_repeat_keys.get(keysym):
+                del self.key_repeat_keys[keysym]
+
+    def handle_key_repeat(self):
+        """Handles key-repeat event keys by monitoring the pressed keys from the 
+           dictionary object and releases those based on the elapsed time
+        """
+        while True:
+            now = time.monotonic()
+
+            # Iterating over a copy of the data
+            for key, timeout in tuple(self.key_repeat_keys.items()):
+                elapsed_time = now - timeout
+
+                # Release the key if elapsed time is over 1.5s 
+                if elapsed_time >= 1.5:
+                    self.send_x11_keypress(key, down=False)
+            time.sleep(0.2)
 
     def send_x11_mouse(self, x, y, button_mask, scroll_magnitude, relative=False):
         """Sends mouse events to the X server.
@@ -587,6 +615,9 @@ def on_message(self, msg):
         elif toks[0] == "kr":
             # Keyboard reset
             self.reset_keyboard()
+        elif toks[0] == "kt":
+            # key-repeat events for a key
+            self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True)
         elif toks[0] in ["m", "m2"]:
             # Mouse action
             # x,y,button_mask