Fix the keyboard key held state occurring due to bad network conditions

addons/gst-web/src/app.js

@@ -578,6 +578,9 @@ window.addEventListener('focus', () => {
 window.addEventListener('blur', () => {
  // reset keyboard to avoid stuck keys.
  webrtc.sendDataChannelMessage("kr");
+
+ // Clear the key-repeat events on window blur
+ webrtc.input.keyRepeatQueue.clear();
 });
 
 webrtc.onclipboardcontent = (content) => {

addons/gst-web/src/input.js

@@ -129,6 +129,9 @@ class Input {
 
  // variable used to scale cursor speed
  this.cursorScaleFactor = null;
+
+ // keys pressed list to send key repeat events to server
+ this.keyRepeatQueue = new Queue();
  }
 
  /**
@@ -610,12 +613,32 @@ class Input {
  this.keyboard = new Guacamole.Keyboard(window);
  this.keyboard.onkeydown = (keysym) => {
  this.send("kd," + keysym);
+ if (!this.keyRepeatQueue.find(keysym)) {
+ this.keyRepeatQueue.enqueue(keysym);
+ }
  };
  this.keyboard.onkeyup = (keysym) => {
  this.send("ku," + keysym);
+ this.keyRepeatQueue.remove(keysym)
  };
 
  this._windowMath();
+
+ this.keyRepeatRunning = true; 
+ this._handleKeyRepeatEvents();
+ }
+
+ // A handler function to send key-repeat events for keys that are pressed and kept hold
+ async _handleKeyRepeatEvents(){
+ while (this.keyRepeatRunning) {
+ var keysyms = this.keyRepeatQueue.toArray();
+
+ for(var keysym of keysyms){
+ this.send("kt," + keysym);
+ }
+
+ await this.sleep(200);
+ }
  }
 
  detach() {
@@ -628,6 +651,10 @@ class Input {
  delete this.keyboard;
  this.send("kr");
  }
+
+ // Reset the key-repeat handler
+ this.keyRepeatQueue.clear();
+ this.keyRepeatRunning = false
  }
 
  /**
@@ -662,6 +689,14 @@ class Input {
  parseInt( (() => {var offsetRatioHeight = document.body.offsetHeight * window.devicePixelRatio; return offsetRatioHeight - offsetRatioHeight % 2})() )
  ];
  }
+
+ async sleep(milliseconds) {
+ await new Promise((resolve, reject) => {
+ setTimeout(() => {
+ resolve();
+ }, milliseconds);
+ });
+ }
 }
 
 /**

addons/gst-web/src/util.js

@@ -29,4 +29,20 @@ class Queue {
  return this.items.length===0;
  }
 
+ toArray() {
+ return [...this.items]
+ }
+
+ remove(element) {
+ var index = this.items.indexOf(element)
+ this.items.splice(index, 1)
+ }
+
+ find(element) {
+ return this.items.indexOf(element) == -1 ? false: true;
+ }
+
+ clear(){
+ this.items.length = 0;
+ }
 }

src/selkies_gstreamer/__main__.py

@@ -864,6 +864,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config):
  loop.run_in_executor(None, lambda: turn_rest_mon.start())
  loop.run_in_executor(None, lambda: rtc_file_mon.start())
  loop.run_in_executor(None, lambda: system_mon.start())
+ loop.run_in_executor(None, lambda: webrtc_input.handle_key_repeat())
 
  while True:
  if using_webrtc_csv:

src/selkies_gstreamer/webrtc_input.py

@@ -114,6 +114,9 @@ def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipbo
 
  self.ping_start = None
 
+ # Stores key-repeat keys with arrival time
+ self.key_repeat_keys = {}
+
  self.on_video_encoder_bit_rate = lambda bitrate: logger.warn(
  'unhandled on_video_encoder_bit_rate')
  self.on_audio_encoder_bit_rate = lambda bitrate: logger.warn(
@@ -313,7 +316,7 @@ def send_mouse(self, action, data):
  else:
  self.mouse.release(btn)
 
- def send_x11_keypress(self, keysym, down=True):
+ def send_x11_keypress(self, keysym, down=True, key_repeat=False):
  """Sends keypress to X server
 
  The key sym is converted to a keycode using the X server library.
@@ -330,11 +333,36 @@ def send_x11_keypress(self, keysym, down=True):
  # Although prevented in most cases, this fix may present issues in some keyboard layouts
  if keysym == 60 and self.keyboard._display.keysym_to_keycode(keysym) == 94:
  keysym = 44
+
+ if key_repeat:
+ # Set or update the timestamp of the key
+ self.key_repeat_keys[keysym] = time.monotonic()
+ return
+
  keycode = pynput.keyboard.KeyCode(keysym)
  if down:
  self.keyboard.press(keycode)
  else:
  self.keyboard.release(keycode)
+
+ if self.key_repeat_keys.get(keysym):
+ del self.key_repeat_keys[keysym]
+
+ def handle_key_repeat(self):
+ """Handles key-repeat event keys by monitoring the pressed keys from the 
+ dictionary object and releases those based on the elapsed time
+ """
+ while True:
+ now = time.monotonic()
+
+ # Iterating over a copy of the data
+ for key, timeout in tuple(self.key_repeat_keys.items()):
+ elapsed_time = now - timeout
+
+ # Release the key if elapsed time is over 1.5s 
+ if elapsed_time >= 1.5:
+ self.send_x11_keypress(key, down=False)
+ time.sleep(0.2)
 
  def send_x11_mouse(self, x, y, button_mask, scroll_magnitude, relative=False):
  """Sends mouse events to the X server.
@@ -587,6 +615,9 @@ def on_message(self, msg):
  elif toks[0] == "kr":
  # Keyboard reset
  self.reset_keyboard()
+ elif toks[0] == "kt":
+ # key-repeat events for a key
+ self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True)
  elif toks[0] in ["m", "m2"]:
  # Mouse action
  # x,y,button_mask