feat(input): Added unicode input to ironrdp-web and ironrdp-input (#374)

crates/ironrdp-client/src/gui.rs

@@ -78,7 +78,22 @@ impl GuiContext {
                         // TODO(#110): File upload
                     }
                     WindowEvent::ReceivedCharacter(_) => {
-                        // TODO(#106): Unicode mode
+                        // Sadly, we can't use this winit event to send RDP unicode events because
+                        // of the several reasons:
+                        // 1. `ReceivedCharacter` event doesn't provide a way to distinguish between
+                        //    key press and key release, therefore the only way to use it is to send
+                        //    a key press + release events sequentially, which will not allow to
+                        //    handle long press and key repeat events.
+                        // 2. This event do not fire for non-printable keys (e.g. Control, Alt, etc.)
+                        // 3. This event fies BEFORE `KeyboardInput` event, so we can't make a
+                        //    reasonable workaround for `1` and `2` by collecting physical key press
+                        //    information first via `KeyboardInput` before processing `ReceivedCharacter`.
+                        //
+                        // However, all of these issues can be solved by updating `winit` to the
+                        // newer version.
+                        //
+                        // TODO(#376): Update winit
+                        // TODO(#376): Implement unicode input in native client
                     }
                     WindowEvent::KeyboardInput { input, .. } => {
                         let scancode = ironrdp::input::Scancode::from_u16(u16::try_from(input.scancode).unwrap());

crates/ironrdp-input/src/lib.rs

@@ -5,8 +5,7 @@ use ironrdp_pdu::input::mouse::PointerFlags;
 use ironrdp_pdu::input::mouse_x::PointerXFlags;
 use ironrdp_pdu::input::{MousePdu, MouseXPdu};
 use smallvec::SmallVec;
-
-// TODO(#106): unicode keyboard event support
+use std::collections::BTreeSet;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[repr(u8)]
@@ -136,6 +135,8 @@ pub enum Operation {
     WheelRotations(WheelRotations),
     KeyPressed(Scancode),
     KeyReleased(Scancode),
+    UnicodeKeyPressed(char),
+    UnicodeKeyReleased(char),
 }
 
 pub type KeyboardState = BitArr!(for 512);
@@ -143,6 +144,7 @@ pub type MouseButtonsState = BitArr!(for 5);
 
 /// In-memory database for maintaining the current keyboard and mouse state.
 pub struct Database {
+    unicode_keyboard_state: BTreeSet<char>,
     keyboard: KeyboardState,
     mouse_buttons: MouseButtonsState,
     mouse_position: MousePosition,
@@ -160,9 +162,14 @@ impl Database {
             keyboard: BitArray::ZERO,
             mouse_buttons: BitArray::ZERO,
             mouse_position: MousePosition { x: 0, y: 0 },
+            unicode_keyboard_state: BTreeSet::new(),
         }
     }
 
+    pub fn is_unicode_key_pressed(&self, character: char) -> bool {
+        self.unicode_keyboard_state.contains(&character)
+    }
+
     pub fn is_key_pressed(&self, scancode: Scancode) -> bool {
         self.keyboard
             .get(scancode.as_idx())
@@ -293,6 +300,34 @@ impl Database {
                         events.push(FastPathInputEvent::KeyboardEvent(flags, scancode.code));
                     }
                 }
+                Operation::UnicodeKeyPressed(character) => {
+                    let was_pressed = !self.unicode_keyboard_state.insert(character);
+
+                    let mut utf16_buffer = [0u16; 2];
+                    let utf16_code_units = character.encode_utf16(&mut utf16_buffer);
+
+                    if was_pressed {
+                        for code in utf16_code_units.iter() {
+                            events.push(FastPathInputEvent::UnicodeKeyboardEvent(KeyboardFlags::RELEASE, *code));
+                        }
+                    }
+
+                    for code in utf16_code_units {
+                        events.push(FastPathInputEvent::UnicodeKeyboardEvent(KeyboardFlags::empty(), *code));
+                    }
+                }
+                Operation::UnicodeKeyReleased(character) => {
+                    let was_pressed = self.unicode_keyboard_state.remove(&character);
+
+                    let mut utf16_buffer = [0u16; 2];
+                    let utf16_code_units = character.encode_utf16(&mut utf16_buffer);
+
+                    if was_pressed {
+                        for code in utf16_code_units {
+                            events.push(FastPathInputEvent::UnicodeKeyboardEvent(KeyboardFlags::RELEASE, *code));
+                        }
+                    }
+                }
             }
         }
 
@@ -340,6 +375,15 @@ impl Database {
             events.push(FastPathInputEvent::KeyboardEvent(flags, scancode));
         }
 
+        for character in std::mem::take(&mut self.unicode_keyboard_state).into_iter() {
+            let mut utf16_buffer = [0u16; 2];
+            let utf16_code_units = character.encode_utf16(&mut utf16_buffer);
+
+            for code in utf16_code_units {
+                events.push(FastPathInputEvent::UnicodeKeyboardEvent(KeyboardFlags::RELEASE, *code));
+            }
+        }
+
         self.mouse_buttons = BitArray::ZERO;
         self.keyboard = BitArray::ZERO;
 

crates/ironrdp-web/src/input.rs

@@ -46,6 +46,14 @@ impl DeviceEvent {
     pub fn new_key_released(scancode: u16) -> Self {
         Self(Operation::KeyReleased(Scancode::from_u16(scancode)))
     }
+
+    pub fn new_unicode_pressed(unicode: char) -> Self {
+        Self(Operation::UnicodeKeyPressed(unicode))
+    }
+
+    pub fn new_unicode_released(unicode: char) -> Self {
+        Self(Operation::UnicodeKeyReleased(unicode))
+    }
 }
 
 #[wasm_bindgen]

crates/ironrdp-web/src/session.rs

@@ -710,6 +710,13 @@ impl Session {
 
         Ok(())
     }
+
+    #[allow(clippy::unused_self)]
+    pub fn supports_unicode_keyboard_shortcuts(&self) -> bool {
+        // RDP does not support Unicode keyboard shortcuts (When key combinations are executed, only
+        // plain scancode events are allowed to function correctly).
+        false
+    }
 }
 
 fn build_config(

web-client/iron-remote-gui/src/interfaces/UserInteraction.ts

@@ -21,6 +21,8 @@ export interface UserInteraction {
         kdc_proxy_url?: string,
     ): Observable<NewSessionInfo>;
 
+    setKeyboardUnicodeMode(use_unicode: boolean): void;
+
     ctrlAltDel(): void;
 
     metaKey(): void;

web-client/iron-remote-gui/src/services/PublicAPI.ts

@@ -60,6 +60,10 @@ export class PublicAPI {
         this.wasmService.shutdown();
     }
 
+    private setKeyboardUnicodeMode(use_unicode: boolean) {
+        this.wasmService.setKeyboardUnicodeMode(use_unicode);
+    }
+
     getExposedFunctions(): UserInteraction {
         return {
             setVisibility: this.setVisibility.bind(this),
@@ -69,6 +73,7 @@ export class PublicAPI {
             ctrlAltDel: this.ctrlAltDel.bind(this),
             metaKey: this.metaKey.bind(this),
             shutdown: this.shutdown.bind(this),
+            setKeyboardUnicodeMode: this.setKeyboardUnicodeMode.bind(this),
         };
     }
 }

web-client/iron-remote-gui/src/services/wasm-bridge.service.ts

@@ -41,6 +41,8 @@ export class WasmBridgeService {
     private scale: BehaviorSubject<ScreenScale> = new BehaviorSubject(ScreenScale.Fit as ScreenScale);
     private canvas?: HTMLCanvasElement;
     private keyboardActive: boolean = false;
+    private keyboardUnicodeMode: boolean = false;
+    private backendSupportsUnicodeKeyboardShortcuts: boolean | undefined = undefined;
     private onRemoteClipboardChanged?: OnRemoteClipboardChanged;
     private onRemoteReceivedFormatList?: OnRemoteReceivedFormatsList;
     private onForceClipboardUpdate?: OnForceClipboardUpdate;
@@ -258,35 +260,90 @@ export class WasmBridgeService {
         return onClipboardChangedPromise();
     }
 
+    setKeyboardUnicodeMode(use_unicode: boolean) {
+        this.keyboardUnicodeMode = use_unicode;
+    }
+
     private releaseAllInputs() {
         this.session?.release_all_inputs();
     }
 
+    private supportsUnicodeKeyboardShortcuts(): boolean {
+        // Use cached value to reduce FFI calls
+        if (this.backendSupportsUnicodeKeyboardShortcuts !== undefined) {
+            return this.backendSupportsUnicodeKeyboardShortcuts;
+        }
+
+        if (this.session?.supports_unicode_keyboard_shortcuts) {
+            this.backendSupportsUnicodeKeyboardShortcuts = this.session?.supports_unicode_keyboard_shortcuts();
+            return this.backendSupportsUnicodeKeyboardShortcuts;
+        }
+
+        // By default we use unicode keyboard shortcuts for backends
+        return true;
+    }
+
     private sendKeyboard(evt: KeyboardEvent) {
         evt.preventDefault();
 
         let keyEvent;
+        let unicodeEvent;
 
         if (evt.type === 'keydown') {
             keyEvent = DeviceEvent.new_key_pressed;
+            unicodeEvent = DeviceEvent.new_unicode_pressed;
         } else if (evt.type === 'keyup') {
             keyEvent = DeviceEvent.new_key_released;
+            unicodeEvent = DeviceEvent.new_unicode_released;
         }
 
-        if (keyEvent) {
-            const isModifierKey = evt.code in ModifierKey;
-            const isLockKey = evt.code in LockKey;
+        let sendAsUnicode = true;
 
-            if (isModifierKey) {
-                this.updateModifierKeyState(evt);
+        if (!this.supportsUnicodeKeyboardShortcuts()) {
+            for (const modifier of ['Alt', 'Control', 'Meta', 'AltGraph', 'OS']) {
+                if (evt.getModifierState(modifier)) {
+                    sendAsUnicode = false;
+                    break;
+                }
+            }
+        }
+
+        const isModifierKey = evt.code in ModifierKey;
+        const isLockKey = evt.code in LockKey;
+
+        if (isModifierKey) {
+            this.updateModifierKeyState(evt);
+        }
+
+        if (isLockKey) {
+            this.syncModifier(evt);
+        }
+
+        if (!evt.repeat || (!isModifierKey && !isLockKey)) {
+            const keyScanCode = scanCode(evt.code, OS.WINDOWS);
+            const unknownScanCode = Number.isNaN(keyScanCode);
+
+            if (!this.keyboardUnicodeMode && keyEvent && !unknownScanCode) {
+                this.doTransactionFromDeviceEvents([keyEvent(keyScanCode)]);
+                return;
             }
 
-            if (isLockKey) {
-                this.syncModifier(evt);
-            }
+            if (this.keyboardUnicodeMode && unicodeEvent && keyEvent) {
+                // `Dead` and `Unidentified` keys should be ignored
+                if (evt.key in ['Dead', 'Unidentified']) {
+                    return;
+                }
 
-            if (!evt.repeat || (!isModifierKey && !isLockKey)) {
-                this.doTransactionFromDeviceEvents([keyEvent(scanCode(evt.code, OS.WINDOWS))]);
+                const keyCode = scanCode(evt.key, OS.WINDOWS);
+                const isUnicodeCharacter = Number.isNaN(keyCode) && evt.key.length === 1;
+
+                if (isUnicodeCharacter && sendAsUnicode) {
+                    this.doTransactionFromDeviceEvents([unicodeEvent(evt.key)]);
+                } else if (!unknownScanCode) {
+                    // Use scancode insdead of key code for non-unicode character values
+                    this.doTransactionFromDeviceEvents([keyEvent(keyScanCode)]);
+                }
+                return;
             }
         }
     }

web-client/iron-svelte-client/src/lib/remote-screen/remote-screen.svelte

@@ -20,6 +20,20 @@
         }
     });
 
+    function onUnicodeModeChange(e: MouseEvent) {
+        if (e.target == null) {
+            return;
+        }
+
+        let element = e.target as HTMLInputElement;
+
+        if (element == null) {
+            return;
+        }
+
+        uiService.setKeyboardUnicodeMode(element.checked);
+    }
+
     onMount(async () => {
         let el = document.querySelector('iron-remote-gui');
 
@@ -51,6 +65,10 @@
             </svg>
         </button>
         <button on:click={() => uiService.shutdown()}>Terminate Session</button>
+        <label style="color: white;">
+            <input on:click={(e) => onUnicodeModeChange(e)} type="checkbox" />
+            Unicode keyboard mode
+        </label>
     </div>
     <iron-remote-gui debugwasm="INFO" verbose="true" scale="fit" flexcenter="true" />
 </div>