From 760997e772ede025169964e287034e0797606c7c Mon Sep 17 00:00:00 2001 From: pallaoro Date: Mon, 9 Nov 2026 22:04:13 +0100 Subject: [PATCH] Type non-ASCII text on keycodes of its own on the remote desktop xdotool binds every character missing from the keymap to one scratch keycode and clears it 3 ms after the key press. An app that looks the key up later reads an empty or a newer binding, so long multilingual text lost characters (up to 137 of 1237 in a 4000-character call). Before typing, desktop-control now binds each such character to an empty keycode of its own with one xmodmap call, so xdotool finds it in the keymap and never rebinds. Bindings stay until a later character needs the keycode and are released when the service stops. Text with more distinct characters than free keycodes (17 on Xvfb) is typed in runs, reusing the keycode pressed longest ago after a 100 ms pause. The browser installer now installs x11-xserver-utils for xmodmap. Without it, typing falls back to xdotool's own rebinding. --- packages/desktop-control/README.md | 14 +- packages/desktop-control/src/server.ts | 1 + packages/desktop-control/src/x11.ts | 164 ++++++++++++++++-- packages/desktop-control/test/linux-smoke.mjs | 10 +- packages/desktop-control/test/x11.test.ts | 81 +++++++++ packages/server/scripts/install-browser.sh | 2 +- 6 files changed, 250 insertions(+), 22 deletions(-) create mode 100644 packages/desktop-control/test/x11.test.ts diff --git a/packages/desktop-control/README.md b/packages/desktop-control/README.md index 7e2d5917..45fd73ba 100644 --- a/packages/desktop-control/README.md +++ b/packages/desktop-control/README.md @@ -7,8 +7,9 @@ configuration, and a panel that opens its viewer. ## Run Requires Node 22+, Xvfb (or another X11 display), a window manager, `xdotool`, -`scrot`, `x11vnc`, and noVNC's browser modules at `/usr/share/novnc`. The VNC -server must use RFB 3.8 with VNC password authentication on loopback port 5900. +`xmodmap`, `scrot`, `x11vnc`, and noVNC's browser modules at +`/usr/share/novnc`. The VNC server must use RFB 3.8 with VNC password +authentication on loopback port 5900. Wayland and control of a user's macOS/Windows desktop are outside this version. ```sh @@ -51,9 +52,12 @@ A second agent must wait; there is no agent force-takeover operation. PNG pixels, regardless of how the viewer scales the display. `key` accepts combinations such as `Control+s` and `Shift+Tab`. `type` sends real key events, including explicit Return/Tab events, with a 4000-character limit. LF/CRLF are -normalized to Return. It is typing, so the focused application's shortcuts, -autoindent, and text transformations still apply. Scroll amounts are wheel -steps (1–20) at the current pointer position. +normalized to Return. Characters missing from the keyboard map (accents, Greek, +CJK) are bound to free keycodes with `xmodmap` before typing and stay bound +until a later character needs the keycode; the service unbinds them when it +stops. It is typing, so the focused application's shortcuts, autoindent, and +text transformations still apply. Scroll amounts are wheel steps (1–20) at the +current pointer position. The viewer fills its frame and accepts mouse and keyboard input directly, without a takeover panel. A keyboard button on touch devices opens a text field for the diff --git a/packages/desktop-control/src/server.ts b/packages/desktop-control/src/server.ts index 363ef075..f698d783 100644 --- a/packages/desktop-control/src/server.ts +++ b/packages/desktop-control/src/server.ts @@ -211,6 +211,7 @@ export function createDesktopServer(opts: { server, close() { control.close(); + desktop.close(); for (const v of viewers.values()) v.socket.terminate(); wss.close(); for (const s of servers) s.close(); diff --git a/packages/desktop-control/src/x11.ts b/packages/desktop-control/src/x11.ts index 8644c770..54b783ff 100644 --- a/packages/desktop-control/src/x11.ts +++ b/packages/desktop-control/src/x11.ts @@ -2,6 +2,7 @@ import { spawn } from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; export function run( file: string, @@ -72,7 +73,99 @@ export function keySequence(value: unknown): string { .join("+"); } +/** `xmodmap -pk` → keycode → its non-empty keysyms. */ +export function parseKeymap(out: string): Map { + const keymap = new Map(); + for (const line of out.split("\n")) { + const code = /^\s*(\d+)(?:\s|$)/.exec(line)?.[1]; + if (code) + keymap.set(Number(code), (line.match(/0x[0-9a-f]+/gi) ?? []).map(Number).filter(Boolean)); + } + return keymap; +} + +export type KeyRun = { text: string; map: [keycode: number, keysym: number][]; overwrite: boolean }; + +/** + * xdotool types a character missing from the keymap by rebinding ONE scratch + * keycode for every key press and clearing it 3 ms later. An application that + * reads the keymap after that has moved on types nothing or a neighbour; a + * 4000-character multilingual text lost up to 137 of its 1237 non-ASCII characters. + * Instead, each such character gets an empty keycode of its own before xdotool + * runs, so xdotool finds it in the keymap and never rebinds. Text needing more + * keycodes than are free is split into runs; a run that rebinds a keycode + * already in use is marked `overwrite` so the caller can let the app catch up. + */ +export function planKeys( + text: string, + keymap: Map, + owned: Map, +): KeyRun[] { + const present = new Set(); + const empty: number[] = []; + for (const [code, syms] of keymap) { + if (owned.has(code)) continue; + if (!syms.length) empty.push(code); + for (const sym of syms) present.add(sym); + } + empty.sort((a, b) => a - b); + // Leave the lowest empty keycode to xdotool's scratch binding and the + // highest to x11vnc's keysym additions for a viewer. + const slots = [...empty.slice(1, -1), ...owned.keys()]; + const chars = Array.from(text); + // The keysym xdotool itself would bind (xdo.c, _xdo_keysym_from_char). + const syms = chars.map((c) => { + const cp = c.codePointAt(0) ?? 0; + const sym = cp >= 0x100 ? 0x1000000 + cp : cp; + return cp >= 0xa0 && !present.has(sym) ? sym : 0; + }); + if (!slots.length || !syms.some(Boolean)) return [{ text, map: [], overwrite: false }]; + const held = new Map(owned); + // Rebind empty keycodes first, then the ones whose last key press is oldest. + const lastUse = new Map(); + const age = (code: number) => { + const sym = held.get(code); + return sym === undefined ? -2 : (lastUse.get(sym) ?? -1); + }; + const runs: KeyRun[] = []; + for (let start = 0; start < chars.length; ) { + const need = new Set(); + let end = start; + for (; end < chars.length; end++) { + const sym = syms[end]; + if (!sym || need.has(sym)) continue; + if (need.size === slots.length) break; + need.add(sym); + } + const holding = new Set(held.values()); + const spare = slots + .filter((code) => !need.has(held.get(code) ?? 0)) + .sort((a, b) => age(a) - age(b)); + const map: KeyRun["map"] = []; + let overwrite = false; + for (const sym of need) { + if (holding.has(sym)) continue; + const code = spare.shift() as number; + overwrite ||= held.has(code); + held.set(code, sym); + map.push([code, sym]); + } + runs.push({ text: chars.slice(start, end).join(""), map, overwrite }); + for (; start < end; start++) if (syms[start]) lastUse.set(syms[start] as number, start); + } + return runs; +} + +// shortcut: a fixed pause before reusing a keycode. Measured on a 1-vCPU box, +// no pause at all lost nothing; 100 ms is margin for a busier app. An app that +// lags longer can still misread the end of the previous run. Binding a second +// keysym per keycode (Shift level) would halve the reuses if this gets slow. +const REBIND_SETTLE_MS = 100; +let warnedNoXmodmap = false; + export class X11Desktop { + /** Keycodes this process bound for typing → keysym. Kept between calls. */ + private readonly owned = new Map(); constructor(private readonly display = ":99") {} async screenshot(): Promise { // Ubuntu's scrot/imlib cannot save to Node's socket-backed stdout pipe. @@ -105,25 +198,41 @@ export class X11Desktop { "text must be at most 4000 characters, with only tab/newline control characters.", ); try { - // LF is not Return in libxdo. Send newlines explicitly. Its dynamic - // Unicode keymaps also need the normal 12ms delay, or GTK loses keys. - for (const part of args.text.replace(/\r\n?/g, "\n").split(/([\n\t])/)) { - if (!part) continue; - if (part === "\n" || part === "\t") - await run( - "xdotool", - ["key", "--clearmodifiers", part === "\n" ? "Return" : "Tab"], - this.display, - signal, - ); - else + for (const chunk of await this.keyRuns(args.text.replace(/\r\n?/g, "\n"), signal)) { + if (chunk.map.length) { + if (chunk.overwrite) await sleep(REBIND_SETTLE_MS, undefined, { signal }); + // Claimed first: the next call drops any an interrupted xmodmap missed. + for (const [code, sym] of chunk.map) this.owned.set(code, sym); await run( - "xdotool", - ["type", "--clearmodifiers", "--delay", "12", "--file", "-"], + "xmodmap", + ["-"], this.display, signal, - part, + chunk.map + .map(([code, sym]) => `keycode ${code} = 0x${sym.toString(16)}`) + .join("\n"), ); + } + // LF is not Return in libxdo. Send newlines explicitly. Keep the + // normal 12ms delay: faster typing loses keys in GTK. + for (const part of chunk.text.split(/([\n\t])/)) { + if (!part) continue; + if (part === "\n" || part === "\t") + await run( + "xdotool", + ["key", "--clearmodifiers", part === "\n" ? "Return" : "Tab"], + this.display, + signal, + ); + else + await run( + "xdotool", + ["type", "--clearmodifiers", "--delay", "12", "--file", "-"], + this.display, + signal, + part, + ); + } } } catch (error) { await this.resetInput(); @@ -195,6 +304,31 @@ export class X11Desktop { throw error; } } + private async keyRuns(text: string, signal: AbortSignal): Promise { + if (!/[\u00a0-\u{10ffff}]/u.test(text)) return [{ text, map: [], overwrite: false }]; + let keymap: Map; + try { + keymap = parseKeymap((await run("xmodmap", ["-pk"], this.display, signal)).toString()); + } catch (error) { + if (signal.aborted) throw error; + // xdotool still types, with its per-character rebinding race. + if (!warnedNoXmodmap) + console.error(`Typing without reserved keycodes: ${(error as Error).message}`); + warnedNoXmodmap = true; + return [{ text, map: [], overwrite: false }]; + } + // A restarted display starts from its default keymap. + for (const [code, sym] of this.owned) + if (!keymap.get(code)?.includes(sym)) this.owned.delete(code); + return planKeys(text, keymap, this.owned); + } + /** Hand the keycodes bound for typing back to the display. */ + close(): void { + if (!this.owned.size) return; + const clear = [...this.owned.keys()].map((code) => `keycode ${code} =`).join("\n"); + this.owned.clear(); + run("xmodmap", ["-"], this.display, undefined, clear).catch(() => {}); + } async resetInput(): Promise { await run( "xdotool", diff --git a/packages/desktop-control/test/linux-smoke.mjs b/packages/desktop-control/test/linux-smoke.mjs index d659e289..f47b0217 100644 --- a/packages/desktop-control/test/linux-smoke.mjs +++ b/packages/desktop-control/test/linux-smoke.mjs @@ -106,8 +106,16 @@ try { assert(png); await writeFile(join(scratch, "screen.png"), Buffer.from(png.data, "base64")); assert.equal(actual, expected); + // Short text survives xdotool's own rebinding by luck; check that each + // non-ASCII character had a keycode of its own instead. + const keymap = execFileSync("xmodmap", ["-pk"], { env, encoding: "utf8" }); + for (const c of new Set(expected.match(/[^\x20-\x7e\n]/gu))) { + const cp = c.codePointAt(0); + const sym = `0x${(cp >= 0x100 ? 0x1000000 + cp : cp).toString(16).padStart(4, "0")}`; + assert(keymap.includes(sym), `${c} (${sym}) was not bound to a keycode`); + } console.log( - `PASS: native click, multilingual typing, Control+s, exact saved bytes and PNG. Evidence: ${scratch}`, + `PASS: native click, multilingual typing on bound keycodes, Control+s, exact saved bytes and PNG. Evidence: ${scratch}`, ); } finally { await call("control", { action: "release", lease }); diff --git a/packages/desktop-control/test/x11.test.ts b/packages/desktop-control/test/x11.test.ts new file mode 100644 index 00000000..00c5ef2b --- /dev/null +++ b/packages/desktop-control/test/x11.test.ts @@ -0,0 +1,81 @@ +import { expect, test } from "bun:test"; +import { parseKeymap, planKeys } from "../src/x11"; + +// Shape of `xmodmap -pk` on Xvfb: empty keycodes print no keysyms. +const sample = [ + "There are 7 KeySyms per KeyCode; KeyCodes range from 8 to 255.", + "", + " KeyCode\tKeysym (Keysym)\t...", + " Value \tValue (Name) \t...", + "", + " 8 \t", + " 38 \t0x0061 (a)\t0x0041 (A)\t0x0061 (a)\t0x0041 (A)\t", + " 65 \t0x0020 (space)\t0x0000 (NoSymbol)\t0x0020 (space)\t", + " 93 \t", + " 97 \t", + " 103 \t", + " 120 \t", + " 150 \t0x00e9 (eacute)\t0x00c9 (Eacute)\t0x00e9 (eacute)\t0x00c9 (Eacute)\t", + " 248 \t", +].join("\n"); +const U = (c: string) => 0x1000000 + (c.codePointAt(0) ?? 0); + +test("parseKeymap reads keysyms per keycode and drops NoSymbol", () => { + const keymap = parseKeymap(sample); + expect(keymap.get(8)).toEqual([]); + expect(keymap.get(38)).toEqual([0x61, 0x41, 0x61, 0x41]); + expect(keymap.get(65)).toEqual([0x20, 0x20]); + expect(keymap.size).toBe(9); +}); + +test("every missing character gets its own keycode, the lowest and highest empty stay free", () => { + const runs = planKeys("ñ 日本 ñ 日", parseKeymap(sample), new Map()); + expect(runs).toEqual([ + { + text: "ñ 日本 ñ 日", + map: [ + [93, 0xf1], + [97, U("日")], + [103, U("本")], + ], + overwrite: false, + }, + ]); +}); + +test("characters already in the keymap and plain ASCII need no keycode", () => { + expect(planKeys("café, Été", parseKeymap(sample), new Map())).toEqual([ + { text: "café, Été", map: [], overwrite: false }, + ]); +}); + +test("a later call reuses its keycodes and only binds what is new", () => { + const keymap = parseKeymap(sample); + keymap.set(93, [U("日"), U("日")]); + const owned = new Map([[93, U("日")]]); + expect(planKeys("日本", keymap, owned)).toEqual([ + { text: "日本", map: [[97, U("本")]], overwrite: false }, + ]); +}); + +test("text needing more keycodes than are free is typed in runs", () => { + // Free for typing: 93, 97, 103, 120. + const runs = planKeys("αβγδ αε ζ", parseKeymap(sample), new Map()); + expect(runs.map((r) => r.text)).toEqual(["αβγδ α", "ε ζ"]); + expect(runs[0]?.overwrite).toBe(false); + // ε and ζ take the keycodes pressed longest ago (β, γ), not α's. + expect(runs[1]).toEqual({ + text: "ε ζ", + map: [ + [97, U("ε")], + [103, U("ζ")], + ], + overwrite: true, + }); +}); + +test("no free keycode leaves typing to xdotool", () => { + const keymap = parseKeymap(sample); + for (const code of [93, 97, 103, 120]) keymap.set(code, [0x61]); + expect(planKeys("日", keymap, new Map())).toEqual([{ text: "日", map: [], overwrite: false }]); +}); diff --git a/packages/server/scripts/install-browser.sh b/packages/server/scripts/install-browser.sh index 6104cf98..635d7789 100755 --- a/packages/server/scripts/install-browser.sh +++ b/packages/server/scripts/install-browser.sh @@ -109,7 +109,7 @@ case "$(dpkg --print-architecture)" in amd64) CHROME_DEB=google-chrome-stable_current_amd64.deb ;; *) die "Google Chrome ships for amd64 only, and this box is $(dpkg --print-architecture)" ;; esac -PKGS="xvfb openbox x11vnc novnc xdotool scrot feh fonts-noto-color-emoji" +PKGS="xvfb openbox x11vnc novnc xdotool x11-xserver-utils scrot feh fonts-noto-color-emoji" if ! command -v google-chrome >/dev/null; then TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT