diff --git a/packages/desktop-control/README.md b/packages/desktop-control/README.md index 7e2d5917..45fd73ba 100644 --- a/packages/desktop-control/README.md +++ b/packages/desktop-control/README.md @@ -7,8 +7,9 @@ configuration, and a panel that opens its viewer. ## Run Requires Node 22+, Xvfb (or another X11 display), a window manager, `xdotool`, -`scrot`, `x11vnc`, and noVNC's browser modules at `/usr/share/novnc`. The VNC -server must use RFB 3.8 with VNC password authentication on loopback port 5900. +`xmodmap`, `scrot`, `x11vnc`, and noVNC's browser modules at +`/usr/share/novnc`. The VNC server must use RFB 3.8 with VNC password +authentication on loopback port 5900. Wayland and control of a user's macOS/Windows desktop are outside this version. ```sh @@ -51,9 +52,12 @@ A second agent must wait; there is no agent force-takeover operation. PNG pixels, regardless of how the viewer scales the display. `key` accepts combinations such as `Control+s` and `Shift+Tab`. `type` sends real key events, including explicit Return/Tab events, with a 4000-character limit. LF/CRLF are -normalized to Return. It is typing, so the focused application's shortcuts, -autoindent, and text transformations still apply. Scroll amounts are wheel -steps (1–20) at the current pointer position. +normalized to Return. Characters missing from the keyboard map (accents, Greek, +CJK) are bound to free keycodes with `xmodmap` before typing and stay bound +until a later character needs the keycode; the service unbinds them when it +stops. It is typing, so the focused application's shortcuts, autoindent, and +text transformations still apply. Scroll amounts are wheel steps (1–20) at the +current pointer position. The viewer fills its frame and accepts mouse and keyboard input directly, without a takeover panel. A keyboard button on touch devices opens a text field for the diff --git a/packages/desktop-control/src/server.ts b/packages/desktop-control/src/server.ts index 363ef075..f698d783 100644 --- a/packages/desktop-control/src/server.ts +++ b/packages/desktop-control/src/server.ts @@ -211,6 +211,7 @@ export function createDesktopServer(opts: { server, close() { control.close(); + desktop.close(); for (const v of viewers.values()) v.socket.terminate(); wss.close(); for (const s of servers) s.close(); diff --git a/packages/desktop-control/src/x11.ts b/packages/desktop-control/src/x11.ts index 8644c770..54b783ff 100644 --- a/packages/desktop-control/src/x11.ts +++ b/packages/desktop-control/src/x11.ts @@ -2,6 +2,7 @@ import { spawn } from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; export function run( file: string, @@ -72,7 +73,99 @@ export function keySequence(value: unknown): string { .join("+"); } +/** `xmodmap -pk` → keycode → its non-empty keysyms. */ +export function parseKeymap(out: string): Map { + const keymap = new Map(); + for (const line of out.split("\n")) { + const code = /^\s*(\d+)(?:\s|$)/.exec(line)?.[1]; + if (code) + keymap.set(Number(code), (line.match(/0x[0-9a-f]+/gi) ?? []).map(Number).filter(Boolean)); + } + return keymap; +} + +export type KeyRun = { text: string; map: [keycode: number, keysym: number][]; overwrite: boolean }; + +/** + * xdotool types a character missing from the keymap by rebinding ONE scratch + * keycode for every key press and clearing it 3 ms later. An application that + * reads the keymap after that has moved on types nothing or a neighbour; a + * 4000-character multilingual text lost up to 137 of its 1237 non-ASCII characters. + * Instead, each such character gets an empty keycode of its own before xdotool + * runs, so xdotool finds it in the keymap and never rebinds. Text needing more + * keycodes than are free is split into runs; a run that rebinds a keycode + * already in use is marked `overwrite` so the caller can let the app catch up. + */ +export function planKeys( + text: string, + keymap: Map, + owned: Map, +): KeyRun[] { + const present = new Set(); + const empty: number[] = []; + for (const [code, syms] of keymap) { + if (owned.has(code)) continue; + if (!syms.length) empty.push(code); + for (const sym of syms) present.add(sym); + } + empty.sort((a, b) => a - b); + // Leave the lowest empty keycode to xdotool's scratch binding and the + // highest to x11vnc's keysym additions for a viewer. + const slots = [...empty.slice(1, -1), ...owned.keys()]; + const chars = Array.from(text); + // The keysym xdotool itself would bind (xdo.c, _xdo_keysym_from_char). + const syms = chars.map((c) => { + const cp = c.codePointAt(0) ?? 0; + const sym = cp >= 0x100 ? 0x1000000 + cp : cp; + return cp >= 0xa0 && !present.has(sym) ? sym : 0; + }); + if (!slots.length || !syms.some(Boolean)) return [{ text, map: [], overwrite: false }]; + const held = new Map(owned); + // Rebind empty keycodes first, then the ones whose last key press is oldest. + const lastUse = new Map(); + const age = (code: number) => { + const sym = held.get(code); + return sym === undefined ? -2 : (lastUse.get(sym) ?? -1); + }; + const runs: KeyRun[] = []; + for (let start = 0; start < chars.length; ) { + const need = new Set(); + let end = start; + for (; end < chars.length; end++) { + const sym = syms[end]; + if (!sym || need.has(sym)) continue; + if (need.size === slots.length) break; + need.add(sym); + } + const holding = new Set(held.values()); + const spare = slots + .filter((code) => !need.has(held.get(code) ?? 0)) + .sort((a, b) => age(a) - age(b)); + const map: KeyRun["map"] = []; + let overwrite = false; + for (const sym of need) { + if (holding.has(sym)) continue; + const code = spare.shift() as number; + overwrite ||= held.has(code); + held.set(code, sym); + map.push([code, sym]); + } + runs.push({ text: chars.slice(start, end).join(""), map, overwrite }); + for (; start < end; start++) if (syms[start]) lastUse.set(syms[start] as number, start); + } + return runs; +} + +// shortcut: a fixed pause before reusing a keycode. Measured on a 1-vCPU box, +// no pause at all lost nothing; 100 ms is margin for a busier app. An app that +// lags longer can still misread the end of the previous run. Binding a second +// keysym per keycode (Shift level) would halve the reuses if this gets slow. +const REBIND_SETTLE_MS = 100; +let warnedNoXmodmap = false; + export class X11Desktop { + /** Keycodes this process bound for typing → keysym. Kept between calls. */ + private readonly owned = new Map(); constructor(private readonly display = ":99") {} async screenshot(): Promise { // Ubuntu's scrot/imlib cannot save to Node's socket-backed stdout pipe. @@ -105,25 +198,41 @@ export class X11Desktop { "text must be at most 4000 characters, with only tab/newline control characters.", ); try { - // LF is not Return in libxdo. Send newlines explicitly. Its dynamic - // Unicode keymaps also need the normal 12ms delay, or GTK loses keys. - for (const part of args.text.replace(/\r\n?/g, "\n").split(/([\n\t])/)) { - if (!part) continue; - if (part === "\n" || part === "\t") - await run( - "xdotool", - ["key", "--clearmodifiers", part === "\n" ? "Return" : "Tab"], - this.display, - signal, - ); - else + for (const chunk of await this.keyRuns(args.text.replace(/\r\n?/g, "\n"), signal)) { + if (chunk.map.length) { + if (chunk.overwrite) await sleep(REBIND_SETTLE_MS, undefined, { signal }); + // Claimed first: the next call drops any an interrupted xmodmap missed. + for (const [code, sym] of chunk.map) this.owned.set(code, sym); await run( - "xdotool", - ["type", "--clearmodifiers", "--delay", "12", "--file", "-"], + "xmodmap", + ["-"], this.display, signal, - part, + chunk.map + .map(([code, sym]) => `keycode ${code} = 0x${sym.toString(16)}`) + .join("\n"), ); + } + // LF is not Return in libxdo. Send newlines explicitly. Keep the + // normal 12ms delay: faster typing loses keys in GTK. + for (const part of chunk.text.split(/([\n\t])/)) { + if (!part) continue; + if (part === "\n" || part === "\t") + await run( + "xdotool", + ["key", "--clearmodifiers", part === "\n" ? "Return" : "Tab"], + this.display, + signal, + ); + else + await run( + "xdotool", + ["type", "--clearmodifiers", "--delay", "12", "--file", "-"], + this.display, + signal, + part, + ); + } } } catch (error) { await this.resetInput(); @@ -195,6 +304,31 @@ export class X11Desktop { throw error; } } + private async keyRuns(text: string, signal: AbortSignal): Promise { + if (!/[\u00a0-\u{10ffff}]/u.test(text)) return [{ text, map: [], overwrite: false }]; + let keymap: Map; + try { + keymap = parseKeymap((await run("xmodmap", ["-pk"], this.display, signal)).toString()); + } catch (error) { + if (signal.aborted) throw error; + // xdotool still types, with its per-character rebinding race. + if (!warnedNoXmodmap) + console.error(`Typing without reserved keycodes: ${(error as Error).message}`); + warnedNoXmodmap = true; + return [{ text, map: [], overwrite: false }]; + } + // A restarted display starts from its default keymap. + for (const [code, sym] of this.owned) + if (!keymap.get(code)?.includes(sym)) this.owned.delete(code); + return planKeys(text, keymap, this.owned); + } + /** Hand the keycodes bound for typing back to the display. */ + close(): void { + if (!this.owned.size) return; + const clear = [...this.owned.keys()].map((code) => `keycode ${code} =`).join("\n"); + this.owned.clear(); + run("xmodmap", ["-"], this.display, undefined, clear).catch(() => {}); + } async resetInput(): Promise { await run( "xdotool", diff --git a/packages/desktop-control/test/linux-smoke.mjs b/packages/desktop-control/test/linux-smoke.mjs index d659e289..f47b0217 100644 --- a/packages/desktop-control/test/linux-smoke.mjs +++ b/packages/desktop-control/test/linux-smoke.mjs @@ -106,8 +106,16 @@ try { assert(png); await writeFile(join(scratch, "screen.png"), Buffer.from(png.data, "base64")); assert.equal(actual, expected); + // Short text survives xdotool's own rebinding by luck; check that each + // non-ASCII character had a keycode of its own instead. + const keymap = execFileSync("xmodmap", ["-pk"], { env, encoding: "utf8" }); + for (const c of new Set(expected.match(/[^\x20-\x7e\n]/gu))) { + const cp = c.codePointAt(0); + const sym = `0x${(cp >= 0x100 ? 0x1000000 + cp : cp).toString(16).padStart(4, "0")}`; + assert(keymap.includes(sym), `${c} (${sym}) was not bound to a keycode`); + } console.log( - `PASS: native click, multilingual typing, Control+s, exact saved bytes and PNG. Evidence: ${scratch}`, + `PASS: native click, multilingual typing on bound keycodes, Control+s, exact saved bytes and PNG. Evidence: ${scratch}`, ); } finally { await call("control", { action: "release", lease }); diff --git a/packages/desktop-control/test/x11.test.ts b/packages/desktop-control/test/x11.test.ts new file mode 100644 index 00000000..00c5ef2b --- /dev/null +++ b/packages/desktop-control/test/x11.test.ts @@ -0,0 +1,81 @@ +import { expect, test } from "bun:test"; +import { parseKeymap, planKeys } from "../src/x11"; + +// Shape of `xmodmap -pk` on Xvfb: empty keycodes print no keysyms. +const sample = [ + "There are 7 KeySyms per KeyCode; KeyCodes range from 8 to 255.", + "", + " KeyCode\tKeysym (Keysym)\t...", + " Value \tValue (Name) \t...", + "", + " 8 \t", + " 38 \t0x0061 (a)\t0x0041 (A)\t0x0061 (a)\t0x0041 (A)\t", + " 65 \t0x0020 (space)\t0x0000 (NoSymbol)\t0x0020 (space)\t", + " 93 \t", + " 97 \t", + " 103 \t", + " 120 \t", + " 150 \t0x00e9 (eacute)\t0x00c9 (Eacute)\t0x00e9 (eacute)\t0x00c9 (Eacute)\t", + " 248 \t", +].join("\n"); +const U = (c: string) => 0x1000000 + (c.codePointAt(0) ?? 0); + +test("parseKeymap reads keysyms per keycode and drops NoSymbol", () => { + const keymap = parseKeymap(sample); + expect(keymap.get(8)).toEqual([]); + expect(keymap.get(38)).toEqual([0x61, 0x41, 0x61, 0x41]); + expect(keymap.get(65)).toEqual([0x20, 0x20]); + expect(keymap.size).toBe(9); +}); + +test("every missing character gets its own keycode, the lowest and highest empty stay free", () => { + const runs = planKeys("ñ 日本 ñ 日", parseKeymap(sample), new Map()); + expect(runs).toEqual([ + { + text: "ñ 日本 ñ 日", + map: [ + [93, 0xf1], + [97, U("日")], + [103, U("本")], + ], + overwrite: false, + }, + ]); +}); + +test("characters already in the keymap and plain ASCII need no keycode", () => { + expect(planKeys("café, Été", parseKeymap(sample), new Map())).toEqual([ + { text: "café, Été", map: [], overwrite: false }, + ]); +}); + +test("a later call reuses its keycodes and only binds what is new", () => { + const keymap = parseKeymap(sample); + keymap.set(93, [U("日"), U("日")]); + const owned = new Map([[93, U("日")]]); + expect(planKeys("日本", keymap, owned)).toEqual([ + { text: "日本", map: [[97, U("本")]], overwrite: false }, + ]); +}); + +test("text needing more keycodes than are free is typed in runs", () => { + // Free for typing: 93, 97, 103, 120. + const runs = planKeys("αβγδ αε ζ", parseKeymap(sample), new Map()); + expect(runs.map((r) => r.text)).toEqual(["αβγδ α", "ε ζ"]); + expect(runs[0]?.overwrite).toBe(false); + // ε and ζ take the keycodes pressed longest ago (β, γ), not α's. + expect(runs[1]).toEqual({ + text: "ε ζ", + map: [ + [97, U("ε")], + [103, U("ζ")], + ], + overwrite: true, + }); +}); + +test("no free keycode leaves typing to xdotool", () => { + const keymap = parseKeymap(sample); + for (const code of [93, 97, 103, 120]) keymap.set(code, [0x61]); + expect(planKeys("日", keymap, new Map())).toEqual([{ text: "日", map: [], overwrite: false }]); +}); diff --git a/packages/server/scripts/install-browser.sh b/packages/server/scripts/install-browser.sh index 6104cf98..635d7789 100755 --- a/packages/server/scripts/install-browser.sh +++ b/packages/server/scripts/install-browser.sh @@ -109,7 +109,7 @@ case "$(dpkg --print-architecture)" in amd64) CHROME_DEB=google-chrome-stable_current_amd64.deb ;; *) die "Google Chrome ships for amd64 only, and this box is $(dpkg --print-architecture)" ;; esac -PKGS="xvfb openbox x11vnc novnc xdotool scrot feh fonts-noto-color-emoji" +PKGS="xvfb openbox x11vnc novnc xdotool x11-xserver-utils scrot feh fonts-noto-color-emoji" if ! command -v google-chrome >/dev/null; then TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT