Stop long multilingual typing from dropping characters on the remote desktop

packages/desktop-control/README.md

@@ -7,8 +7,9 @@ configuration, and a panel that opens its viewer.
 ## Run
 
 Requires Node 22+, Xvfb (or another X11 display), a window manager, `xdotool`,
-`scrot`, `x11vnc`, and noVNC's browser modules at `/usr/share/novnc`. The VNC
-server must use RFB 3.8 with VNC password authentication on loopback port 5900.
+`xmodmap`, `scrot`, `x11vnc`, and noVNC's browser modules at
+`/usr/share/novnc`. The VNC server must use RFB 3.8 with VNC password
+authentication on loopback port 5900.
 Wayland and control of a user's macOS/Windows desktop are outside this version.
 
 ```sh
@@ -51,9 +52,12 @@ A second agent must wait; there is no agent force-takeover operation.
 PNG pixels, regardless of how the viewer scales the display. `key` accepts
 combinations such as `Control+s` and `Shift+Tab`. `type` sends real key events,
 including explicit Return/Tab events, with a 4000-character limit. LF/CRLF are
-normalized to Return. It is typing, so the focused application's shortcuts,
-autoindent, and text transformations still apply. Scroll amounts are wheel
-steps (1–20) at the current pointer position.
+normalized to Return. Characters missing from the keyboard map (accents, Greek,
+CJK) are bound to free keycodes with `xmodmap` before typing and stay bound
+until a later character needs the keycode; the service unbinds them when it
+stops. It is typing, so the focused application's shortcuts, autoindent, and
+text transformations still apply. Scroll amounts are wheel steps (1–20) at the
+current pointer position.
 
 The viewer fills its frame and accepts mouse and keyboard input directly, without
 a takeover panel. A keyboard button on touch devices opens a text field for the

packages/desktop-control/src/server.ts

@@ -211,6 +211,7 @@ export function createDesktopServer(opts: {
 		server,
 		close() {
 			control.close();
+			desktop.close();
 			for (const v of viewers.values()) v.socket.terminate();
 			wss.close();
 			for (const s of servers) s.close();

packages/desktop-control/src/x11.ts

@@ -2,6 +2,7 @@ import { spawn } from "node:child_process";
 import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
 
 export function run(
 	file: string,
@@ -72,7 +73,99 @@ export function keySequence(value: unknown): string {
 		.join("+");
 }
 
+/** `xmodmap -pk` → keycode → its non-empty keysyms. */
+export function parseKeymap(out: string): Map<number, number[]> {
+	const keymap = new Map<number, number[]>();
+	for (const line of out.split("\n")) {
+		const code = /^\s*(\d+)(?:\s|$)/.exec(line)?.[1];
+		if (code)
+			keymap.set(Number(code), (line.match(/0x[0-9a-f]+/gi) ?? []).map(Number).filter(Boolean));
+	}
+	return keymap;
+}
+
+export type KeyRun = { text: string; map: [keycode: number, keysym: number][]; overwrite: boolean };
+
+/**
+ * xdotool types a character missing from the keymap by rebinding ONE scratch
+ * keycode for every key press and clearing it 3 ms later. An application that
+ * reads the keymap after that has moved on types nothing or a neighbour; a
+ * 4000-character multilingual text lost up to 137 of its 1237 non-ASCII characters.
+ * Instead, each such character gets an empty keycode of its own before xdotool
+ * runs, so xdotool finds it in the keymap and never rebinds. Text needing more
+ * keycodes than are free is split into runs; a run that rebinds a keycode
+ * already in use is marked `overwrite` so the caller can let the app catch up.
+ */
+export function planKeys(
+	text: string,
+	keymap: Map<number, number[]>,
+	owned: Map<number, number>,
+): KeyRun[] {
+	const present = new Set<number>();
+	const empty: number[] = [];
+	for (const [code, syms] of keymap) {
+		if (owned.has(code)) continue;
+		if (!syms.length) empty.push(code);
+		for (const sym of syms) present.add(sym);
+	}
+	empty.sort((a, b) => a - b);
+	// Leave the lowest empty keycode to xdotool's scratch binding and the
+	// highest to x11vnc's keysym additions for a viewer.
+	const slots = [...empty.slice(1, -1), ...owned.keys()];
+	const chars = Array.from(text);
+	// The keysym xdotool itself would bind (xdo.c, _xdo_keysym_from_char).
+	const syms = chars.map((c) => {
+		const cp = c.codePointAt(0) ?? 0;
+		const sym = cp >= 0x100 ? 0x1000000 + cp : cp;
+		return cp >= 0xa0 && !present.has(sym) ? sym : 0;
+	});
+	if (!slots.length || !syms.some(Boolean)) return [{ text, map: [], overwrite: false }];
+	const held = new Map(owned);
+	// Rebind empty keycodes first, then the ones whose last key press is oldest.
+	const lastUse = new Map<number, number>();
+	const age = (code: number) => {
+		const sym = held.get(code);
+		return sym === undefined ? -2 : (lastUse.get(sym) ?? -1);
+	};
+	const runs: KeyRun[] = [];
+	for (let start = 0; start < chars.length; ) {
+		const need = new Set<number>();
+		let end = start;
+		for (; end < chars.length; end++) {
+			const sym = syms[end];
+			if (!sym || need.has(sym)) continue;
+			if (need.size === slots.length) break;
+			need.add(sym);
+		}
+		const holding = new Set(held.values());
+		const spare = slots
+			.filter((code) => !need.has(held.get(code) ?? 0))
+			.sort((a, b) => age(a) - age(b));
+		const map: KeyRun["map"] = [];
+		let overwrite = false;
+		for (const sym of need) {
+			if (holding.has(sym)) continue;
+			const code = spare.shift() as number;
+			overwrite ||= held.has(code);
+			held.set(code, sym);
+			map.push([code, sym]);
+		}
+		runs.push({ text: chars.slice(start, end).join(""), map, overwrite });
+		for (; start < end; start++) if (syms[start]) lastUse.set(syms[start] as number, start);
+	}
+	return runs;
+}
+
+// shortcut: a fixed pause before reusing a keycode. Measured on a 1-vCPU box,
+// no pause at all lost nothing; 100 ms is margin for a busier app. An app that
+// lags longer can still misread the end of the previous run. Binding a second
+// keysym per keycode (Shift level) would halve the reuses if this gets slow.
+const REBIND_SETTLE_MS = 100;
+let warnedNoXmodmap = false;
+
 export class X11Desktop {
+	/** Keycodes this process bound for typing → keysym. Kept between calls. */
+	private readonly owned = new Map<number, number>();
 	constructor(private readonly display = ":99") {}
 	async screenshot(): Promise<Buffer> {
 		// Ubuntu's scrot/imlib cannot save to Node's socket-backed stdout pipe.
@@ -105,25 +198,41 @@ export class X11Desktop {
 						"text must be at most 4000 characters, with only tab/newline control characters.",
 					);
 				try {
-					// LF is not Return in libxdo. Send newlines explicitly. Its dynamic
-					// Unicode keymaps also need the normal 12ms delay, or GTK loses keys.
-					for (const part of args.text.replace(/\r\n?/g, "\n").split(/([\n\t])/)) {
-						if (!part) continue;
-						if (part === "\n" || part === "\t")
-							await run(
-								"xdotool",
-								["key", "--clearmodifiers", part === "\n" ? "Return" : "Tab"],
-								this.display,
-								signal,
-							);
-						else
+					for (const chunk of await this.keyRuns(args.text.replace(/\r\n?/g, "\n"), signal)) {
+						if (chunk.map.length) {
+							if (chunk.overwrite) await sleep(REBIND_SETTLE_MS, undefined, { signal });
+							// Claimed first: the next call drops any an interrupted xmodmap missed.
+							for (const [code, sym] of chunk.map) this.owned.set(code, sym);
 							await run(
-								"xdotool",
-								["type", "--clearmodifiers", "--delay", "12", "--file", "-"],
+								"xmodmap",
+								["-"],
 								this.display,
 								signal,
-								part,
+								chunk.map
+									.map(([code, sym]) => `keycode ${code} = 0x${sym.toString(16)}`)
+									.join("\n"),
 							);
+						}
+						// LF is not Return in libxdo. Send newlines explicitly. Keep the
+						// normal 12ms delay: faster typing loses keys in GTK.
+						for (const part of chunk.text.split(/([\n\t])/)) {
+							if (!part) continue;
+							if (part === "\n" || part === "\t")
+								await run(
+									"xdotool",
+									["key", "--clearmodifiers", part === "\n" ? "Return" : "Tab"],
+									this.display,
+									signal,
+								);
+							else
+								await run(
+									"xdotool",
+									["type", "--clearmodifiers", "--delay", "12", "--file", "-"],
+									this.display,
+									signal,
+									part,
+								);
+						}
 					}
 				} catch (error) {
 					await this.resetInput();
@@ -195,6 +304,31 @@ export class X11Desktop {
 			throw error;
 		}
 	}
+	private async keyRuns(text: string, signal: AbortSignal): Promise<KeyRun[]> {
+		if (!/[\u00a0-\u{10ffff}]/u.test(text)) return [{ text, map: [], overwrite: false }];
+		let keymap: Map<number, number[]>;
+		try {
+			keymap = parseKeymap((await run("xmodmap", ["-pk"], this.display, signal)).toString());
+		} catch (error) {
+			if (signal.aborted) throw error;
+			// xdotool still types, with its per-character rebinding race.
+			if (!warnedNoXmodmap)
+				console.error(`Typing without reserved keycodes: ${(error as Error).message}`);
+			warnedNoXmodmap = true;
+			return [{ text, map: [], overwrite: false }];
+		}
+		// A restarted display starts from its default keymap.
+		for (const [code, sym] of this.owned)
+			if (!keymap.get(code)?.includes(sym)) this.owned.delete(code);
+		return planKeys(text, keymap, this.owned);
+	}
+	/** Hand the keycodes bound for typing back to the display. */
+	close(): void {
+		if (!this.owned.size) return;
+		const clear = [...this.owned.keys()].map((code) => `keycode ${code} =`).join("\n");
+		this.owned.clear();
+		run("xmodmap", ["-"], this.display, undefined, clear).catch(() => {});
+	}
 	async resetInput(): Promise<void> {
 		await run(
 			"xdotool",

packages/desktop-control/test/linux-smoke.mjs

@@ -106,8 +106,16 @@ try {
 	assert(png);
 	await writeFile(join(scratch, "screen.png"), Buffer.from(png.data, "base64"));
 	assert.equal(actual, expected);
+	// Short text survives xdotool's own rebinding by luck; check that each
+	// non-ASCII character had a keycode of its own instead.
+	const keymap = execFileSync("xmodmap", ["-pk"], { env, encoding: "utf8" });
+	for (const c of new Set(expected.match(/[^\x20-\x7e\n]/gu))) {
+		const cp = c.codePointAt(0);
+		const sym = `0x${(cp >= 0x100 ? 0x1000000 + cp : cp).toString(16).padStart(4, "0")}`;
+		assert(keymap.includes(sym), `${c} (${sym}) was not bound to a keycode`);
+	}
 	console.log(
-		`PASS: native click, multilingual typing, Control+s, exact saved bytes and PNG. Evidence: ${scratch}`,
+		`PASS: native click, multilingual typing on bound keycodes, Control+s, exact saved bytes and PNG. Evidence: ${scratch}`,
 	);
 } finally {
 	await call("control", { action: "release", lease });

packages/desktop-control/test/x11.test.ts

@@ -0,0 +1,81 @@
+import { expect, test } from "bun:test";
+import { parseKeymap, planKeys } from "../src/x11";
+
+// Shape of `xmodmap -pk` on Xvfb: empty keycodes print no keysyms.
+const sample = [
+	"There are 7 KeySyms per KeyCode; KeyCodes range from 8 to 255.",
+	"",
+	"    KeyCode\tKeysym (Keysym)\t...",
+	"    Value  \tValue   (Name) \t...",
+	"",
+	"      8    \t",
+	"     38    \t0x0061 (a)\t0x0041 (A)\t0x0061 (a)\t0x0041 (A)\t",
+	"     65    \t0x0020 (space)\t0x0000 (NoSymbol)\t0x0020 (space)\t",
+	"     93    \t",
+	"     97    \t",
+	"    103    \t",
+	"    120    \t",
+	"    150    \t0x00e9 (eacute)\t0x00c9 (Eacute)\t0x00e9 (eacute)\t0x00c9 (Eacute)\t",
+	"    248    \t",
+].join("\n");
+const U = (c: string) => 0x1000000 + (c.codePointAt(0) ?? 0);
+
+test("parseKeymap reads keysyms per keycode and drops NoSymbol", () => {
+	const keymap = parseKeymap(sample);
+	expect(keymap.get(8)).toEqual([]);
+	expect(keymap.get(38)).toEqual([0x61, 0x41, 0x61, 0x41]);
+	expect(keymap.get(65)).toEqual([0x20, 0x20]);
+	expect(keymap.size).toBe(9);
+});
+
+test("every missing character gets its own keycode, the lowest and highest empty stay free", () => {
+	const runs = planKeys("ñ 日本 ñ 日", parseKeymap(sample), new Map());
+	expect(runs).toEqual([
+		{
+			text: "ñ 日本 ñ 日",
+			map: [
+				[93, 0xf1],
+				[97, U("日")],
+				[103, U("本")],
+			],
+			overwrite: false,
+		},
+	]);
+});
+
+test("characters already in the keymap and plain ASCII need no keycode", () => {
+	expect(planKeys("café, Été", parseKeymap(sample), new Map())).toEqual([
+		{ text: "café, Été", map: [], overwrite: false },
+	]);
+});
+
+test("a later call reuses its keycodes and only binds what is new", () => {
+	const keymap = parseKeymap(sample);
+	keymap.set(93, [U("日"), U("日")]);
+	const owned = new Map([[93, U("日")]]);
+	expect(planKeys("日本", keymap, owned)).toEqual([
+		{ text: "日本", map: [[97, U("本")]], overwrite: false },
+	]);
+});
+
+test("text needing more keycodes than are free is typed in runs", () => {
+	// Free for typing: 93, 97, 103, 120.
+	const runs = planKeys("αβγδ αε ζ", parseKeymap(sample), new Map());
+	expect(runs.map((r) => r.text)).toEqual(["αβγδ α", "ε ζ"]);
+	expect(runs[0]?.overwrite).toBe(false);
+	// ε and ζ take the keycodes pressed longest ago (β, γ), not α's.
+	expect(runs[1]).toEqual({
+		text: "ε ζ",
+		map: [
+			[97, U("ε")],
+			[103, U("ζ")],
+		],
+		overwrite: true,
+	});
+});
+
+test("no free keycode leaves typing to xdotool", () => {
+	const keymap = parseKeymap(sample);
+	for (const code of [93, 97, 103, 120]) keymap.set(code, [0x61]);
+	expect(planKeys("日", keymap, new Map())).toEqual([{ text: "日", map: [], overwrite: false }]);
+});

packages/server/scripts/install-browser.sh

@@ -109,7 +109,7 @@ case "$(dpkg --print-architecture)" in
 amd64) CHROME_DEB=google-chrome-stable_current_amd64.deb ;;
 *) die "Google Chrome ships for amd64 only, and this box is $(dpkg --print-architecture)" ;;
 esac
-PKGS="xvfb openbox x11vnc novnc xdotool scrot feh fonts-noto-color-emoji"
+PKGS="xvfb openbox x11vnc novnc xdotool x11-xserver-utils scrot feh fonts-noto-color-emoji"
 if ! command -v google-chrome >/dev/null; then
 	TMP="$(mktemp -d)"
 	trap 'rm -rf "$TMP"' EXIT