diff --git a/packages/tool-server/src/tools/keyboard/index.ts b/packages/tool-server/src/tools/keyboard/index.ts index 0c47964e2..5799a0c6e 100644 --- a/packages/tool-server/src/tools/keyboard/index.ts +++ b/packages/tool-server/src/tools/keyboard/index.ts @@ -21,7 +21,7 @@ const zodSchema = z.object({ .string() .optional() .describe( - "Named key to press: enter, escape, backspace, tab, space, arrow-up, arrow-down, arrow-left, arrow-right, f1–f12. Not supported on TV targets — move focus with `tv-remote` (up/down/left/right) instead." + "Named key to press: enter, escape, backspace, tab, space, arrow-up, arrow-down, arrow-left, arrow-right, f1–f12. When combined with `text`, the key is pressed AFTER the text is typed (so text + enter types and submits). Not supported on TV targets — move focus with `tv-remote` (up/down/left/right) instead." ), delayMs: z .number() @@ -58,7 +58,7 @@ Returns { typed: string, keys: number }. Fails if an unsupported key name is pro - text: types a string character by character (supports uppercase, digits, common punctuation) - key: presses a single named key (enter, escape, backspace, tab, arrow-up/down/left/right, f1–f12) — NOT supported on TV targets; move focus with \`tv-remote\` instead. On a TV target (runtimeKind 'tv') only \`text\` applies — focus a text field first (with \`tv-remote\`), then type into it (injected HID keyboard on Apple TV, \`adb input text\` on Android TV). -Provide text, key, or both.`, +Provide text, key, or both — when both are given, the text is typed first and the key is pressed after it (text + key:"enter" types and submits).`, zodSchema, capability, searchHint: diff --git a/packages/tool-server/src/tools/keyboard/platforms/chromium.ts b/packages/tool-server/src/tools/keyboard/platforms/chromium.ts index 90cc4103a..7de953127 100644 --- a/packages/tool-server/src/tools/keyboard/platforms/chromium.ts +++ b/packages/tool-server/src/tools/keyboard/platforms/chromium.ts @@ -10,8 +10,11 @@ async function runChromium(api: ChromiumCdpApi, params: KeyboardParams): Promise const delay = params.delayMs ?? 50; let keysPressed = 0; + // Resolve the named key before typing anything so an unknown name fails + // fast instead of after the text has already been typed. + let named: (typeof CHROMIUM_NAMED_KEYS)[string] | undefined; if (params.key) { - const named = CHROMIUM_NAMED_KEYS[params.key.toLowerCase()]; + named = CHROMIUM_NAMED_KEYS[params.key.toLowerCase()]; if (!named) { throw new FailureError( `Unknown key "${params.key}". Supported: ${Object.keys(CHROMIUM_NAMED_KEYS).join(", ")}`, @@ -23,20 +26,6 @@ async function runChromium(api: ChromiumCdpApi, params: KeyboardParams): Promise } ); } - await api.dispatchKeyEvent({ - type: "keyDown", - key: named.key, - code: named.code, - windowsVirtualKeyCode: named.windowsVirtualKeyCode, - }); - await sleep(delay); - await api.dispatchKeyEvent({ - type: "keyUp", - key: named.key, - code: named.code, - windowsVirtualKeyCode: named.windowsVirtualKeyCode, - }); - keysPressed++; } if (params.text) { @@ -70,6 +59,25 @@ async function runChromium(api: ChromiumCdpApi, params: KeyboardParams): Promise } } + // Key after text: a combined call means "type, then submit" (text + + // key:"enter"). Pressing the key first submits the still-empty field. + if (named) { + await api.dispatchKeyEvent({ + type: "keyDown", + key: named.key, + code: named.code, + windowsVirtualKeyCode: named.windowsVirtualKeyCode, + }); + await sleep(delay); + await api.dispatchKeyEvent({ + type: "keyUp", + key: named.key, + code: named.code, + windowsVirtualKeyCode: named.windowsVirtualKeyCode, + }); + keysPressed++; + } + return { typed: params.text ?? params.key ?? "", keys: keysPressed }; } diff --git a/packages/tool-server/src/tools/keyboard/platforms/vega.ts b/packages/tool-server/src/tools/keyboard/platforms/vega.ts index 30dcae46f..52b0a1f6f 100644 --- a/packages/tool-server/src/tools/keyboard/platforms/vega.ts +++ b/packages/tool-server/src/tools/keyboard/platforms/vega.ts @@ -1,5 +1,9 @@ import type { PlatformImpl } from "../../../utils/cross-platform-tool"; -import { injectVegaNamedKey, injectVegaText } from "../../../utils/vega-input"; +import { + injectVegaNamedKey, + injectVegaText, + resolveVegaNamedKeycode, +} from "../../../utils/vega-input"; import type { KeyboardParams, KeyboardResult } from "../types"; // Vega has no simulator-server: input is injected over `adb` (on-device @@ -8,14 +12,18 @@ import type { KeyboardParams, KeyboardResult } from "../types"; // adb fails with a clean 424 install hint rather than a spawn ENOENT. async function runVega(params: KeyboardParams): Promise { let keysPressed = 0; - if (params.key) { - await injectVegaNamedKey(params.key); - keysPressed++; - } + // Resolve the named key before injecting text so an unknown name fails fast. + if (params.key) resolveVegaNamedKeycode(params.key); if (params.text) { await injectVegaText(params.text); keysPressed += [...params.text].length; } + // Key after text: a combined call means "type, then submit" (text + + // key:"enter"). Pressing the key first submits the still-empty field. + if (params.key) { + await injectVegaNamedKey(params.key); + keysPressed++; + } return { typed: params.text ?? params.key ?? "", keys: keysPressed }; } diff --git a/packages/tool-server/src/tools/keyboard/simulator-server-keys.ts b/packages/tool-server/src/tools/keyboard/simulator-server-keys.ts index 09e42982d..d98a76387 100644 --- a/packages/tool-server/src/tools/keyboard/simulator-server-keys.ts +++ b/packages/tool-server/src/tools/keyboard/simulator-server-keys.ts @@ -35,9 +35,12 @@ export async function typeSimulatorServer( keysPressed++; }; + // Resolve the named key before typing anything so an unknown name fails + // fast instead of after the text has already been typed. + let namedKeyCode: number | undefined; if (params.key) { - const code = NAMED_KEYS[params.key.toLowerCase()]; - if (code == null) { + namedKeyCode = NAMED_KEYS[params.key.toLowerCase()]; + if (namedKeyCode == null) { throw new FailureError( `Unknown key "${params.key}". Supported: ${Object.keys(NAMED_KEYS).join(", ")}`, { @@ -48,7 +51,6 @@ export async function typeSimulatorServer( } ); } - await pressKeyCode(code); } if (params.text) { @@ -66,5 +68,13 @@ export async function typeSimulatorServer( } } + // Key after text: a combined call means "type, then submit" (text + + // key:"enter"). Pressing the key first fires enter into the still-empty + // field, which can blur it and leak the text to app-level key commands + // (e.g. "d" toggles the React Native dev menu when nothing is focused). + if (namedKeyCode != null) { + await pressKeyCode(namedKeyCode); + } + return { typed: params.text ?? params.key ?? "", keys: keysPressed }; } diff --git a/packages/tool-server/src/tools/run-sequence/index.ts b/packages/tool-server/src/tools/run-sequence/index.ts index 4502a2073..2f2b3810f 100644 --- a/packages/tool-server/src/tools/run-sequence/index.ts +++ b/packages/tool-server/src/tools/run-sequence/index.ts @@ -104,7 +104,7 @@ Allowed tools and their args (udid is auto-injected, do NOT include it in args): gesture-pinch: { centerX: number, centerY: number, startDistance: number, endDistance: number, angle?: number, durationMs?: number } [ios only] gesture-rotate: { centerX: number, centerY: number, radius: number, startAngle: number, endAngle: number, durationMs?: number } [ios only] button: { button: "home"|"back"|"power"|"volumeUp"|"volumeDown"|"appSwitch"|"actionButton" } [ios/android] - keyboard: { text?: string, key?: string, delayMs?: number } (TV: text only) [ios/android/chromium/vega/tv] + keyboard: { text?: string, key?: string, delayMs?: number } (key pressed after text; TV: text only) [ios/android/chromium/vega/tv] rotate: { orientation: "Portrait"|"LandscapeLeft"|"LandscapeRight"|"PortraitUpsideDown" } [ios/android] tv-remote: { button: , repeat?: number } [apple tv/android tv/vega] buttons: up/down/left/right/select/back/home/menu/playPause (+ rewind/fastForward/next/previous/volumeUp/volumeDown/mute — work on Android TV and Vega; rejected on the Apple TV simulator) @@ -117,10 +117,9 @@ Example — scroll down three times (use gesture-scroll with positive deltaY on { "tool": "gesture-swipe", "args": { "fromX": 0.5, "fromY": 0.7, "toX": 0.5, "toY": 0.3 } } ]} -Example — type text and submit: +Example — type text and submit (one step: the key is pressed after the text is typed): { "udid": "", "steps": [ - { "tool": "keyboard", "args": { "text": "hello world" } }, - { "tool": "keyboard", "args": { "key": "enter" } } + { "tool": "keyboard", "args": { "text": "hello world", "key": "enter" } } ]} Example — TV: move focus right twice then activate (one tv-remote step with a path is cheaper): diff --git a/packages/tool-server/src/utils/vega-input.ts b/packages/tool-server/src/utils/vega-input.ts index a2d3c0cfd..b0d8bbeae 100644 --- a/packages/tool-server/src/utils/vega-input.ts +++ b/packages/tool-server/src/utils/vega-input.ts @@ -157,8 +157,8 @@ export async function injectVegaButtons(buttons: RemoteButton[]): Promise await injectViaInputd(remoteButtonsToKeycodes(buttons).map((code) => `button_press ${code}`)); } -/** Press a single named key (keyboard tool `key` vocabulary). */ -export async function injectVegaNamedKey(name: string): Promise { +/** Resolve a named key (keyboard tool `key` vocabulary) to its inputd keycode, or throw. */ +export function resolveVegaNamedKeycode(name: string): string { const code = NAMED_KEYCODES[name.toLowerCase()]; if (!code) { throw new FailureError( @@ -171,7 +171,12 @@ export async function injectVegaNamedKey(name: string): Promise { } ); } - await injectViaInputd([`button_press ${code}`]); + return code; +} + +/** Press a single named key (keyboard tool `key` vocabulary). */ +export async function injectVegaNamedKey(name: string): Promise { + await injectViaInputd([`button_press ${resolveVegaNamedKeycode(name)}`]); } /** Type text into the focused field via `inputd-cli send_text`. */ diff --git a/packages/tool-server/test/keyboard-key-order.test.ts b/packages/tool-server/test/keyboard-key-order.test.ts new file mode 100644 index 000000000..634f2c7f4 --- /dev/null +++ b/packages/tool-server/test/keyboard-key-order.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from "vitest"; +import type { DeviceInfo } from "@argent/registry"; +import { typeSimulatorServer } from "../src/tools/keyboard/simulator-server-keys"; +import { makeChromiumImpl } from "../src/tools/keyboard/platforms/chromium"; +import { vegaImpl } from "../src/tools/keyboard/platforms/vega"; + +vi.mock("../src/utils/vega-input", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + injectVegaText: vi.fn(async () => {}), + injectVegaNamedKey: vi.fn(async () => {}), + }; +}); + +import { injectVegaNamedKey, injectVegaText } from "../src/utils/vega-input"; + +const IOS_SIM: DeviceInfo = { id: "TEST-UDID", platform: "ios", kind: "simulator" }; +const CHROMIUM: DeviceInfo = { id: "chromium-cdp-9222", platform: "chromium", kind: "app" }; +const VEGA: DeviceInfo = { id: "vega-serial", platform: "vega", kind: "vvd" }; + +const ENTER_HID_KEYCODE = 40; + +function registryWith(api: unknown) { + return { resolveService: vi.fn(async () => api) } as any; +} + +// A combined text+key call means "type, then submit". Pressing the key first +// fires enter into the still-empty field, which blurs it and leaks the text to +// app-level key commands (the React Native dev menu opens on a bare "d" when +// nothing is focused) — the regression behind these tests. +describe("keyboard text+key ordering", () => { + it("simulator-server: presses the named key after the text", async () => { + const downs: number[] = []; + const api = { + pressKey: (direction: "Down" | "Up", keyCode: number) => { + if (direction === "Down") downs.push(keyCode); + }, + }; + + const result = await typeSimulatorServer(registryWith(api), IOS_SIM, { + udid: IOS_SIM.id, + text: "hi", + key: "enter", + delayMs: 0, + }); + + expect(downs).toHaveLength(3); + expect(downs[downs.length - 1]).toBe(ENTER_HID_KEYCODE); + expect(result.keys).toBe(3); + }); + + it("simulator-server: rejects an unknown key before typing any text", async () => { + const pressKey = vi.fn(); + + await expect( + typeSimulatorServer(registryWith({ pressKey }), IOS_SIM, { + udid: IOS_SIM.id, + text: "hi", + key: "bogus", + delayMs: 0, + }) + ).rejects.toThrow(/Unknown key "bogus"/); + expect(pressKey).not.toHaveBeenCalled(); + }); + + it("chromium: dispatches the named key after the text", async () => { + const events: Array<{ type: string; key?: string }> = []; + const api = { + dispatchKeyEvent: async (event: { type: string; key?: string }) => { + events.push(event); + }, + }; + + await makeChromiumImpl(registryWith(api)).handler( + {}, + { udid: CHROMIUM.id, text: "hi", key: "enter", delayMs: 0 }, + CHROMIUM + ); + + const keyDowns = events.filter((e) => e.type === "keyDown").map((e) => e.key); + expect(keyDowns).toEqual(["h", "i", "Enter"]); + }); + + it("chromium: rejects an unknown key before typing any text", async () => { + const dispatchKeyEvent = vi.fn(async () => {}); + + await expect( + makeChromiumImpl(registryWith({ dispatchKeyEvent })).handler( + {}, + { udid: CHROMIUM.id, text: "hi", key: "bogus", delayMs: 0 }, + CHROMIUM + ) + ).rejects.toThrow(/Unknown key "bogus"/); + expect(dispatchKeyEvent).not.toHaveBeenCalled(); + }); + + it("vega: injects the named key after the text", async () => { + const order: string[] = []; + vi.mocked(injectVegaText).mockImplementationOnce(async () => { + order.push("text"); + }); + vi.mocked(injectVegaNamedKey).mockImplementationOnce(async () => { + order.push("key"); + }); + + await vegaImpl.handler({}, { udid: VEGA.id, text: "hi", key: "enter" }, VEGA); + + expect(order).toEqual(["text", "key"]); + }); + + it("vega: rejects an unknown key before typing any text", async () => { + vi.mocked(injectVegaText).mockClear(); + vi.mocked(injectVegaNamedKey).mockClear(); + + await expect( + vegaImpl.handler({}, { udid: VEGA.id, text: "hi", key: "bogus" }, VEGA) + ).rejects.toThrow(/Unknown Vega key "bogus"/); + expect(injectVegaText).not.toHaveBeenCalled(); + expect(injectVegaNamedKey).not.toHaveBeenCalled(); + }); +});