Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/tool-server/src/tools/keyboard/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const zodSchema = z.object({
.string()
.optional()
.describe(
"Named key to press: enter, escape, backspace, tab, space, arrow-up, arrow-down, arrow-left, arrow-right, f1–f12. Not supported on TV targets — move focus with `tv-remote` (up/down/left/right) instead."
"Named key to press: enter, escape, backspace, tab, space, arrow-up, arrow-down, arrow-left, arrow-right, f1–f12. When combined with `text`, the key is pressed AFTER the text is typed (so text + enter types and submits). Not supported on TV targets — move focus with `tv-remote` (up/down/left/right) instead."
),
delayMs: z
.number()
Expand Down Expand Up @@ -58,7 +58,7 @@ Returns { typed: string, keys: number }. Fails if an unsupported key name is pro
- text: types a string character by character (supports uppercase, digits, common punctuation)
- key: presses a single named key (enter, escape, backspace, tab, arrow-up/down/left/right, f1–f12) — NOT supported on TV targets; move focus with \`tv-remote\` instead.
On a TV target (runtimeKind 'tv') only \`text\` applies — focus a text field first (with \`tv-remote\`), then type into it (injected HID keyboard on Apple TV, \`adb input text\` on Android TV).
Provide text, key, or both.`,
Provide text, key, or both — when both are given, the text is typed first and the key is pressed after it (text + key:"enter" types and submits).`,
zodSchema,
capability,
searchHint:
Expand Down
38 changes: 23 additions & 15 deletions packages/tool-server/src/tools/keyboard/platforms/chromium.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@ async function runChromium(api: ChromiumCdpApi, params: KeyboardParams): Promise
const delay = params.delayMs ?? 50;
let keysPressed = 0;

// Resolve the named key before typing anything so an unknown name fails
// fast instead of after the text has already been typed.
let named: (typeof CHROMIUM_NAMED_KEYS)[string] | undefined;
if (params.key) {
const named = CHROMIUM_NAMED_KEYS[params.key.toLowerCase()];
named = CHROMIUM_NAMED_KEYS[params.key.toLowerCase()];
if (!named) {
throw new FailureError(
`Unknown key "${params.key}". Supported: ${Object.keys(CHROMIUM_NAMED_KEYS).join(", ")}`,
Expand All @@ -23,20 +26,6 @@ async function runChromium(api: ChromiumCdpApi, params: KeyboardParams): Promise
}
);
}
await api.dispatchKeyEvent({
type: "keyDown",
key: named.key,
code: named.code,
windowsVirtualKeyCode: named.windowsVirtualKeyCode,
});
await sleep(delay);
await api.dispatchKeyEvent({
type: "keyUp",
key: named.key,
code: named.code,
windowsVirtualKeyCode: named.windowsVirtualKeyCode,
});
keysPressed++;
}

if (params.text) {
Expand Down Expand Up @@ -70,6 +59,25 @@ async function runChromium(api: ChromiumCdpApi, params: KeyboardParams): Promise
}
}

// Key after text: a combined call means "type, then submit" (text +
// key:"enter"). Pressing the key first submits the still-empty field.
if (named) {
await api.dispatchKeyEvent({
type: "keyDown",
key: named.key,
code: named.code,
windowsVirtualKeyCode: named.windowsVirtualKeyCode,
});
await sleep(delay);
await api.dispatchKeyEvent({
type: "keyUp",
key: named.key,
code: named.code,
windowsVirtualKeyCode: named.windowsVirtualKeyCode,
});
keysPressed++;
}

return { typed: params.text ?? params.key ?? "", keys: keysPressed };
}

Expand Down
18 changes: 13 additions & 5 deletions packages/tool-server/src/tools/keyboard/platforms/vega.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import type { PlatformImpl } from "../../../utils/cross-platform-tool";
import { injectVegaNamedKey, injectVegaText } from "../../../utils/vega-input";
import {
injectVegaNamedKey,
injectVegaText,
resolveVegaNamedKeycode,
} from "../../../utils/vega-input";
import type { KeyboardParams, KeyboardResult } from "../types";

// Vega has no simulator-server: input is injected over `adb` (on-device
Expand All @@ -8,14 +12,18 @@ import type { KeyboardParams, KeyboardResult } from "../types";
// adb fails with a clean 424 install hint rather than a spawn ENOENT.
async function runVega(params: KeyboardParams): Promise<KeyboardResult> {
let keysPressed = 0;
if (params.key) {
await injectVegaNamedKey(params.key);
keysPressed++;
}
// Resolve the named key before injecting text so an unknown name fails fast.
if (params.key) resolveVegaNamedKeycode(params.key);
if (params.text) {
await injectVegaText(params.text);
keysPressed += [...params.text].length;
}
// Key after text: a combined call means "type, then submit" (text +
// key:"enter"). Pressing the key first submits the still-empty field.
if (params.key) {
await injectVegaNamedKey(params.key);
keysPressed++;
}
return { typed: params.text ?? params.key ?? "", keys: keysPressed };
}

Expand Down
16 changes: 13 additions & 3 deletions packages/tool-server/src/tools/keyboard/simulator-server-keys.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,12 @@ export async function typeSimulatorServer(
keysPressed++;
};

// Resolve the named key before typing anything so an unknown name fails
// fast instead of after the text has already been typed.
let namedKeyCode: number | undefined;
if (params.key) {
const code = NAMED_KEYS[params.key.toLowerCase()];
if (code == null) {
namedKeyCode = NAMED_KEYS[params.key.toLowerCase()];
if (namedKeyCode == null) {
throw new FailureError(
`Unknown key "${params.key}". Supported: ${Object.keys(NAMED_KEYS).join(", ")}`,
{
Expand All @@ -48,7 +51,6 @@ export async function typeSimulatorServer(
}
);
}
await pressKeyCode(code);
}

if (params.text) {
Expand All @@ -66,5 +68,13 @@ export async function typeSimulatorServer(
}
}

// Key after text: a combined call means "type, then submit" (text +
// key:"enter"). Pressing the key first fires enter into the still-empty
// field, which can blur it and leak the text to app-level key commands
// (e.g. "d" toggles the React Native dev menu when nothing is focused).
if (namedKeyCode != null) {
await pressKeyCode(namedKeyCode);
}

return { typed: params.text ?? params.key ?? "", keys: keysPressed };
}
7 changes: 3 additions & 4 deletions packages/tool-server/src/tools/run-sequence/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ Allowed tools and their args (udid is auto-injected, do NOT include it in args):
gesture-pinch: { centerX: number, centerY: number, startDistance: number, endDistance: number, angle?: number, durationMs?: number } [ios only]
gesture-rotate: { centerX: number, centerY: number, radius: number, startAngle: number, endAngle: number, durationMs?: number } [ios only]
button: { button: "home"|"back"|"power"|"volumeUp"|"volumeDown"|"appSwitch"|"actionButton" } [ios/android]
keyboard: { text?: string, key?: string, delayMs?: number } (TV: text only) [ios/android/chromium/vega/tv]
keyboard: { text?: string, key?: string, delayMs?: number } (key pressed after text; TV: text only) [ios/android/chromium/vega/tv]
rotate: { orientation: "Portrait"|"LandscapeLeft"|"LandscapeRight"|"PortraitUpsideDown" } [ios/android]
tv-remote: { button: <remote button | array of them>, repeat?: number } [apple tv/android tv/vega]
buttons: up/down/left/right/select/back/home/menu/playPause (+ rewind/fastForward/next/previous/volumeUp/volumeDown/mute — work on Android TV and Vega; rejected on the Apple TV simulator)
Expand All @@ -117,10 +117,9 @@ Example — scroll down three times (use gesture-scroll with positive deltaY on
{ "tool": "gesture-swipe", "args": { "fromX": 0.5, "fromY": 0.7, "toX": 0.5, "toY": 0.3 } }
]}

Example — type text and submit:
Example — type text and submit (one step: the key is pressed after the text is typed):
{ "udid": "<UDID>", "steps": [
{ "tool": "keyboard", "args": { "text": "hello world" } },
{ "tool": "keyboard", "args": { "key": "enter" } }
{ "tool": "keyboard", "args": { "text": "hello world", "key": "enter" } }
]}

Example — TV: move focus right twice then activate (one tv-remote step with a path is cheaper):
Expand Down
11 changes: 8 additions & 3 deletions packages/tool-server/src/utils/vega-input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,8 @@ export async function injectVegaButtons(buttons: RemoteButton[]): Promise<void>
await injectViaInputd(remoteButtonsToKeycodes(buttons).map((code) => `button_press ${code}`));
}

/** Press a single named key (keyboard tool `key` vocabulary). */
export async function injectVegaNamedKey(name: string): Promise<void> {
/** Resolve a named key (keyboard tool `key` vocabulary) to its inputd keycode, or throw. */
export function resolveVegaNamedKeycode(name: string): string {
const code = NAMED_KEYCODES[name.toLowerCase()];
if (!code) {
throw new FailureError(
Expand All @@ -171,7 +171,12 @@ export async function injectVegaNamedKey(name: string): Promise<void> {
}
);
}
await injectViaInputd([`button_press ${code}`]);
return code;
}

/** Press a single named key (keyboard tool `key` vocabulary). */
export async function injectVegaNamedKey(name: string): Promise<void> {
await injectViaInputd([`button_press ${resolveVegaNamedKeycode(name)}`]);
}

/** Type text into the focused field via `inputd-cli send_text`. */
Expand Down
122 changes: 122 additions & 0 deletions packages/tool-server/test/keyboard-key-order.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { describe, expect, it, vi } from "vitest";
import type { DeviceInfo } from "@argent/registry";
import { typeSimulatorServer } from "../src/tools/keyboard/simulator-server-keys";
import { makeChromiumImpl } from "../src/tools/keyboard/platforms/chromium";
import { vegaImpl } from "../src/tools/keyboard/platforms/vega";

vi.mock("../src/utils/vega-input", async (importOriginal) => {
const actual = await importOriginal<typeof import("../src/utils/vega-input")>();
return {
...actual,
injectVegaText: vi.fn(async () => {}),
injectVegaNamedKey: vi.fn(async () => {}),
};
});

import { injectVegaNamedKey, injectVegaText } from "../src/utils/vega-input";

const IOS_SIM: DeviceInfo = { id: "TEST-UDID", platform: "ios", kind: "simulator" };
const CHROMIUM: DeviceInfo = { id: "chromium-cdp-9222", platform: "chromium", kind: "app" };
const VEGA: DeviceInfo = { id: "vega-serial", platform: "vega", kind: "vvd" };

const ENTER_HID_KEYCODE = 40;

function registryWith(api: unknown) {
return { resolveService: vi.fn(async () => api) } as any;
}

// A combined text+key call means "type, then submit". Pressing the key first
// fires enter into the still-empty field, which blurs it and leaks the text to
// app-level key commands (the React Native dev menu opens on a bare "d" when
// nothing is focused) — the regression behind these tests.
describe("keyboard text+key ordering", () => {
it("simulator-server: presses the named key after the text", async () => {
const downs: number[] = [];
const api = {
pressKey: (direction: "Down" | "Up", keyCode: number) => {
if (direction === "Down") downs.push(keyCode);
},
};

const result = await typeSimulatorServer(registryWith(api), IOS_SIM, {
udid: IOS_SIM.id,
text: "hi",
key: "enter",
delayMs: 0,
});

expect(downs).toHaveLength(3);
expect(downs[downs.length - 1]).toBe(ENTER_HID_KEYCODE);
expect(result.keys).toBe(3);
});

it("simulator-server: rejects an unknown key before typing any text", async () => {
const pressKey = vi.fn();

await expect(
typeSimulatorServer(registryWith({ pressKey }), IOS_SIM, {
udid: IOS_SIM.id,
text: "hi",
key: "bogus",
delayMs: 0,
})
).rejects.toThrow(/Unknown key "bogus"/);
expect(pressKey).not.toHaveBeenCalled();
});

it("chromium: dispatches the named key after the text", async () => {
const events: Array<{ type: string; key?: string }> = [];
const api = {
dispatchKeyEvent: async (event: { type: string; key?: string }) => {
events.push(event);
},
};

await makeChromiumImpl(registryWith(api)).handler(
{},
{ udid: CHROMIUM.id, text: "hi", key: "enter", delayMs: 0 },
CHROMIUM
);

const keyDowns = events.filter((e) => e.type === "keyDown").map((e) => e.key);
expect(keyDowns).toEqual(["h", "i", "Enter"]);
});

it("chromium: rejects an unknown key before typing any text", async () => {
const dispatchKeyEvent = vi.fn(async () => {});

await expect(
makeChromiumImpl(registryWith({ dispatchKeyEvent })).handler(
{},
{ udid: CHROMIUM.id, text: "hi", key: "bogus", delayMs: 0 },
CHROMIUM
)
).rejects.toThrow(/Unknown key "bogus"/);
expect(dispatchKeyEvent).not.toHaveBeenCalled();
});

it("vega: injects the named key after the text", async () => {
const order: string[] = [];
vi.mocked(injectVegaText).mockImplementationOnce(async () => {
order.push("text");
});
vi.mocked(injectVegaNamedKey).mockImplementationOnce(async () => {
order.push("key");
});

await vegaImpl.handler({}, { udid: VEGA.id, text: "hi", key: "enter" }, VEGA);

expect(order).toEqual(["text", "key"]);
});

it("vega: rejects an unknown key before typing any text", async () => {
vi.mocked(injectVegaText).mockClear();
vi.mocked(injectVegaNamedKey).mockClear();

await expect(
vegaImpl.handler({}, { udid: VEGA.id, text: "hi", key: "bogus" }, VEGA)
).rejects.toThrow(/Unknown Vega key "bogus"/);
expect(injectVegaText).not.toHaveBeenCalled();
expect(injectVegaNamedKey).not.toHaveBeenCalled();
});
});
Loading