From 18012fd55a2dd5940f55e743a9e0244f40f3ac3e Mon Sep 17 00:00:00 2001 From: billzh <10249149+thebillzh@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:07:20 -0600 Subject: [PATCH] feat: add native keyboard scenario and compact describe --- .../tool-server/src/blueprints/ax-service.ts | 1 + .../src/tools/await-ui-element/index.ts | 78 +- .../src/tools/describe/contract.ts | 9 +- .../src/tools/describe/format-tree.ts | 201 +++++ .../tool-server/src/tools/describe/index.ts | 138 +++- .../platforms/android/uiautomator-parser.ts | 13 +- .../describe/platforms/ios/ios-ax-adapter.ts | 4 +- .../platforms/ios/ios-native-adapter.ts | 4 +- .../src/tools/describe/platforms/tv.ts | 44 +- .../src/tools/describe/selectors.ts | 92 +++ .../src/tools/keyboard-scenario/index.ts | 722 ++++++++++++++++++ .../native-describe-contract.ts | 1 + .../tool-server/src/utils/setup-registry.ts | 2 + .../test/describe-ax-adapter.test.ts | 11 + .../tool-server/test/describe-compact.test.ts | 159 ++++ .../test/describe-native-adapter.test.ts | 13 + .../tool-server/test/describe-tool.test.ts | 3 + .../test/keyboard-scenario.test.ts | 425 +++++++++++ .../test/native-describe-contract.test.ts | 2 + .../test/uiautomator-parser-v2-trim.test.ts | 22 + 20 files changed, 1846 insertions(+), 98 deletions(-) create mode 100644 packages/tool-server/src/tools/describe/selectors.ts create mode 100644 packages/tool-server/src/tools/keyboard-scenario/index.ts create mode 100644 packages/tool-server/test/describe-compact.test.ts create mode 100644 packages/tool-server/test/keyboard-scenario.test.ts diff --git a/packages/tool-server/src/blueprints/ax-service.ts b/packages/tool-server/src/blueprints/ax-service.ts index 7e7a48116..2225c32df 100644 --- a/packages/tool-server/src/blueprints/ax-service.ts +++ b/packages/tool-server/src/blueprints/ax-service.ts @@ -57,6 +57,7 @@ export interface AXDescribeElement { traits?: string[]; value?: string; identifier?: string; + focused?: boolean; } export interface AXDescribeResponse { diff --git a/packages/tool-server/src/tools/await-ui-element/index.ts b/packages/tool-server/src/tools/await-ui-element/index.ts index 864fa4037..0fef74920 100644 --- a/packages/tool-server/src/tools/await-ui-element/index.ts +++ b/packages/tool-server/src/tools/await-ui-element/index.ts @@ -14,6 +14,11 @@ import { assertSupported } from "../../utils/capability"; import { ensureDeps } from "../../utils/check-deps"; import { pollDescribeTree } from "../../utils/poll-describe-tree"; import type { DescribeNode, DescribeTreeData } from "../describe/contract"; +import { + describeSelectorSchema, + findDescribeMatches, + type DescribeSelector, +} from "../describe/selectors"; import { describeIos, iosRequires } from "../describe/platforms/ios"; import { describeAndroid, androidRequires } from "../describe/platforms/android"; import { describeChromium } from "../describe/platforms/chromium"; @@ -42,36 +47,8 @@ export function isUnmetUiWaitResult(tool: string, result: unknown): boolean { const DEFAULT_TIMEOUT_MS = 5000; const DEFAULT_POLL_INTERVAL_MS = 400; -// A selector locates a node in the accessibility / DOM tree returned by -// `describe`. Every provided field must match (logical AND); matching is a -// case-insensitive substring test so the agent doesn't need the exact label. -const selectorSchema = z - .object({ - text: z - .string() - .min(1) - .optional() - .describe("Case-insensitive substring of the element's visible label or value."), - identifier: z - .string() - .min(1) - .optional() - .describe( - "Case-insensitive substring of the element's identifier (accessibilityIdentifier / resource-id / testid)." - ), - role: z - .string() - .min(1) - .optional() - .describe( - "Case-insensitive substring of the element's role (e.g. AXButton, button, TextView)." - ), - }) - .refine((s) => Boolean(s.text || s.identifier || s.role), { - message: "selector needs at least one of text, identifier, or role", - }); - -type Selector = z.infer; +// Selector matching is shared with `describe` so both tools interpret the same +// selector identically. const zodSchema = z .object({ @@ -87,7 +64,9 @@ const zodSchema = z "or zero-area. `text`: the first match in reading order (topmost) contains expectedText — if a loose " + "selector hits several elements, only that topmost one is checked, so narrow it to target the intended element." ), - selector: selectorSchema.describe("Element to match (text / identifier / role)."), + selector: describeSelectorSchema.describe( + "Element to match (text / identifier / role / Android package)." + ), expectedText: z .string() .min(1) @@ -143,30 +122,6 @@ function nodeText(node: DescribeNode): string { return [node.label, node.value].filter(Boolean).join(" "); } -function includesCI(haystack: string | undefined, needle: string): boolean { - return Boolean(haystack) && haystack!.toLowerCase().includes(needle.toLowerCase()); -} - -function matchNode(node: DescribeNode, selector: Selector): boolean { - if (selector.text !== undefined) { - if (!includesCI(node.label, selector.text) && !includesCI(node.value, selector.text)) { - return false; - } - } - if (selector.identifier !== undefined && !includesCI(node.identifier, selector.identifier)) { - return false; - } - if (selector.role !== undefined && !includesCI(node.role, selector.role)) { - return false; - } - return true; -} - -function collectMatches(node: DescribeNode, selector: Selector, acc: DescribeNode[]): void { - if (matchNode(node, selector)) acc.push(node); - for (const child of node.children) collectMatches(child, selector, acc); -} - // Every node matching the selector in the subtree, EXCLUDING `root` itself. // // `root` is the top-level container describe puts at the head of the tree. On @@ -192,10 +147,8 @@ function collectMatches(node: DescribeNode, selector: Selector, acc: DescribeNod // `visible`/`exists` selector is broad. A substring selector can also match // several real nodes, so conditions are evaluated across the whole set (see // evaluateMatches). Exported for unit tests. -export function findAll(root: DescribeNode, selector: Selector): DescribeNode[] { - const acc: DescribeNode[] = []; - for (const child of root.children) collectMatches(child, selector, acc); - return acc; +export function findAll(root: DescribeNode, selector: DescribeSelector): DescribeNode[] { + return findDescribeMatches(root, selector); } // describe prunes off-screen / zero-size nodes on Chromium and the compressed @@ -242,7 +195,10 @@ export function evaluateMatches(params: Params, matches: DescribeNode[]): boolea return !matches.some(isVisible); case "text": { const first = firstInReadingOrder(matches); - return first !== undefined && includesCI(nodeText(first), params.expectedText!); + return ( + first !== undefined && + nodeText(first).toLowerCase().includes(params.expectedText!.toLowerCase()) + ); } default: return false; @@ -352,7 +308,7 @@ Conditions: substring). A loose selector can match several elements; only that topmost one is inspected, so if a lower match is the one holding the text the wait still reports failure — narrow the selector to target it. -The selector is { text?, identifier?, role? }; every provided field must match (case-insensitive substring). +The selector is { text?, identifier?, role?, package? }; every provided field must match (case-insensitive substring). text matches the element's label or value. It polls the same accessibility / DOM tree as \`describe\` (iOS AXRuntime, Android uiautomator, Chromium CDP) every pollIntervalMs (default ${DEFAULT_POLL_INTERVAL_MS}ms) until timeoutMs (default ${DEFAULT_TIMEOUT_MS}ms). diff --git a/packages/tool-server/src/tools/describe/contract.ts b/packages/tool-server/src/tools/describe/contract.ts index cae35e884..b2ae3e719 100644 --- a/packages/tool-server/src/tools/describe/contract.ts +++ b/packages/tool-server/src/tools/describe/contract.ts @@ -16,6 +16,7 @@ export interface DescribeNode { label?: string; identifier?: string; value?: string; + packageName?: string; // Interactivity flags surfaced by the Android uiautomator dump. iOS // consumers leave these unset; adding them as optional avoids breaking // existing payloads. `scrollHidden` counts children that fell outside an @@ -32,7 +33,8 @@ export interface DescribeNode { // `focused` is the element holding input focus; `selected` is the visually // highlighted / active item (e.g. the current nav tab). On Vega the toolkit // often reports the highlighted item via `selected` while `focused` stays - // false, so both are surfaced. Other platforms leave these unset. + // false, so both are surfaced. iOS and Android also surface `focused` when + // their native accessibility providers report an element holding input focus. focused?: boolean; selected?: boolean; } @@ -46,6 +48,7 @@ export const describeNodeSchema: z.ZodType = z.lazy(() => label: z.string().optional(), identifier: z.string().optional(), value: z.string().optional(), + packageName: z.string().optional(), clickable: z.boolean().optional(), longClickable: z.boolean().optional(), scrollable: z.boolean().optional(), @@ -98,6 +101,10 @@ export interface DescribeResult { source: DescribeSource; should_restart?: boolean; hint?: string; + // Present only for selector-driven compact describe calls. + matched?: number; + emitted?: number; + truncated?: boolean; } export function parseDescribeResult(input: unknown): DescribeNode { diff --git a/packages/tool-server/src/tools/describe/format-tree.ts b/packages/tool-server/src/tools/describe/format-tree.ts index 82d3ef156..fc2269f68 100644 --- a/packages/tool-server/src/tools/describe/format-tree.ts +++ b/packages/tool-server/src/tools/describe/format-tree.ts @@ -1,4 +1,5 @@ import type { DescribeFrame, DescribeNode, DescribeSource } from "./contract"; +import { findDescribeMatches, type DescribeSelector } from "./selectors"; // Token-efficient view of a pruned DescribeNode tree. The previous shape sent // the full JSON tree to the agent, which on a typical iOS screen ran ~6× the @@ -220,3 +221,203 @@ export function formatDescribeTree(root: DescribeNode, opts: FormatDescribeOptio const body = mode === "flat" ? renderFlat(root, contentRoles) : renderNested(root, contentRoles); return [...header, ...body].join("\n").replace(/\n+$/, "\n"); } + +export const DESCRIBE_FIELDS = [ + "role", + "label", + "value", + "identifier", + "package", + "flags", + "frame", +] as const; +export type DescribeField = (typeof DESCRIBE_FIELDS)[number]; +export type DescribeProjection = "matches" | "matches-and-ancestors" | "full"; + +export interface FormatDescribeSelectionOptions { + source: DescribeSource; + selector: DescribeSelector; + projection: DescribeProjection; + fields: readonly DescribeField[]; + limit: number; + maxChars: number; +} + +export interface FormattedDescribeSelection { + description: string; + matched: number; + emitted: number; + truncated: boolean; +} + +interface SelectedLine { + node: DescribeNode; + depth: number; + matched: boolean; +} + +function selectionLine( + node: DescribeNode, + depth: number, + fields: ReadonlySet, + matched: boolean +): string { + const parts: string[] = []; + if (fields.has("role")) parts.push(node.role); + if (fields.has("label") && node.label) parts.push(formatLabel(node.label)); + if (fields.has("value") && node.value && node.value !== node.label) { + parts.push(`value="${escapeForLine(node.value)}"`); + } + if (fields.has("identifier") && node.identifier) { + parts.push(`id="${escapeForLine(node.identifier)}"`); + } + if (fields.has("package") && node.packageName) { + parts.push(`package="${escapeForLine(node.packageName)}"`); + } + if (fields.has("flags")) { + const flags = formatFlags(node).trim(); + if (flags) parts.push(flags); + } + // Match highlighting is projection metadata rather than a node field. Keep it + // even when `flags` is omitted so a `full` projection remains interpretable. + if (matched) parts.push("[match]"); + if (fields.has("frame")) parts.push(fmtFrame(node.frame)); + return `${" ".repeat(depth)}${parts.join(" ") || "[element]"}`; +} + +function isNestedSource(source: DescribeSource): boolean { + return ( + source === "uiautomator" || + source === "android-devtools" || + source === "cdp-dom" || + source === "vega-automation" + ); +} + +function orderedFlatChildren(root: DescribeNode): DescribeNode[] { + return root.children.slice().sort((a, b) => a.frame.y - b.frame.y || a.frame.x - b.frame.x); +} + +function collectFullSelection( + root: DescribeNode, + source: DescribeSource, + matchSet: ReadonlySet +): SelectedLine[] { + const contentRoles = source === "vega-automation" ? VEGA_CONTENT_ROLES : CONTENT_ROLES; + if (!isNestedSource(source)) { + return orderedFlatChildren(root) + .filter((node) => shouldEmit(node, contentRoles)) + .map((node) => ({ node, depth: 1, matched: matchSet.has(node) })); + } + + const lines: SelectedLine[] = []; + const stack = root.children + .slice() + .reverse() + .map((node) => ({ node, depth: 1 })); + while (stack.length > 0) { + const { node, depth } = stack.pop()!; + if (shouldEmit(node, contentRoles) || node.children.length > 0) { + lines.push({ node, depth, matched: matchSet.has(node) }); + } + for (let index = node.children.length - 1; index >= 0; index--) { + stack.push({ node: node.children[index]!, depth: depth + 1 }); + } + } + return lines; +} + +function collectMatchedPaths( + root: DescribeNode, + matchSet: ReadonlySet +): SelectedLine[] { + const lines: SelectedLine[] = []; + function visit(node: DescribeNode, depth: number): boolean { + const start = lines.length; + const ownMatch = matchSet.has(node); + lines.push({ node, depth, matched: ownMatch }); + let descendantMatch = false; + for (const child of node.children) { + if (visit(child, depth + 1)) descendantMatch = true; + } + if (!ownMatch && !descendantMatch) lines.splice(start); + return ownMatch || descendantMatch; + } + for (const child of root.children) visit(child, 1); + return lines; +} + +function renderBoundedSelection( + header: readonly string[], + candidateLines: readonly string[], + matched: number, + limit: number, + maxChars: number +): { description: string; emitted: number; truncated: boolean } { + let lines = candidateLines.slice(0, limit); + let truncated = candidateLines.length > limit; + + const assemble = (body: readonly string[], isTruncated: boolean): string => { + const sections = [...header, ...body]; + if (isTruncated) { + sections.push(`… truncated (matched=${matched}, emitted=${body.length}).`); + } + return `${sections.join("\n")}\n`; + }; + + let description = assemble(lines, truncated); + if (description.length > maxChars) { + truncated = true; + while (lines.length > 0 && assemble(lines, true).length > maxChars) lines.pop(); + description = assemble(lines, true); + // The schema minimum leaves room for the compact header and marker. Keep a + // defensive bounded fallback in case future header copy grows. + if (description.length > maxChars) { + const marker = `… truncated (matched=${matched}, emitted=0).\n`; + description = `${header.join("\n").slice(0, Math.max(0, maxChars - marker.length - 1))}\n${marker}`; + lines = []; + } + } + + return { description, emitted: lines.length, truncated }; +} + +export function formatDescribeSelection( + root: DescribeNode, + opts: FormatDescribeSelectionOptions +): FormattedDescribeSelection { + const matches = findDescribeMatches(root, opts.selector); + const matchSet = new Set(matches); + let selected: SelectedLine[]; + + switch (opts.projection) { + case "matches": { + const ordered = isNestedSource(opts.source) + ? matches.map((node) => ({ node, depth: 0, matched: true })) + : orderedFlatChildren(root) + .filter((node) => matchSet.has(node)) + .map((node) => ({ node, depth: 0, matched: true })); + selected = ordered.map((line) => ({ ...line, depth: 0 })); + break; + } + case "matches-and-ancestors": + selected = collectMatchedPaths(root, matchSet); + break; + case "full": + selected = collectFullSelection(root, opts.source, matchSet); + break; + } + + const fields = new Set(opts.fields); + const body = selected.map(({ node, depth, matched }) => + selectionLine(node, depth, fields, matched) + ); + const header = [ + `Source: ${opts.source}`, + `Projection: ${opts.projection}`, + `Matched: ${matches.length}`, + "", + ]; + const bounded = renderBoundedSelection(header, body, matches.length, opts.limit, opts.maxChars); + return { matched: matches.length, ...bounded }; +} diff --git a/packages/tool-server/src/tools/describe/index.ts b/packages/tool-server/src/tools/describe/index.ts index 3f4221830..77be2078e 100644 --- a/packages/tool-server/src/tools/describe/index.ts +++ b/packages/tool-server/src/tools/describe/index.ts @@ -17,39 +17,107 @@ import { chromiumCdpRef, type ChromiumCdpApi } from "../../blueprints/chromium-c import { resolveDevice } from "../../utils/device-info"; import { isTvOsSimulator } from "../../utils/ios-devices"; import { isAndroidTv } from "../../utils/adb"; -import { formatDescribeTree } from "./format-tree"; +import { DESCRIBE_FIELDS, formatDescribeSelection, formatDescribeTree } from "./format-tree"; +import { describeSelectorSchema } from "./selectors"; // In-between layer between the per-platform adapters (which still own all // pruning — the Android v2 trimmer in uiautomator-parser stays untouched) and // the public DescribeResult. The internal `tree` is converted to a token- // efficient text rendering here and then dropped, so the caller (LLM) never // pays for the JSON tree. -function withDescription(data: DescribeTreeData): DescribeResult { - const out: DescribeResult = { - description: formatDescribeTree(data.tree, { source: data.source }), - source: data.source, - }; +function compactOptions(params: Params) { + if (!params.selector) return undefined; + return { + selector: params.selector, + projection: params.projection ?? "matches", + fields: params.fields ?? DESCRIBE_FIELDS, + limit: params.limit ?? 50, + maxChars: params.maxChars ?? 12_000, + } as const; +} + +function withDescription(data: DescribeTreeData, params: Params): DescribeResult { + // The selector-less path deliberately stays byte-for-byte compatible with + // the original formatter and response shape. + const options = compactOptions(params); + const compact = options + ? formatDescribeSelection(data.tree, { source: data.source, ...options }) + : undefined; + const out: DescribeResult = compact + ? { source: data.source, ...compact } + : { + description: formatDescribeTree(data.tree, { source: data.source }), + source: data.source, + }; if (data.should_restart) out.should_restart = data.should_restart; if (data.hint) out.hint = data.hint; return out; } -const zodSchema = z.object({ - udid: z - .string() - .min(1) - .describe( - "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, or Chromium id)." - ), - bundleId: z - .string() - .optional() - .describe( - "Optional app bundle ID. Used as a target hint on iOS when the AX-service returns no elements " + - "and the describe tool falls back to native-devtools inspection. " + - "If omitted, the fallback auto-detects the frontmost connected app. Ignored on Android / Chromium." - ), -}); +const zodSchema = z + .object({ + udid: z + .string() + .min(1) + .describe( + "Target device id from `list-devices` (iOS UDID, Android serial, Vega serial, or Chromium id)." + ), + bundleId: z + .string() + .optional() + .describe( + "Optional app bundle ID. Used as a target hint on iOS when the AX-service returns no elements " + + "and the describe tool falls back to native-devtools inspection. " + + "If omitted, the fallback auto-detects the frontmost connected app. Ignored on Android / Chromium." + ), + selector: describeSelectorSchema + .optional() + .describe( + "Optional element selector. When omitted, describe preserves its full legacy output exactly." + ), + projection: z + .enum(["matches", "matches-and-ancestors", "full"]) + .optional() + .describe( + "Selector output shape (default `matches`): matching elements only, their ancestor paths, or the full tree with matches highlighted." + ), + fields: z + .array(z.enum(DESCRIBE_FIELDS)) + .min(1) + .optional() + .describe( + "Element fields to render in selector mode. Defaults to role, label, value, identifier, package, flags, and frame." + ), + limit: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe("Maximum element lines emitted in selector mode (default 50)."), + maxChars: z + .number() + .int() + .min(256) + .max(100_000) + .optional() + .describe("Maximum description characters in selector mode (default 12000)."), + }) + .superRefine((params, ctx) => { + if ( + params.selector === undefined && + (params.projection !== undefined || + params.fields !== undefined || + params.limit !== undefined || + params.maxChars !== undefined) + ) { + ctx.addIssue({ + code: "custom", + path: ["selector"], + message: "projection, fields, limit, and maxChars require selector", + }); + } + }); type Params = z.infer; @@ -104,8 +172,8 @@ function makeDescribeExecute( handler: async (_services, params, device) => // Probe tvOS once here, then pass the verdict into describeIos. (await isTvOsSimulator(device.id)) - ? describeTv(registry, device) - : withDescription(await describeIos(registry, device, params, { isTvOs: false })), + ? describeTv(registry, device, compactOptions(params)) + : withDescription(await describeIos(registry, device, params, { isTvOs: false }), params), }, iosRemote: { // describeIos already handles both ax-service (TCP) and native-devtools @@ -114,7 +182,7 @@ function makeDescribeExecute( // (never tvOS), so the isTvOs verdict is always false. requires: ["sim-remote"], handler: async (_services, params, device) => - withDescription(await describeIos(registry, device, params, { isTvOs: false })), + withDescription(await describeIos(registry, device, params, { isTvOs: false }), params), }, android: { requires: androidRequires, @@ -123,15 +191,20 @@ function makeDescribeExecute( // focus-driven describe, a phone to the uiautomator tree — and pass the // known `isTv: false` through so describeAndroid doesn't re-probe. (await isAndroidTv(device.id)) - ? describeTv(registry, device) - : withDescription(await describeAndroid(registry, params.udid, params.bundleId, false)), + ? describeTv(registry, device, compactOptions(params)) + : withDescription( + await describeAndroid(registry, params.udid, params.bundleId, false), + params + ), }, chromium: { - handler: async (services) => withDescription(await describeChromium(services.chromium)), + handler: async (services, params) => + withDescription(await describeChromium(services.chromium), params), }, vega: { requires: vegaRequires, - handler: async (_services, params) => withDescription(await describeVega(params.udid)), + handler: async (_services, params) => + withDescription(await describeVega(params.udid), params), }, }); } @@ -157,6 +230,13 @@ line per element with its role, label/value/id, interactivity flags, and frame. are normalized [0,1] fractions of the screen / window width/height (not pixels) — the same space as gesture-tap / gesture-swipe / gesture-pinch. +For a smaller targeted response, pass selector \`{ text?, identifier?, role?, package? }\`; every supplied +field matches as a case-insensitive substring. Selector mode defaults to projection \`matches\`, all +fields (including Android package provenance), limit 50, and maxChars 12000. Use \`matches-and-ancestors\` to retain the path to each match, +or \`full\` to retain the complete tree while highlighting matches. Selector responses also return +\`matched\`, \`emitted\`, and \`truncated\`; truncation is marked in the description. Omitting selector +preserves the original full description and response shape exactly. + To tap an element use the centre of its frame: \`tap_x = frame.x + frame.width / 2\`, \`tap_y = frame.y + frame.height / 2\`. The same formula appears in the response header so it can be applied to a line in isolation. diff --git a/packages/tool-server/src/tools/describe/platforms/android/uiautomator-parser.ts b/packages/tool-server/src/tools/describe/platforms/android/uiautomator-parser.ts index 0fad33474..2e8dd869b 100644 --- a/packages/tool-server/src/tools/describe/platforms/android/uiautomator-parser.ts +++ b/packages/tool-server/src/tools/describe/platforms/android/uiautomator-parser.ts @@ -233,6 +233,8 @@ interface UiNode { checked: boolean; disabled: boolean; password: boolean; + packageName?: string; + focused: boolean; scrollHidden: number; children: UiNode[]; } @@ -252,7 +254,8 @@ function isInteractive(attrs: Record): boolean { attrIsTrue(attrs, "clickable") || attrIsTrue(attrs, "long-clickable") || attrIsTrue(attrs, "checkable") || - attrIsTrue(attrs, "scrollable") + attrIsTrue(attrs, "scrollable") || + attrIsTrue(attrs, "focused") ) { return true; } @@ -359,6 +362,8 @@ function makeUiNode( checked: attrIsTrue(attrs, "checked"), disabled: attrs.enabled === "false", password: attrIsTrue(attrs, "password"), + focused: attrIsTrue(attrs, "focused"), + packageName: attrs.package || undefined, scrollHidden: 0, }; if (label) out.label = label; @@ -514,6 +519,8 @@ function computeNodeOutput( if (!c.label && label) c.label = label; const rid = attrs["resource-id"]; if (!c.identifier && rid) c.identifier = rid; + if (!c.packageName && attrs.package) c.packageName = attrs.package; + if (attrIsTrue(attrs, "focused")) c.focused = true; return [c]; } } @@ -601,7 +608,7 @@ function finalizeUiNode( // Keep it as a node whose frame is the union of its (here, sole) child so // neither the wrapper's nor the child's label is dropped. if (children.length === 0) return null; - if (children.length === 1 && !n.label && !n.identifier) return children[0]!; + if (children.length === 1 && !n.label && !n.identifier && !n.focused) return children[0]!; const x1 = Math.min(...children.map((c) => c.frame.x)); const y1 = Math.min(...children.map((c) => c.frame.y)); const x2 = Math.max(...children.map((c) => c.frame.x + c.frame.width)); @@ -623,6 +630,8 @@ function finalizeUiNode( if (n.checked) out.checked = true; if (n.disabled) out.disabled = true; if (n.password) out.password = true; + if (n.packageName) out.packageName = n.packageName; + if (n.focused) out.focused = true; if (n.scrollHidden > 0) out.scrollHidden = n.scrollHidden; return out; } diff --git a/packages/tool-server/src/tools/describe/platforms/ios/ios-ax-adapter.ts b/packages/tool-server/src/tools/describe/platforms/ios/ios-ax-adapter.ts index 8bdcac2ba..690b71901 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/ios-ax-adapter.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/ios-ax-adapter.ts @@ -20,7 +20,7 @@ export function adaptAXElement(el: AXDescribeElement): DescribeNode | null { const height = y2 - y1; if (width <= 0 || height <= 0) return null; - return { + const node: DescribeNode = { role: mapNativeTraitsToDescribeRole(el.traits ?? []), frame: { x: roundNormalized(x1), @@ -33,6 +33,8 @@ export function adaptAXElement(el: AXDescribeElement): DescribeNode | null { value: el.value, identifier: el.identifier, }; + if (el.focused) node.focused = true; + return node; } export function adaptAXDescribeToDescribeResult(response: AXDescribeResponse): DescribeNode { diff --git a/packages/tool-server/src/tools/describe/platforms/ios/ios-native-adapter.ts b/packages/tool-server/src/tools/describe/platforms/ios/ios-native-adapter.ts index 185288151..bcc630c89 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/ios-native-adapter.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/ios-native-adapter.ts @@ -51,7 +51,7 @@ export function adaptNativeDescribeElementToDescribeNode( const frame = clampNormalizedFrame(element.normalizedFrame); if (!frame) return null; - return { + const node: DescribeNode = { role: mapNativeTraitsToDescribeRole(element.traits), frame, children: [], @@ -59,6 +59,8 @@ export function adaptNativeDescribeElementToDescribeNode( identifier: element.identifier, value: element.value, }; + if (element.focused) node.focused = true; + return node; } export function adaptNativeDescribeToDescribeResult( diff --git a/packages/tool-server/src/tools/describe/platforms/tv.ts b/packages/tool-server/src/tools/describe/platforms/tv.ts index 7502cae74..64f8ccd73 100644 --- a/packages/tool-server/src/tools/describe/platforms/tv.ts +++ b/packages/tool-server/src/tools/describe/platforms/tv.ts @@ -1,6 +1,10 @@ import type { DeviceInfo, Registry } from "@argent/registry"; -import type { DescribeResult } from "../contract"; -import { formatDescribeTree } from "../format-tree"; +import type { DescribeNode, DescribeResult } from "../contract"; +import { + formatDescribeSelection, + formatDescribeTree, + type FormatDescribeSelectionOptions, +} from "../format-tree"; import { resolveTvApi } from "../../tv/tv-service"; import { describeAndroid } from "./android"; import type { @@ -112,7 +116,11 @@ function renderFocusView(res: TvDescribeResponse): string { * agent uses one `describe` for every target and never has to know up front * whether a UDID is a phone or an Apple TV. */ -export async function describeTv(registry: Registry, device: DeviceInfo): Promise { +export async function describeTv( + registry: Registry, + device: DeviceInfo, + compact?: Omit +): Promise { const api: TvControlApi = await resolveTvApi(registry, device.id); // Ride out a brief post-launch / transition window where the focus engine @@ -153,6 +161,14 @@ export async function describeTv(registry: Registry, device: DeviceInfo): Promis // overlay, or an adb-level failure), all strictly more useful than masking // it as the generic "app is probably still launching" EMPTY_HINT below. const data = await describeAndroid(registry, device.id, undefined, true); + if (compact) { + const formatted = formatDescribeSelection(data.tree, { source: data.source, ...compact }); + return { + source: data.source, + hint: ANDROID_FOCUS_EMPTY_HINT, + ...formatted, + }; + } return { description: `${ANDROID_FOCUS_EMPTY_HINT}\n\n${formatDescribeTree(data.tree, { source: data.source, @@ -163,6 +179,28 @@ export async function describeTv(registry: Registry, device: DeviceInfo): Promis } const empty = isEmpty(res); + if (compact) { + const elements = res.focusable.slice(); + if (res.focused && !elements.includes(res.focused)) elements.unshift(res.focused); + const tree: DescribeNode = { + role: "tv-root", + frame: { x: 0, y: 0, width: 1, height: 1 }, + children: elements.map((element) => ({ + role: element.traits?.[0] ?? "focusable", + label: element.label, + value: element.value, + focused: Boolean(element.isFocused || element === res.focused), + frame: { x: 0, y: 0, width: 0, height: 0 }, + children: [], + })), + }; + const formatted = formatDescribeSelection(tree, { source: "tv-focus", ...compact }); + return { + source: "tv-focus", + ...(empty ? { hint: EMPTY_HINT } : {}), + ...formatted, + }; + } const description = empty ? `${renderFocusView(res)}\n\nNote: ${EMPTY_HINT}` : renderFocusView(res); diff --git a/packages/tool-server/src/tools/describe/selectors.ts b/packages/tool-server/src/tools/describe/selectors.ts new file mode 100644 index 000000000..2c3f192ea --- /dev/null +++ b/packages/tool-server/src/tools/describe/selectors.ts @@ -0,0 +1,92 @@ +import { z } from "zod"; +import type { DescribeNode } from "./contract"; + +// Shared selector contract for tools that inspect the describe tree. Every +// supplied field is required to match; individual matches are +// case-insensitive substrings so callers do not need platform-exact labels. +export const describeSelectorSchema = z + .object({ + text: z + .string() + .min(1) + .optional() + .describe("Case-insensitive substring of the element's visible label or value."), + identifier: z + .string() + .min(1) + .optional() + .describe( + "Case-insensitive substring of the element's identifier (accessibilityIdentifier / resource-id / testid)." + ), + role: z + .string() + .min(1) + .optional() + .describe( + "Case-insensitive substring of the element's role (e.g. AXButton, button, TextView)." + ), + package: z + .string() + .min(1) + .optional() + .describe("Case-insensitive substring of the native package owning the element (Android)."), + }) + .refine( + (selector) => + Boolean(selector.text || selector.identifier || selector.role || selector.package), + { + message: "selector needs at least one of text, identifier, role, or package", + } + ); + +export type DescribeSelector = z.infer; + +function includesCaseInsensitive(haystack: string | undefined, needle: string): boolean { + return Boolean(haystack) && haystack!.toLowerCase().includes(needle.toLowerCase()); +} + +export function matchesDescribeSelector(node: DescribeNode, selector: DescribeSelector): boolean { + if ( + selector.text !== undefined && + !includesCaseInsensitive(node.label, selector.text) && + !includesCaseInsensitive(node.value, selector.text) + ) { + return false; + } + if ( + selector.identifier !== undefined && + !includesCaseInsensitive(node.identifier, selector.identifier) + ) { + return false; + } + if (selector.role !== undefined && !includesCaseInsensitive(node.role, selector.role)) { + return false; + } + if ( + selector.package !== undefined && + !includesCaseInsensitive(node.packageName, selector.package) + ) { + return false; + } + return true; +} + +function collectMatches( + node: DescribeNode, + selector: DescribeSelector, + matches: DescribeNode[] +): void { + if (matchesDescribeSelector(node, selector)) matches.push(node); + for (const child of node.children) collectMatches(child, selector, matches); +} + +// The root is a synthetic/non-selectable container in public describe output, +// so selectors consistently start at its children. +export function findDescribeMatches( + root: DescribeNode, + selector: DescribeSelector +): DescribeNode[] { + const matches: DescribeNode[] = []; + for (const child of root.children) collectMatches(child, selector, matches); + return matches; +} diff --git a/packages/tool-server/src/tools/keyboard-scenario/index.ts b/packages/tool-server/src/tools/keyboard-scenario/index.ts new file mode 100644 index 000000000..ebb395beb --- /dev/null +++ b/packages/tool-server/src/tools/keyboard-scenario/index.ts @@ -0,0 +1,722 @@ +import { z } from "zod"; +import type { Registry, ToolCapability, ToolContext, ToolDefinition } from "@argent/registry"; +import { resolveDevice } from "../../utils/device-info"; +import { invokeSubTool } from "../../utils/sub-invoke"; +import { settleWithin, sleepOrAbort } from "../../utils/timing"; +import type { DescribeFrame, DescribeNode } from "../describe/contract"; +import { + describeSelectorSchema, + matchesDescribeSelector, + type DescribeSelector, +} from "../describe/selectors"; + +const DEFAULT_KEY_DELAY_MS = 75; +const DEFAULT_KEYBOARD_TIMEOUT_MS = 5_000; +const DEFAULT_SETTLE_TIMEOUT_MS = 1_500; +const KEYBOARD_POLL_MS = 250; +const SETTLE_POLL_MS = 100; +const WRAP_HEIGHT_DELTA = 0.005; +const KEYBOARD_TOP_LIMIT = 0.45; +const QWERTY_LETTERS = "abcdefghijklmnopqrstuvwxyz"; +const ANDROID_IME_PACKAGES = [ + "com.google.android.inputmethod", + "com.android.inputmethod", + "com.samsung.android.honeyboard", + "com.touchtype.swiftkey", + "com.microsoft.swiftkey", + "org.futo.inputmethod", +] as const; + +const assertionSchema = z + .object({ + finalText: z + .string() + .optional() + .describe("Exact final input value. Defaults to the value before the scenario plus `text`."), + keyboardVisible: z + .boolean() + .optional() + .default(true) + .describe( + "Whether the visible soft keyboard must remain present after typing (default true)." + ), + inputFocused: z + .boolean() + .optional() + .describe( + "Optional focus assertion. Fails as unavailable when the platform accessibility provider does not expose focus." + ), + wrapped: z + .boolean() + .optional() + .describe( + "Optional line-wrap assertion, derived from whether the input frame height grew from its focused baseline." + ), + }) + .default({ keyboardVisible: true }); + +const zodSchema = z.object({ + udid: z + .string() + .min(1) + .describe("Target iOS simulator or Android emulator/device id from `list-devices`."), + input: describeSelectorSchema.describe( + "Selector for the text input. Every provided field matches as a case-insensitive substring." + ), + text: z + .string() + .min(1) + .describe( + "Text to enter by tapping the visible soft keyboard. Only keys visible on the current English QWERTY layout are used." + ), + bundleId: z + .string() + .optional() + .describe("Optional iOS app bundle id used by describe's native fallback."), + perKeyDelayMs: z + .number() + .int() + .min(0) + .max(2_000) + .optional() + .default(DEFAULT_KEY_DELAY_MS) + .describe(`Delay after each visible key tap (default ${DEFAULT_KEY_DELAY_MS}ms).`), + keyboardTimeoutMs: z + .number() + .int() + .positive() + .max(30_000) + .optional() + .default(DEFAULT_KEYBOARD_TIMEOUT_MS) + .describe( + `Time to wait for a visible English QWERTY key tree (default ${DEFAULT_KEYBOARD_TIMEOUT_MS}ms).` + ), + settleTimeoutMs: z + .number() + .int() + .positive() + .max(10_000) + .optional() + .default(DEFAULT_SETTLE_TIMEOUT_MS) + .describe( + `Time to poll for the expected input value after a word or final key (default ${DEFAULT_SETTLE_TIMEOUT_MS}ms).` + ), + assertions: assertionSchema, +}); + +type Params = z.infer; + +interface ScreenElement extends DescribeNode { + line: string; +} + +interface ScreenSnapshot { + source: string; + elements: ScreenElement[]; +} + +interface KeyboardLayout { + letters: Map; + space: ScreenElement; + shift?: ScreenElement; + direct: Map; +} + +type KeyboardPlatform = "ios" | "android"; + +interface Checkpoint { + charIndex: number; + value: string | null; + focused: boolean | null; + keyboardVisible: boolean; + inputFrame: DescribeFrame | null; + wrapped: boolean | null; +} + +interface AssertionResult { + expected: T; + actual: T | null; + passed: boolean; + available: boolean; +} + +interface ScenarioResult { + success: boolean; + typed: string; + checkpoints: Checkpoint[]; + assertions: { + finalText: AssertionResult; + keyboardVisible: AssertionResult; + inputFocused?: AssertionResult; + wrapped?: AssertionResult; + }; + error?: { stage: "input" | "keyboard" | "typing" | "inspection"; message: string }; +} + +type ScenarioStage = NonNullable["stage"]; + +const capability: ToolCapability = { + apple: { simulator: true }, + appleRemote: { simulator: true }, + android: { emulator: true, device: true, unknown: true }, +}; + +const FRAME_RE = /\(([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\)\s*$/; +const ATTR_RE = /\b(label|value|id|package)="((?:\\.|[^"\\])*)"/g; + +function unescapeLineValue(value: string): string { + return value + .replace(/\\n/g, "\n") + .replace(/\\r/g, "\r") + .replace(/\\t/g, "\t") + .replace(/\\"/g, '"') + .replace(/\\\\/g, "\\"); +} + +export function parseDescribeScreen(result: unknown): ScreenSnapshot { + const record = + typeof result === "object" && result !== null ? (result as Record) : {}; + const description = typeof record.description === "string" ? record.description : ""; + const source = typeof record.source === "string" ? record.source : "unknown"; + const elements: ScreenElement[] = []; + + for (const line of description.split("\n")) { + if (line.trimStart().startsWith("ROOT ")) continue; + const frameMatch = FRAME_RE.exec(line); + if (!frameMatch) continue; + const trimmed = line.trim(); + const role = trimmed.split(/\s+/, 1)[0]; + if (!role) continue; + const attrs: Record = {}; + for (const match of line.matchAll(ATTR_RE)) { + attrs[match[1]!] = unescapeLineValue(match[2]!); + } + // A label is the first quoted token and intentionally has no `label=` prefix + // in describe's compact rendering. + const labelMatch = /\s"((?:\\.|[^"\\])*)"/.exec(line); + const flagsMatch = /\[([^\]]+)]/.exec(line); + const flags = new Set( + (flagsMatch?.[1] ?? "") + .split(",") + .map((flag) => flag.trim()) + .filter(Boolean) + ); + elements.push({ + role, + frame: { + x: Number(frameMatch[1]), + y: Number(frameMatch[2]), + width: Number(frameMatch[3]), + height: Number(frameMatch[4]), + }, + children: [], + label: labelMatch ? unescapeLineValue(labelMatch[1]!) : attrs.label, + value: attrs.value, + identifier: attrs.id, + packageName: attrs.package, + clickable: flags.has("clickable") ? true : undefined, + focused: flags.has("focused") ? true : undefined, + line, + }); + } + + return { source, elements }; +} + +function center(frame: DescribeFrame): { x: number; y: number } { + return { x: frame.x + frame.width / 2, y: frame.y + frame.height / 2 }; +} + +function elementText(element: ScreenElement): string[] { + return [element.label, element.value, element.identifier] + .filter((value): value is string => Boolean(value)) + .map((value) => value.trim()); +} + +function isButtonLike(element: ScreenElement): boolean { + const role = element.role.toLowerCase(); + return role.includes("button") || role.includes("key"); +} + +function isKeyboardRegion(element: ScreenElement): boolean { + return ( + element.frame.y >= KEYBOARD_TOP_LIMIT && element.frame.width > 0 && element.frame.height > 0 + ); +} + +function isNativeAndroidIme(element: ScreenElement): boolean { + const packageName = element.packageName?.toLowerCase(); + return Boolean( + packageName && ANDROID_IME_PACKAGES.some((prefix) => packageName.startsWith(prefix)) + ); +} + +export function discoverEnglishQwerty( + elements: ScreenElement[], + platform: KeyboardPlatform = "ios" +): KeyboardLayout | null { + const imeCandidates = elements.filter( + (element) => + isNativeAndroidIme(element) && + isKeyboardRegion(element) && + (element.clickable || isButtonLike(element)) + ); + // Android requires native IME package provenance. iOS AX does not expose an + // owning bundle, so its fallback is a complete QWERTY + native-control + // signature rather than accepting an arbitrary group of app buttons. + const candidates = + platform === "android" + ? imeCandidates + : elements.filter( + (element) => !element.packageName && isButtonLike(element) && isKeyboardRegion(element) + ); + const letters = new Map(); + const direct = new Map(); + let space: ScreenElement | undefined; + let shift: ScreenElement | undefined; + let deleteKey: ScreenElement | undefined; + + for (const element of candidates) { + for (const text of elementText(element)) { + if (/^[A-Za-z]$/.test(text)) letters.set(text.toLowerCase(), element); + if (text.length === 1) direct.set(text, element); + const normalized = text.toLowerCase(); + if (normalized === "space" || normalized === "spacebar") space = element; + if (normalized === "shift" || normalized.includes("shift key")) shift = element; + if ( + normalized === "delete" || + normalized === "backspace" || + normalized.includes("delete key") + ) { + deleteKey = element; + } + } + } + + if ( + !space || + !shift || + !deleteKey || + [...QWERTY_LETTERS].some((letter) => !letters.has(letter)) + ) { + return null; + } + return { letters, space, shift, direct }; +} + +function usesUppercaseLabels(layout: KeyboardLayout): boolean { + let uppercase = 0; + let lowercase = 0; + for (const element of new Set(layout.letters.values())) { + const label = elementText(element).find((text) => /^[A-Za-z]$/.test(text)); + if (!label) continue; + if (label === label.toUpperCase()) uppercase += 1; + else lowercase += 1; + } + return uppercase > lowercase; +} + +function findInput( + snapshot: ScreenSnapshot, + selector: DescribeSelector +): ScreenElement | undefined { + return snapshot.elements.find((element) => matchesDescribeSelector(element, selector)); +} + +function inputValue(element: ScreenElement | undefined): string | null { + if (!element) return null; + return element.value ?? ""; +} + +function inputFocus(snapshot: ScreenSnapshot, input: ScreenElement | undefined): boolean | null { + if (!input) return null; + if (input.focused === true) return true; + // A focused peer proves that the provider exposed focus and this input is + // false. With no positive focus signal, preserve unknown as null. + return snapshot.elements.some((element) => element.focused === true) ? false : null; +} + +function didWrap(baseline: DescribeFrame | null, current: DescribeFrame | null): boolean | null { + if (!baseline || !current) return null; + return current.height > baseline.height + WRAP_HEIGHT_DELTA; +} + +async function readScreen( + registry: Registry, + ctx: ToolContext | undefined, + params: Pick +): Promise { + const result = await invokeSubTool(registry, ctx, "describe", { + udid: params.udid, + bundleId: params.bundleId, + }); + const snapshot = parseDescribeScreen(result); + if (resolveDevice(params.udid).platform !== "android") return snapshot; + + // The legacy full rendering intentionally omits Android package names. Ask + // compact describe for native IME-owned nodes and merge them into the app + // snapshot so Gboard's clickable `android.view.View` keys remain discoverable + // without changing selector-less describe output. + for (const packageMarker of ["inputmethod", "honeyboard", "swiftkey"] as const) { + const imeResult = await invokeSubTool(registry, ctx, "describe", { + udid: params.udid, + selector: { package: packageMarker }, + projection: "matches", + fields: ["role", "label", "value", "identifier", "package", "flags", "frame"], + limit: 200, + maxChars: 50_000, + }); + const imeSnapshot = parseDescribeScreen(imeResult); + const nativeImeElements = imeSnapshot.elements.filter(isNativeAndroidIme); + if (nativeImeElements.length > 0) { + return { source: snapshot.source, elements: [...snapshot.elements, ...nativeImeElements] }; + } + } + return snapshot; +} + +async function tapElement( + registry: Registry, + ctx: ToolContext | undefined, + udid: string, + element: ScreenElement +): Promise { + const point = center(element.frame); + await invokeSubTool(registry, ctx, "gesture-tap", { udid, ...point }); +} + +async function waitForKeyboard( + registry: Registry, + ctx: ToolContext | undefined, + params: Params +): Promise<{ snapshot: ScreenSnapshot; layout: KeyboardLayout } | null> { + const startedAt = Date.now(); + while (Date.now() - startedAt < params.keyboardTimeoutMs) { + if (ctx?.signal?.aborted) return null; + const remaining = params.keyboardTimeoutMs - (Date.now() - startedAt); + const settled = await settleWithin(readScreen(registry, ctx, params), remaining, ctx?.signal); + if (settled.type !== "value") return null; + const snapshot = settled.value; + const platform = resolveDevice(params.udid).platform === "android" ? "android" : "ios"; + const layout = discoverEnglishQwerty(snapshot.elements, platform); + if (layout) return { snapshot, layout }; + if (!(await sleepOrAbort(KEYBOARD_POLL_MS, ctx?.signal))) return null; + } + return null; +} + +interface SettledInspection { + snapshot: ScreenSnapshot; + input: ScreenElement | undefined; + layout: KeyboardLayout | null; + settled: boolean; + keyboardLost: boolean; +} + +async function waitForExpectedState( + registry: Registry, + ctx: ToolContext | undefined, + params: Params, + expectedValue: string, + expectedKeyboardVisible: boolean, + failImmediatelyWhenKeyboardLost: boolean +): Promise { + const startedAt = Date.now(); + let last: SettledInspection | null = null; + while (Date.now() - startedAt < params.settleTimeoutMs) { + const remaining = params.settleTimeoutMs - (Date.now() - startedAt); + const read = await settleWithin(readScreen(registry, ctx, params), remaining, ctx?.signal); + if (read.type !== "value") break; + const snapshot = read.value; + const input = findInput(snapshot, params.input); + const platform = resolveDevice(params.udid).platform === "android" ? "android" : "ios"; + const layout = discoverEnglishQwerty(snapshot.elements, platform); + const keyboardVisible = layout !== null; + last = { + snapshot, + input, + layout, + settled: inputValue(input) === expectedValue && keyboardVisible === expectedKeyboardVisible, + keyboardLost: !keyboardVisible, + }; + if (last.settled) return last; + if (failImmediatelyWhenKeyboardLost && last.keyboardLost) return last; + if (!(await sleepOrAbort(Math.min(SETTLE_POLL_MS, remaining), ctx?.signal))) break; + } + + return ( + last ?? { + snapshot: { source: "unknown", elements: [] }, + input: undefined, + layout: null, + settled: false, + keyboardLost: true, + } + ); +} + +function failedResult( + typed: string, + checkpoints: Checkpoint[], + stage: ScenarioStage, + message: string, + expectedFinalText: string +): ScenarioResult { + return { + success: false, + typed, + checkpoints, + assertions: { + finalText: { + expected: expectedFinalText, + actual: null, + passed: false, + available: false, + }, + keyboardVisible: { expected: true, actual: null, passed: false, available: false }, + }, + error: { stage, message }, + }; +} + +function assertion(expected: T, actual: T | null): AssertionResult { + return { + expected, + actual, + passed: actual !== null && actual === expected, + available: actual !== null, + }; +} + +function isCheckpoint(text: string, index: number): boolean { + return index === text.length - 1 || /\s/.test(text[index]!); +} + +export function createKeyboardScenarioTool( + registry: Registry +): ToolDefinition { + return { + id: "keyboard-scenario", + description: `Type an exact string by tapping the VISIBLE native soft keyboard and verify the input at word boundaries. + +This is intentionally different from \`keyboard\`: it never injects text or key events. It discovers the +English QWERTY soft-key tree exposed by iOS accessibility / Android uiautomator and taps each key's +on-screen centre, preserving predictive-text, autocorrect, focus, layout, and line-wrap behavior. + +The tool taps the selected input, waits for a visible QWERTY keyboard, types at a deterministic cadence, +and returns compact word-boundary checkpoints only. It always asserts the exact final value and keyboard +visibility; callers can also assert focus and whether the input wrapped. A missing QWERTY key, unavailable +focus signal, unexpected value, hidden keyboard, or wrap mismatch returns success=false with a stage. + +Use this for keyboard regressions that hardware-key injection cannot reproduce. Configure the device to a +normal English QWERTY keyboard first. The tool refuses to guess fixed keyboard coordinates or switch layouts.`, + alwaysLoad: true, + longRunning: true, + searchHint: + "visible native soft keyboard tap qwerty predictive autocorrect focus multiline wrap regression scenario", + zodSchema, + capability, + services: () => ({}), + async execute(_services, params, ctx) { + // Resolve up front so unsupported target shapes fail before any interaction. + resolveDevice(params.udid); + const checkpoints: Checkpoint[] = []; + const before = await readScreen(registry, ctx, params); + const inputBefore = findInput(before, params.input); + const initialValue = inputValue(inputBefore) ?? ""; + const expectedFinalText = params.assertions.finalText ?? `${initialValue}${params.text}`; + + if (!inputBefore) { + return failedResult( + "", + checkpoints, + "input", + "No element matched the input selector.", + expectedFinalText + ); + } + + await tapElement(registry, ctx, params.udid, inputBefore); + const keyboardReady = await waitForKeyboard(registry, ctx, params); + if (!keyboardReady) { + return failedResult( + "", + checkpoints, + "keyboard", + "A visible English QWERTY soft-key tree did not appear before timeout.", + expectedFinalText + ); + } + + const focusedInput = findInput(keyboardReady.snapshot, params.input); + const baselineFrame = focusedInput?.frame ?? inputBefore.frame; + let layout = keyboardReady.layout; + let uppercaseLayout = usesUppercaseLabels(layout); + let typed = ""; + + for (let index = 0; index < params.text.length; index++) { + if (ctx?.signal?.aborted) { + return failedResult( + typed, + checkpoints, + "typing", + "The scenario was cancelled.", + expectedFinalText + ); + } + const char = params.text[index]!; + let key: ScreenElement | undefined; + + if (char === " ") { + key = layout.space; + } else if (/^[A-Za-z]$/.test(char)) { + key = layout.letters.get(char.toLowerCase()); + const wantsUppercase = char === char.toUpperCase(); + const needsShift = wantsUppercase !== uppercaseLayout; + if (needsShift) { + if (!layout.shift) { + return failedResult( + typed, + checkpoints, + "keyboard", + `Visible shift key is required for ${JSON.stringify(char)} but was not found.`, + expectedFinalText + ); + } + await tapElement(registry, ctx, params.udid, layout.shift); + if (!(await sleepOrAbort(params.perKeyDelayMs, ctx?.signal))) { + return failedResult( + typed, + checkpoints, + "typing", + "The scenario was cancelled.", + expectedFinalText + ); + } + uppercaseLayout = !uppercaseLayout; + } + } else { + key = layout.direct.get(char); + } + + if (!key) { + return failedResult( + typed, + checkpoints, + "keyboard", + `No currently visible soft key matches ${JSON.stringify(char)}. The scenario does not guess coordinates or switch layouts.`, + expectedFinalText + ); + } + + await tapElement(registry, ctx, params.udid, key); + typed += char; + // Normal mobile shift is one-shot. Re-read the layout at the next word + // checkpoint, but keep mixed-case words correct without a tree fetch per key. + if (/^[A-Za-z]$/.test(char)) uppercaseLayout = false; + if (!(await sleepOrAbort(params.perKeyDelayMs, ctx?.signal))) { + return failedResult( + typed, + checkpoints, + "typing", + "The scenario was cancelled.", + expectedFinalText + ); + } + + if (isCheckpoint(params.text, index) && index !== params.text.length - 1) { + const inspection = await waitForExpectedState( + registry, + ctx, + params, + `${initialValue}${typed}`, + true, + true + ); + const { snapshot, input, layout: currentLayout } = inspection; + const currentFrame = input?.frame ?? null; + checkpoints.push({ + charIndex: index + 1, + value: inputValue(input), + focused: inputFocus(snapshot, input), + keyboardVisible: currentLayout !== null, + inputFrame: currentFrame, + wrapped: didWrap(baselineFrame, currentFrame), + }); + if (inspection.keyboardLost) { + return failedResult( + typed, + checkpoints, + "keyboard", + "The visible native keyboard disappeared at a word checkpoint; stopped before tapping stale coordinates.", + expectedFinalText + ); + } + if (!inspection.settled) { + return failedResult( + typed, + checkpoints, + "inspection", + `The input did not settle to ${JSON.stringify(`${initialValue}${typed}`)} before timeout.`, + expectedFinalText + ); + } + if (currentLayout) { + layout = currentLayout; + uppercaseLayout = usesUppercaseLabels(currentLayout); + } + } + } + + const finalInspection = await waitForExpectedState( + registry, + ctx, + params, + expectedFinalText, + params.assertions.keyboardVisible, + false + ); + const finalSnapshot = finalInspection.snapshot; + const finalInput = finalInspection.input; + const finalLayout = finalInspection.layout; + const finalText = inputValue(finalInput); + const keyboardVisible = finalLayout !== null; + const focused = inputFocus(finalSnapshot, finalInput); + const wrapped = didWrap(baselineFrame, finalInput?.frame ?? null); + checkpoints.push({ + charIndex: params.text.length, + value: finalText, + focused, + keyboardVisible, + inputFrame: finalInput?.frame ?? null, + wrapped, + }); + const assertions: ScenarioResult["assertions"] = { + finalText: assertion(expectedFinalText, finalText), + keyboardVisible: assertion(params.assertions.keyboardVisible, keyboardVisible), + }; + if (params.assertions.inputFocused !== undefined) { + assertions.inputFocused = assertion(params.assertions.inputFocused, focused); + } + if (params.assertions.wrapped !== undefined) { + assertions.wrapped = assertion(params.assertions.wrapped, wrapped); + } + const success = Object.values(assertions).every((result) => result.passed); + + return { + success, + typed, + checkpoints, + assertions, + ...(success + ? {} + : { + error: { + stage: "inspection" as const, + message: "One or more final keyboard scenario assertions failed.", + }, + }), + }; + }, + }; +} diff --git a/packages/tool-server/src/tools/native-devtools/native-describe-contract.ts b/packages/tool-server/src/tools/native-devtools/native-describe-contract.ts index 219889046..d682b8d71 100644 --- a/packages/tool-server/src/tools/native-devtools/native-describe-contract.ts +++ b/packages/tool-server/src/tools/native-devtools/native-describe-contract.ts @@ -24,6 +24,7 @@ export const nativeDescribeElementSchema = z value: z.string().optional(), identifier: z.string().optional(), viewClassName: z.string().optional(), + focused: z.boolean().optional(), }) .passthrough(); diff --git a/packages/tool-server/src/utils/setup-registry.ts b/packages/tool-server/src/utils/setup-registry.ts index 32a8253b9..7a48af3f5 100644 --- a/packages/tool-server/src/utils/setup-registry.ts +++ b/packages/tool-server/src/utils/setup-registry.ts @@ -37,6 +37,7 @@ import { createKeyboardTool } from "../tools/keyboard"; import { rotateTool } from "../tools/rotate"; import { createTvRemoteTool } from "../tools/tv-remote"; import { createRunSequenceTool } from "../tools/run-sequence"; +import { createKeyboardScenarioTool } from "../tools/keyboard-scenario"; import { debuggerConnectTool } from "../tools/debugger/debugger-connect"; import { debuggerStatusTool } from "../tools/debugger/debugger-status"; import { debuggerEvaluateTool } from "../tools/debugger/debugger-evaluate"; @@ -128,6 +129,7 @@ export function createRegistry(): Registry { registry.registerTool(rotateTool); registry.registerTool(createTvRemoteTool(registry)); registry.registerTool(createRunSequenceTool(registry)); + registry.registerTool(createKeyboardScenarioTool(registry)); registry.registerTool(debuggerConnectTool); registry.registerTool(debuggerStatusTool); registry.registerTool(debuggerEvaluateTool); diff --git a/packages/tool-server/test/describe-ax-adapter.test.ts b/packages/tool-server/test/describe-ax-adapter.test.ts index a3e35d760..430cc2455 100644 --- a/packages/tool-server/test/describe-ax-adapter.test.ts +++ b/packages/tool-server/test/describe-ax-adapter.test.ts @@ -81,6 +81,17 @@ describe("describe ax-service adapter", () => { expect(node).toBeNull(); }); + it("propagates focused state from the AX service", () => { + const node = adaptAXElement({ + label: "Email", + frame: { x: 0.05, y: 0.1, width: 0.9, height: 0.04 }, + traits: ["searchField"], + focused: true, + }); + + expect(node?.focused).toBe(true); + }); + it("preserves label and value", () => { const node = adaptAXElement({ label: "Search", diff --git a/packages/tool-server/test/describe-compact.test.ts b/packages/tool-server/test/describe-compact.test.ts new file mode 100644 index 000000000..1f81e8c32 --- /dev/null +++ b/packages/tool-server/test/describe-compact.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; +import { Registry } from "@argent/registry"; +import type { DescribeNode } from "../src/tools/describe/contract"; +import { + DESCRIBE_FIELDS, + formatDescribeSelection, + formatDescribeTree, +} from "../src/tools/describe/format-tree"; +import { createDescribeTool } from "../src/tools/describe"; +import { + describeSelectorSchema, + findDescribeMatches, + matchesDescribeSelector, +} from "../src/tools/describe/selectors"; + +const frame = { x: 0, y: 0, width: 1, height: 1 }; + +function node( + role: string, + options: Partial> = {}, + children: DescribeNode[] = [] +): DescribeNode { + return { role, frame, children, ...options }; +} + +const tree = node("hierarchy", {}, [ + node("Panel", { identifier: "profile-panel" }, [ + node("TextView", { label: "Account" }), + node("Button", { label: "Save profile", identifier: "save-button", clickable: true }), + ]), + node("Panel", {}, [node("Button", { label: "Cancel", identifier: "cancel-button" })]), +]); + +describe("shared describe selector", () => { + it("uses AND semantics with case-insensitive substring matching", () => { + const selector = describeSelectorSchema.parse({ + text: "SAVE", + identifier: "save-", + role: "button", + }); + expect(matchesDescribeSelector(tree.children[0]!.children[1]!, selector)).toBe(true); + expect(matchesDescribeSelector(tree.children[1]!.children[0]!, selector)).toBe(false); + }); + + it("finds matching descendants but never the synthetic root", () => { + expect(findDescribeMatches(tree, { role: "hierarchy" })).toEqual([]); + expect(findDescribeMatches(tree, { role: "button" })).toHaveLength(2); + }); +}); + +describe("compact describe rendering", () => { + it("renders matches as flat lines and defaults can include every field", () => { + const result = formatDescribeSelection(tree, { + source: "uiautomator", + selector: { role: "button" }, + projection: "matches", + fields: DESCRIBE_FIELDS, + limit: 50, + maxChars: 12_000, + }); + + expect(result).toMatchObject({ matched: 2, emitted: 2, truncated: false }); + expect(result.description).toContain( + 'Button "Save profile" id="save-button" [clickable] [match]' + ); + expect(result.description).toContain('Button "Cancel" id="cancel-button" [match]'); + expect(result.description).not.toContain("Panel"); + expect(result.description).not.toMatch(/^ {2}Button/m); + }); + + it("keeps only paths to matches for matches-and-ancestors", () => { + const result = formatDescribeSelection(tree, { + source: "uiautomator", + selector: { text: "Save" }, + projection: "matches-and-ancestors", + fields: ["role", "label"], + limit: 50, + maxChars: 12_000, + }); + + expect(result.description).toContain(" Panel"); + expect(result.description).toContain(' Button "Save profile" [match]'); + expect(result.description).not.toContain("Account"); + expect(result.description).not.toContain("Cancel"); + }); + + it("renders the full projection with matches highlighted and requested fields only", () => { + const result = formatDescribeSelection(tree, { + source: "uiautomator", + selector: { identifier: "save-button" }, + projection: "full", + fields: ["label"], + limit: 50, + maxChars: 12_000, + }); + + expect(result.matched).toBe(1); + expect(result.description).toContain('"Save profile" [match]'); + expect(result.description).toContain('"Cancel"'); + expect(result.description).not.toContain("(0.000"); + expect(result.description).not.toContain("id="); + }); + + it("reports limit and character truncation explicitly", () => { + const byLimit = formatDescribeSelection(tree, { + source: "uiautomator", + selector: { role: "button" }, + projection: "matches", + fields: DESCRIBE_FIELDS, + limit: 1, + maxChars: 12_000, + }); + expect(byLimit).toMatchObject({ matched: 2, emitted: 1, truncated: true }); + expect(byLimit.description).toContain("… truncated"); + + const byChars = formatDescribeSelection(tree, { + source: "uiautomator", + selector: { role: "button" }, + projection: "full", + fields: DESCRIBE_FIELDS, + limit: 50, + maxChars: 256, + }); + expect(byChars.truncated).toBe(true); + expect(byChars.description.length).toBeLessThanOrEqual(256); + expect(byChars.description).toContain("… truncated"); + }); +}); + +describe("describe compact schema compatibility", () => { + const schema = createDescribeTool(new Registry()).zodSchema!; + const udid = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; + + it("keeps the selector-less request valid and the legacy formatter unchanged", () => { + expect(schema.safeParse({ udid }).success).toBe(true); + const before = formatDescribeTree(tree, { source: "uiautomator" }); + const after = formatDescribeTree(tree, { source: "uiautomator" }); + expect(after).toBe(before); + }); + + it("accepts compact controls with a selector and validates their bounds", () => { + expect( + schema.safeParse({ + udid, + selector: { text: "Save" }, + projection: "matches-and-ancestors", + fields: ["role", "frame"], + limit: 500, + maxChars: 100_000, + }).success + ).toBe(true); + expect(schema.safeParse({ udid, selector: {}, limit: 1 }).success).toBe(false); + expect(schema.safeParse({ udid, selector: { text: "Save" }, limit: 0 }).success).toBe(false); + expect(schema.safeParse({ udid, selector: { text: "Save" }, maxChars: 255 }).success).toBe( + false + ); + expect(schema.safeParse({ udid, limit: 1 }).success).toBe(false); + }); +}); diff --git a/packages/tool-server/test/describe-native-adapter.test.ts b/packages/tool-server/test/describe-native-adapter.test.ts index 9d3e53c0c..b4c42fabe 100644 --- a/packages/tool-server/test/describe-native-adapter.test.ts +++ b/packages/tool-server/test/describe-native-adapter.test.ts @@ -36,6 +36,19 @@ describe("describe native adapter", () => { }); }); + it("propagates focused state from native describe elements", () => { + const node = adaptNativeDescribeElementToDescribeNode({ + frame: { x: 16, y: 80, width: 100, height: 44 }, + tapPoint: { x: 66, y: 102 }, + normalizedFrame: { x: 0.1, y: 0.2, width: 0.3, height: 0.1 }, + normalizedTapPoint: { x: 0.25, y: 0.25 }, + traits: ["searchField"], + focused: true, + }); + + expect(node?.focused).toBe(true); + }); + it("clamps partially off-screen normalized frames into the public [0,1] contract", () => { const node = adaptNativeDescribeElementToDescribeNode({ frame: { x: -10, y: 800, width: 420, height: 100 }, diff --git a/packages/tool-server/test/describe-tool.test.ts b/packages/tool-server/test/describe-tool.test.ts index 7f7768240..2c5ec2aa9 100644 --- a/packages/tool-server/test/describe-tool.test.ts +++ b/packages/tool-server/test/describe-tool.test.ts @@ -122,6 +122,9 @@ describe("describe tool", () => { expect(result.source).toBe("ax-service"); expect(result.description).toContain("ROOT AXGroup"); expect(result.description).toMatch(/AXButton\s+"General"/); + expect(result.matched).toBeUndefined(); + expect(result.emitted).toBeUndefined(); + expect(result.truncated).toBeUndefined(); }); it("returns dialog elements when alertVisible is true", async () => { diff --git a/packages/tool-server/test/keyboard-scenario.test.ts b/packages/tool-server/test/keyboard-scenario.test.ts new file mode 100644 index 000000000..6695c98f8 --- /dev/null +++ b/packages/tool-server/test/keyboard-scenario.test.ts @@ -0,0 +1,425 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Registry } from "@argent/registry"; +import { createRegistry } from "../src/utils/setup-registry"; +import { parseUiAutomatorDump } from "../src/tools/describe/platforms/android/uiautomator-parser"; +import { findDescribeMatches } from "../src/tools/describe/selectors"; +import { + createKeyboardScenarioTool, + discoverEnglishQwerty, + parseDescribeScreen, +} from "../src/tools/keyboard-scenario"; + +const IOS = "11111111-1111-1111-1111-111111111111"; +const ANDROID = "emulator-5554"; + +function inputLine(value: string, options: { focused?: boolean; height?: number } = {}): string { + const flags = options.focused ? " [focused]" : ""; + return ` AXTextField "Message" value="${value}" id="chat-input"${flags} (0.100, 0.400, 0.800, ${(options.height ?? 0.04).toFixed(3)})`; +} + +const KEY_LABELS = [..."abcdefghijklmnopqrstuvwxyz"]; + +function keyboardLines( + options: { + shift?: boolean; + omit?: string; + uppercase?: boolean; + packageName?: string; + role?: string; + } = {} +): string[] { + const lines = KEY_LABELS.filter((label) => label !== options.omit).map((label, index) => { + const visibleLabel = options.uppercase ? label.toUpperCase() : label; + const packagePart = options.packageName ? ` package="${options.packageName}"` : ""; + const flags = options.packageName ? " [clickable]" : ""; + return ` ${options.role ?? "AXButton"} "${visibleLabel}"${packagePart}${flags} (${(0.02 + (index % 10) * 0.095).toFixed(3)}, ${(0.59 + Math.floor(index / 10) * 0.08).toFixed(3)}, 0.070, 0.060)`; + }); + const packagePart = options.packageName ? ` package="${options.packageName}"` : ""; + const flags = options.packageName ? " [clickable]" : ""; + const role = options.role ?? "AXButton"; + lines.push(` ${role} "space"${packagePart}${flags} (0.250, 0.850, 0.500, 0.070)`); + lines.push(` ${role} "delete"${packagePart}${flags} (0.870, 0.760, 0.100, 0.070)`); + if (options.shift !== false) { + lines.push(` ${role} "shift"${packagePart}${flags} (0.030, 0.760, 0.100, 0.070)`); + } + return lines; +} + +function screen( + value: string, + options: { + keyboard?: boolean; + focused?: boolean; + height?: number; + shift?: boolean; + uppercase?: boolean; + } = {} +): { source: string; description: string } { + return { + source: "ax-service", + description: [ + "Source: ax-service", + "Mode: flat", + "", + "ROOT AXGroup (0.000, 0.000, 1.000, 1.000)", + "", + inputLine(value, options), + ...(options.keyboard === false + ? [] + : keyboardLines({ shift: options.shift, uppercase: options.uppercase })), + ].join("\n"), + }; +} + +function mockRegistry(describes: Array<{ source: string; description: string }>): Registry { + let describeIndex = 0; + return { + invokeTool: vi.fn(async (id: string) => { + if (id === "describe") { + const snapshot = describes[Math.min(describeIndex, describes.length - 1)]; + describeIndex += 1; + return snapshot; + } + if (id === "gesture-tap") return { tapped: true }; + throw new Error(`Unexpected tool ${id}`); + }), + } as unknown as Registry; +} + +function params( + text: string, + assertions: { + finalText?: string; + keyboardVisible?: boolean; + inputFocused?: boolean; + wrapped?: boolean; + } = { keyboardVisible: true } +) { + return { + udid: IOS, + input: { identifier: "chat-input" }, + text, + perKeyDelayMs: 0, + keyboardTimeoutMs: 10, + settleTimeoutMs: 250, + assertions: { keyboardVisible: true, ...assertions }, + }; +} + +describe("keyboard-scenario", () => { + it("parses compact describe lines without surfacing the full tree", () => { + const snapshot = parseDescribeScreen(screen("hello", { focused: true })); + const input = snapshot.elements.find((element) => element.identifier === "chat-input"); + + expect(snapshot.source).toBe("ax-service"); + expect(input).toMatchObject({ + role: "AXTextField", + label: "Message", + value: "hello", + identifier: "chat-input", + focused: true, + frame: { x: 0.1, y: 0.4, width: 0.8, height: 0.04 }, + }); + }); + + it("accepts a visible English QWERTY tree and rejects app buttons alone", () => { + const keyboard = parseDescribeScreen(screen("", { keyboard: true })); + const appOnly = parseDescribeScreen(screen("", { keyboard: false })); + + expect(discoverEnglishQwerty(keyboard.elements)?.letters.size).toBe(26); + expect(discoverEnglishQwerty(keyboard.elements)?.space.label).toBe("space"); + expect(discoverEnglishQwerty(appOnly.elements)).toBeNull(); + }); + + it("rejects an app-owned custom keypad that only matches the old loose signature", () => { + const customLines = keyboardLines().filter((line) => + /"[a-j]"|"space"|"shift"|"delete"/.test(line) + ); + const custom = parseDescribeScreen({ + source: "ax-service", + description: [inputLine(""), ...customLines].join("\n"), + }); + + expect(discoverEnglishQwerty(custom.elements)).toBeNull(); + }); + + it("accepts clickable Android View keys only with native IME package provenance", () => { + const packageName = "com.google.android.inputmethod.latin"; + const labels = [...KEY_LABELS, "space", "shift", "delete"]; + const nodes = labels + .map((label, index) => { + const column = index % 10; + const row = Math.floor(index / 10); + const left = 20 + column * 100; + const top = 1400 + row * 180; + return ``; + }) + .join(""); + const tree = parseUiAutomatorDump( + `${nodes}`, + 1080, + 2400 + ); + const imeNodes = findDescribeMatches(tree, { package: "inputmethod" }).map((node) => ({ + ...node, + line: "", + })); + + expect(imeNodes).toHaveLength(labels.length); + expect(imeNodes.every((node) => node.role === "View" && node.clickable)).toBe(true); + expect(discoverEnglishQwerty(imeNodes, "android")?.letters.size).toBe(26); + }); + + it("rejects a package-less legacy Android app QWERTY end to end", async () => { + let fullReads = 0; + const registry = { + invokeTool: vi.fn(async (id: string, args: Record) => { + if (id === "gesture-tap") return { tapped: true }; + if (id !== "describe") throw new Error(`Unexpected tool ${id}`); + if (args.selector) return { source: "uiautomator", description: "" }; + fullReads += 1; + const snapshot = + fullReads === 1 + ? screen("", { keyboard: false }) + : screen("", { focused: true, keyboard: true }); + return { ...snapshot, source: "uiautomator" }; + }), + } as unknown as Registry; + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, { ...params("h"), udid: ANDROID, keyboardTimeoutMs: 10 }); + + expect(result).toMatchObject({ success: false, error: { stage: "keyboard" } }); + expect( + (registry.invokeTool as ReturnType).mock.calls.filter( + ([id]) => id === "gesture-tap" + ) + ).toHaveLength(1); + }); + + it("taps visible keys, records word checkpoints, and verifies focus and wrapping", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true }), + screen("hi ", { focused: true }), + screen("hi go", { focused: true, height: 0.08 }), + screen("hi go", { focused: true, height: 0.08 }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("hi go", { inputFocused: true, wrapped: true })); + + expect(result.success).toBe(true); + expect(result.typed).toBe("hi go"); + expect(result.checkpoints).toEqual([ + expect.objectContaining({ charIndex: 3, value: "hi ", keyboardVisible: true }), + expect.objectContaining({ + charIndex: 5, + value: "hi go", + focused: true, + keyboardVisible: true, + wrapped: true, + }), + ]); + expect(result.assertions).toMatchObject({ + finalText: { expected: "hi go", actual: "hi go", passed: true }, + keyboardVisible: { expected: true, actual: true, passed: true }, + inputFocused: { expected: true, actual: true, passed: true, available: true }, + wrapped: { expected: true, actual: true, passed: true, available: true }, + }); + + const calls = (registry.invokeTool as ReturnType).mock.calls; + expect(calls.filter(([id]) => id === "gesture-tap")).toHaveLength(6); + expect(calls.some(([id]) => id === "keyboard")).toBe(false); + }); + + it("stops immediately when the keyboard disappears at a word checkpoint", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true }), + screen("hi ", { keyboard: false, focused: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("hi x")); + + expect(result).toMatchObject({ + success: false, + typed: "hi ", + error: { stage: "keyboard", message: expect.stringContaining("stale coordinates") }, + }); + const taps = (registry.invokeTool as ReturnType).mock.calls.filter( + ([id]) => id === "gesture-tap" + ); + // Input + h + i + space. The trailing x is never tapped. + expect(taps).toHaveLength(4); + }); + + it("polls through delayed space/autocorrect state before continuing", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true }), + screen("hi", { focused: true }), + screen("hi ", { focused: true }), + screen("hi ", { focused: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("hi ")); + + expect(result.success).toBe(true); + expect(result.checkpoints).toEqual([ + expect.objectContaining({ charIndex: 3, value: "hi ", keyboardVisible: true }), + ]); + expect( + (registry.invokeTool as ReturnType).mock.calls.filter( + ([id]) => id === "describe" + ).length + ).toBeGreaterThanOrEqual(4); + }); + + it("uses the visible shift key for an uppercase character", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true, shift: true }), + screen("H", { focused: true, shift: true }), + screen("H", { focused: true, shift: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("H")); + + expect(result.success).toBe(true); + const taps = (registry.invokeTool as ReturnType).mock.calls.filter( + ([id]) => id === "gesture-tap" + ); + // Input, shift, H. + expect(taps).toHaveLength(3); + }); + + it("does not re-toggle one-shot shift after an uppercase first character", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true, shift: true, uppercase: true }), + screen("Hi", { focused: true, shift: true }), + screen("Hi", { focused: true, shift: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("Hi")); + + expect(result.success).toBe(true); + const taps = (registry.invokeTool as ReturnType).mock.calls.filter( + ([id]) => id === "gesture-tap" + ); + // Input, H, i — no shift tap, because the initial visible keyboard was + // already uppercase and native shift becomes lowercase after H. + expect(taps).toHaveLength(3); + }); + + it("reports focus as unavailable instead of guessing from keyboard visibility", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen(""), + screen("h"), + screen("h"), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("h", { inputFocused: true })); + + expect(result.success).toBe(false); + expect(result.assertions.inputFocused).toEqual({ + expected: true, + actual: null, + passed: false, + available: false, + }); + expect(result.error?.stage).toBe("inspection"); + }); + + it("fails closed when the requested character is not a visible soft key", async () => { + const registry = mockRegistry([screen("", { keyboard: false }), screen("", { focused: true })]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("!")); + + expect(result).toMatchObject({ + success: false, + typed: "", + error: { + stage: "keyboard", + message: expect.stringContaining("No currently visible soft key"), + }, + }); + expect( + (registry.invokeTool as ReturnType).mock.calls.some(([id]) => id === "keyboard") + ).toBe(false); + }); + + it("surfaces predictive-text changes as an exact final-value failure", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true }), + screen("gone", { focused: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("go")); + + expect(result.success).toBe(false); + expect(result.assertions.finalText).toMatchObject({ + expected: "go", + actual: "gone", + passed: false, + }); + expect(result.error?.stage).toBe("inspection"); + }); + + it("uses approved final autocorrection instead of raw typed text at the final checkpoint", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true }), + screen("gone", { focused: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("go", { finalText: "gone" })); + + expect(result.success).toBe(true); + expect(result.checkpoints).toEqual([ + expect.objectContaining({ charIndex: 2, value: "gone", keyboardVisible: true }), + ]); + }); + + it("lets final inspection own an expected keyboard dismissal", async () => { + const registry = mockRegistry([ + screen("", { keyboard: false }), + screen("", { focused: true }), + screen("h", { keyboard: false, focused: true }), + ]); + const tool = createKeyboardScenarioTool(registry); + + const result = await tool.execute({}, params("h", { keyboardVisible: false })); + + expect(result.success).toBe(true); + expect(result.assertions.keyboardVisible).toMatchObject({ + expected: false, + actual: false, + passed: true, + }); + expect(result.checkpoints).toEqual([ + expect.objectContaining({ charIndex: 1, value: "h", keyboardVisible: false }), + ]); + }); + + it("is registered with a compact generated schema", () => { + const tool = createRegistry().getTool("keyboard-scenario"); + + expect(tool).toBeDefined(); + expect(tool?.inputSchema).toMatchObject({ + type: "object", + required: expect.arrayContaining(["udid", "input", "text"]), + }); + }); +}); diff --git a/packages/tool-server/test/native-describe-contract.test.ts b/packages/tool-server/test/native-describe-contract.test.ts index 3c1a92014..dfa829126 100644 --- a/packages/tool-server/test/native-describe-contract.test.ts +++ b/packages/tool-server/test/native-describe-contract.test.ts @@ -38,6 +38,7 @@ describe("native describe screen contract", () => { traits: ["header"], label: "Settings", viewClassName: "UILabel", + focused: true, }, ], }); @@ -46,6 +47,7 @@ describe("native describe screen contract", () => { expect(result.elements).toHaveLength(1); expect(result.elements[0]?.normalizedFrame.x).toBeCloseTo(0.041025641, 6); expect(result.elements[0]?.normalizedTapPoint.y).toBeCloseTo(125 / 844, 6); + expect(result.elements[0]?.focused).toBe(true); }); it("requires screenFrame metadata", () => { diff --git a/packages/tool-server/test/uiautomator-parser-v2-trim.test.ts b/packages/tool-server/test/uiautomator-parser-v2-trim.test.ts index 4df357a28..a67c70e59 100644 --- a/packages/tool-server/test/uiautomator-parser-v2-trim.test.ts +++ b/packages/tool-server/test/uiautomator-parser-v2-trim.test.ts @@ -59,6 +59,28 @@ describe("parseUiAutomatorDump — v2 trim focused behaviour", () => { expect(all[0]?.role).toBe("Button"); }); + it("surfaces the focused uiautomator node", () => { + const xml = ` + + +`; + const tree = parseUiAutomatorDump(xml, 100, 40); + const field = flatten(tree).find((n) => n.role === "TextField"); + expect(field?.focused).toBe(true); + }); + + it("preserves outer focused state when collapsing duplicate clickable wrappers", () => { + const xml = ` + + + + +`; + const tree = parseUiAutomatorDump(xml, 100, 100); + const button = flatten(tree).find((n) => n.role === "Button"); + expect(button?.focused).toBe(true); + }); + it("redacts the value of password fields", () => { const xml = `