feat: add native keyboard scenario and compact describe

packages/tool-server/src/blueprints/ax-service.ts

@@ -57,6 +57,7 @@ export interface AXDescribeElement {
   traits?: string[];
   value?: string;
   identifier?: string;
+  focused?: boolean;
 }
 
 export interface AXDescribeResponse {

packages/tool-server/src/tools/await-ui-element/index.ts

@@ -14,6 +14,11 @@ import { assertSupported } from "../../utils/capability";
 import { ensureDeps } from "../../utils/check-deps";
 import { pollDescribeTree } from "../../utils/poll-describe-tree";
 import type { DescribeNode, DescribeTreeData } from "../describe/contract";
+import {
+  describeSelectorSchema,
+  findDescribeMatches,
+  type DescribeSelector,
+} from "../describe/selectors";
 import { describeIos, iosRequires } from "../describe/platforms/ios";
 import { describeAndroid, androidRequires } from "../describe/platforms/android";
 import { describeChromium } from "../describe/platforms/chromium";
@@ -42,36 +47,8 @@ export function isUnmetUiWaitResult(tool: string, result: unknown): boolean {
 const DEFAULT_TIMEOUT_MS = 5000;
 const DEFAULT_POLL_INTERVAL_MS = 400;
 
-// A selector locates a node in the accessibility / DOM tree returned by
-// `describe`. Every provided field must match (logical AND); matching is a
-// case-insensitive substring test so the agent doesn't need the exact label.
-const selectorSchema = z
-  .object({
-    text: z
-      .string()
-      .min(1)
-      .optional()
-      .describe("Case-insensitive substring of the element's visible label or value."),
-    identifier: z
-      .string()
-      .min(1)
-      .optional()
-      .describe(
-        "Case-insensitive substring of the element's identifier (accessibilityIdentifier / resource-id / testid)."
-      ),
-    role: z
-      .string()
-      .min(1)
-      .optional()
-      .describe(
-        "Case-insensitive substring of the element's role (e.g. AXButton, button, TextView)."
-      ),
-  })
-  .refine((s) => Boolean(s.text || s.identifier || s.role), {
-    message: "selector needs at least one of text, identifier, or role",
-  });
-
-type Selector = z.infer<typeof selectorSchema>;
+// Selector matching is shared with `describe` so both tools interpret the same
+// selector identically.
 
 const zodSchema = z
   .object({
@@ -87,7 +64,9 @@ const zodSchema = z
           "or zero-area. `text`: the first match in reading order (topmost) contains expectedText — if a loose " +
           "selector hits several elements, only that topmost one is checked, so narrow it to target the intended element."
       ),
-    selector: selectorSchema.describe("Element to match (text / identifier / role)."),
+    selector: describeSelectorSchema.describe(
+      "Element to match (text / identifier / role / Android package)."
+    ),
     expectedText: z
       .string()
       .min(1)
@@ -143,30 +122,6 @@ function nodeText(node: DescribeNode): string {
   return [node.label, node.value].filter(Boolean).join(" ");
 }
 
-function includesCI(haystack: string | undefined, needle: string): boolean {
-  return Boolean(haystack) && haystack!.toLowerCase().includes(needle.toLowerCase());
-}
-
-function matchNode(node: DescribeNode, selector: Selector): boolean {
-  if (selector.text !== undefined) {
-    if (!includesCI(node.label, selector.text) && !includesCI(node.value, selector.text)) {
-      return false;
-    }
-  }
-  if (selector.identifier !== undefined && !includesCI(node.identifier, selector.identifier)) {
-    return false;
-  }
-  if (selector.role !== undefined && !includesCI(node.role, selector.role)) {
-    return false;
-  }
-  return true;
-}
-
-function collectMatches(node: DescribeNode, selector: Selector, acc: DescribeNode[]): void {
-  if (matchNode(node, selector)) acc.push(node);
-  for (const child of node.children) collectMatches(child, selector, acc);
-}
-
 // Every node matching the selector in the subtree, EXCLUDING `root` itself.
 //
 // `root` is the top-level container describe puts at the head of the tree. On
@@ -192,10 +147,8 @@ function collectMatches(node: DescribeNode, selector: Selector, acc: DescribeNod
 // `visible`/`exists` selector is broad. A substring selector can also match
 // several real nodes, so conditions are evaluated across the whole set (see
 // evaluateMatches). Exported for unit tests.
-export function findAll(root: DescribeNode, selector: Selector): DescribeNode[] {
-  const acc: DescribeNode[] = [];
-  for (const child of root.children) collectMatches(child, selector, acc);
-  return acc;
+export function findAll(root: DescribeNode, selector: DescribeSelector): DescribeNode[] {
+  return findDescribeMatches(root, selector);
 }
 
 // describe prunes off-screen / zero-size nodes on Chromium and the compressed
@@ -242,7 +195,10 @@ export function evaluateMatches(params: Params, matches: DescribeNode[]): boolea
       return !matches.some(isVisible);
     case "text": {
       const first = firstInReadingOrder(matches);
-      return first !== undefined && includesCI(nodeText(first), params.expectedText!);
+      return (
+        first !== undefined &&
+        nodeText(first).toLowerCase().includes(params.expectedText!.toLowerCase())
+      );
     }
     default:
       return false;
@@ -352,7 +308,7 @@ Conditions:
              substring). A loose selector can match several elements; only that topmost one is inspected, so if a
              lower match is the one holding the text the wait still reports failure — narrow the selector to target it.
 
-The selector is { text?, identifier?, role? }; every provided field must match (case-insensitive substring).
+The selector is { text?, identifier?, role?, package? }; every provided field must match (case-insensitive substring).
 text matches the element's label or value. It polls the same accessibility / DOM tree as \`describe\`
 (iOS AXRuntime, Android uiautomator, Chromium CDP) every pollIntervalMs (default ${DEFAULT_POLL_INTERVAL_MS}ms)
 until timeoutMs (default ${DEFAULT_TIMEOUT_MS}ms).

packages/tool-server/src/tools/describe/contract.ts

@@ -16,6 +16,7 @@ export interface DescribeNode {
   label?: string;
   identifier?: string;
   value?: string;
+  packageName?: string;
   // Interactivity flags surfaced by the Android uiautomator dump. iOS
   // consumers leave these unset; adding them as optional avoids breaking
   // existing payloads. `scrollHidden` counts children that fell outside an
@@ -32,7 +33,8 @@ export interface DescribeNode {
   // `focused` is the element holding input focus; `selected` is the visually
   // highlighted / active item (e.g. the current nav tab). On Vega the toolkit
   // often reports the highlighted item via `selected` while `focused` stays
-  // false, so both are surfaced. Other platforms leave these unset.
+  // false, so both are surfaced. iOS and Android also surface `focused` when
+  // their native accessibility providers report an element holding input focus.
   focused?: boolean;
   selected?: boolean;
 }
@@ -46,6 +48,7 @@ export const describeNodeSchema: z.ZodType<DescribeNode> = z.lazy(() =>
       label: z.string().optional(),
       identifier: z.string().optional(),
       value: z.string().optional(),
+      packageName: z.string().optional(),
       clickable: z.boolean().optional(),
       longClickable: z.boolean().optional(),
       scrollable: z.boolean().optional(),
@@ -98,6 +101,10 @@ export interface DescribeResult {
   source: DescribeSource;
   should_restart?: boolean;
   hint?: string;
+  // Present only for selector-driven compact describe calls.
+  matched?: number;
+  emitted?: number;
+  truncated?: boolean;
 }
 
 export function parseDescribeResult(input: unknown): DescribeNode {

packages/tool-server/src/tools/describe/format-tree.ts

@@ -1,4 +1,5 @@
 import type { DescribeFrame, DescribeNode, DescribeSource } from "./contract";
+import { findDescribeMatches, type DescribeSelector } from "./selectors";
 
 // Token-efficient view of a pruned DescribeNode tree. The previous shape sent
 // the full JSON tree to the agent, which on a typical iOS screen ran ~6× the
@@ -220,3 +221,203 @@ export function formatDescribeTree(root: DescribeNode, opts: FormatDescribeOptio
   const body = mode === "flat" ? renderFlat(root, contentRoles) : renderNested(root, contentRoles);
   return [...header, ...body].join("\n").replace(/\n+$/, "\n");
 }
+
+export const DESCRIBE_FIELDS = [
+  "role",
+  "label",
+  "value",
+  "identifier",
+  "package",
+  "flags",
+  "frame",
+] as const;
+export type DescribeField = (typeof DESCRIBE_FIELDS)[number];
+export type DescribeProjection = "matches" | "matches-and-ancestors" | "full";
+
+export interface FormatDescribeSelectionOptions {
+  source: DescribeSource;
+  selector: DescribeSelector;
+  projection: DescribeProjection;
+  fields: readonly DescribeField[];
+  limit: number;
+  maxChars: number;
+}
+
+export interface FormattedDescribeSelection {
+  description: string;
+  matched: number;
+  emitted: number;
+  truncated: boolean;
+}
+
+interface SelectedLine {
+  node: DescribeNode;
+  depth: number;
+  matched: boolean;
+}
+
+function selectionLine(
+  node: DescribeNode,
+  depth: number,
+  fields: ReadonlySet<DescribeField>,
+  matched: boolean
+): string {
+  const parts: string[] = [];
+  if (fields.has("role")) parts.push(node.role);
+  if (fields.has("label") && node.label) parts.push(formatLabel(node.label));
+  if (fields.has("value") && node.value && node.value !== node.label) {
+    parts.push(`value="${escapeForLine(node.value)}"`);
+  }
+  if (fields.has("identifier") && node.identifier) {
+    parts.push(`id="${escapeForLine(node.identifier)}"`);
+  }
+  if (fields.has("package") && node.packageName) {
+    parts.push(`package="${escapeForLine(node.packageName)}"`);
+  }
+  if (fields.has("flags")) {
+    const flags = formatFlags(node).trim();
+    if (flags) parts.push(flags);
+  }
+  // Match highlighting is projection metadata rather than a node field. Keep it
+  // even when `flags` is omitted so a `full` projection remains interpretable.
+  if (matched) parts.push("[match]");
+  if (fields.has("frame")) parts.push(fmtFrame(node.frame));
+  return `${"  ".repeat(depth)}${parts.join(" ") || "[element]"}`;
+}
+
+function isNestedSource(source: DescribeSource): boolean {
+  return (
+    source === "uiautomator" ||
+    source === "android-devtools" ||
+    source === "cdp-dom" ||
+    source === "vega-automation"
+  );
+}
+
+function orderedFlatChildren(root: DescribeNode): DescribeNode[] {
+  return root.children.slice().sort((a, b) => a.frame.y - b.frame.y || a.frame.x - b.frame.x);
+}
+
+function collectFullSelection(
+  root: DescribeNode,
+  source: DescribeSource,
+  matchSet: ReadonlySet<DescribeNode>
+): SelectedLine[] {
+  const contentRoles = source === "vega-automation" ? VEGA_CONTENT_ROLES : CONTENT_ROLES;
+  if (!isNestedSource(source)) {
+    return orderedFlatChildren(root)
+      .filter((node) => shouldEmit(node, contentRoles))
+      .map((node) => ({ node, depth: 1, matched: matchSet.has(node) }));
+  }
+
+  const lines: SelectedLine[] = [];
+  const stack = root.children
+    .slice()
+    .reverse()
+    .map((node) => ({ node, depth: 1 }));
+  while (stack.length > 0) {
+    const { node, depth } = stack.pop()!;
+    if (shouldEmit(node, contentRoles) || node.children.length > 0) {
+      lines.push({ node, depth, matched: matchSet.has(node) });
+    }
+    for (let index = node.children.length - 1; index >= 0; index--) {
+      stack.push({ node: node.children[index]!, depth: depth + 1 });
+    }
+  }
+  return lines;
+}
+
+function collectMatchedPaths(
+  root: DescribeNode,
+  matchSet: ReadonlySet<DescribeNode>
+): SelectedLine[] {
+  const lines: SelectedLine[] = [];
+  function visit(node: DescribeNode, depth: number): boolean {
+    const start = lines.length;
+    const ownMatch = matchSet.has(node);
+    lines.push({ node, depth, matched: ownMatch });
+    let descendantMatch = false;
+    for (const child of node.children) {
+      if (visit(child, depth + 1)) descendantMatch = true;
+    }
+    if (!ownMatch && !descendantMatch) lines.splice(start);
+    return ownMatch || descendantMatch;
+  }
+  for (const child of root.children) visit(child, 1);
+  return lines;
+}
+
+function renderBoundedSelection(
+  header: readonly string[],
+  candidateLines: readonly string[],
+  matched: number,
+  limit: number,
+  maxChars: number
+): { description: string; emitted: number; truncated: boolean } {
+  let lines = candidateLines.slice(0, limit);
+  let truncated = candidateLines.length > limit;
+
+  const assemble = (body: readonly string[], isTruncated: boolean): string => {
+    const sections = [...header, ...body];
+    if (isTruncated) {
+      sections.push(`… truncated (matched=${matched}, emitted=${body.length}).`);
+    }
+    return `${sections.join("\n")}\n`;
+  };
+
+  let description = assemble(lines, truncated);
+  if (description.length > maxChars) {
+    truncated = true;
+    while (lines.length > 0 && assemble(lines, true).length > maxChars) lines.pop();
+    description = assemble(lines, true);
+    // The schema minimum leaves room for the compact header and marker. Keep a
+    // defensive bounded fallback in case future header copy grows.
+    if (description.length > maxChars) {
+      const marker = `… truncated (matched=${matched}, emitted=0).\n`;
+      description = `${header.join("\n").slice(0, Math.max(0, maxChars - marker.length - 1))}\n${marker}`;
+      lines = [];
+    }
+  }
+
+  return { description, emitted: lines.length, truncated };
+}
+
+export function formatDescribeSelection(
+  root: DescribeNode,
+  opts: FormatDescribeSelectionOptions
+): FormattedDescribeSelection {
+  const matches = findDescribeMatches(root, opts.selector);
+  const matchSet = new Set(matches);
+  let selected: SelectedLine[];
+
+  switch (opts.projection) {
+    case "matches": {
+      const ordered = isNestedSource(opts.source)
+        ? matches.map((node) => ({ node, depth: 0, matched: true }))
+        : orderedFlatChildren(root)
+            .filter((node) => matchSet.has(node))
+            .map((node) => ({ node, depth: 0, matched: true }));
+      selected = ordered.map((line) => ({ ...line, depth: 0 }));
+      break;
+    }
+    case "matches-and-ancestors":
+      selected = collectMatchedPaths(root, matchSet);
+      break;
+    case "full":
+      selected = collectFullSelection(root, opts.source, matchSet);
+      break;
+  }
+
+  const fields = new Set(opts.fields);
+  const body = selected.map(({ node, depth, matched }) =>
+    selectionLine(node, depth, fields, matched)
+  );
+  const header = [
+    `Source: ${opts.source}`,
+    `Projection: ${opts.projection}`,
+    `Matched: ${matches.length}`,
+    "",
+  ];
+  const bounded = renderBoundedSelection(header, body, matches.length, opts.limit, opts.maxChars);
+  return { matched: matches.length, ...bounded };
+}