feat(tool-server): add find tool to locate an element and optionally act on it

packages/argent-mcp/src/auto-screenshot.ts

@@ -23,6 +23,7 @@ export const AUTO_SCREENSHOT_TOOLS = new Set([
   "restart-app",
   "open-url",
   "describe",
+  "find",
   "run-sequence",
 ]);
 
@@ -46,10 +47,27 @@ export const AUTO_SCREENSHOT_DELAY_MS_BY_TOOL: Record<string, number> = {
   "rotate": 1000,
   "keyboard": 300,
   "describe": 100,
+  // `find`'s headline action is a tap (which can trigger a transition), so match
+  // gesture-tap's settle delay. Its read-only actions (exists/get-text/get-attrs/
+  // wait) don't mutate the screen, so they use the shorter describe-style delay
+  // instead — see `getAutoScreenshotDelayMs`, which keys off `args.action`.
+  "find": 1500,
 };
 
 const DEFAULT_DELAY_MS = 1400;
 
+// `find` actions that don't touch the device — their auto-screenshot doesn't need
+// the tap settle delay (it would just over-wait ~1.5s for an unchanged screen).
+// Kept in sync with the tool's ACTIONS by name (argent-mcp must not import from
+// tool-server); the default action is `tap`, so an omitted action is NOT read-only.
+const READ_ONLY_FIND_ACTIONS = new Set(["exists", "get-text", "get-attrs", "wait"]);
+
+function isReadOnlyFindAction(args: unknown): boolean {
+  if (!args || typeof args !== "object") return false;
+  const action = (args as { action?: unknown }).action;
+  return typeof action === "string" && READ_ONLY_FIND_ACTIONS.has(action);
+}
+
 // Auto-screenshot is on by default; the opt-out is the off-by-default
 // `disable-auto-screenshot` flag. `options` mirrors isFlagEnabled so tests can
 // point storage at a temp dir.
@@ -83,9 +101,12 @@ export function shouldAutoScreenshot(toolName: string): boolean {
   return canonical !== "screenshot" && AUTO_SCREENSHOT_TOOLS.has(canonical);
 }
 
-export function getAutoScreenshotDelayMs(toolName: string): number {
+export function getAutoScreenshotDelayMs(toolName: string, args?: unknown): number {
   const canonical = normalizeToolName(toolName);
-  const base = AUTO_SCREENSHOT_DELAY_MS_BY_TOOL[canonical] ?? DEFAULT_DELAY_MS;
+  const base =
+    canonical === "find" && isReadOnlyFindAction(args)
+      ? AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["describe"]!
+      : (AUTO_SCREENSHOT_DELAY_MS_BY_TOOL[canonical] ?? DEFAULT_DELAY_MS);
   const envOverride = process.env.ARGENT_AUTO_SCREENSHOT_DELAY_MS;
   if (envOverride) {
     const envMs = parseInt(envOverride, 10);

packages/argent-mcp/src/mcp-server.ts

@@ -304,7 +304,7 @@ export async function startMcpServer(options: StartMcpServerOptions): Promise<vo
 
       const udid = getUdidFromArgs(params.arguments);
       if (autoScreenshotOn && udid && shouldAutoScreenshot(params.name)) {
-        const delayMs = getAutoScreenshotDelayMs(params.name);
+        const delayMs = getAutoScreenshotDelayMs(params.name, params.arguments);
         if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
 
         try {

packages/argent-mcp/test/auto-screenshot.test.ts

@@ -180,6 +180,32 @@ describe("getAutoScreenshotDelayMs", () => {
     process.env.ARGENT_AUTO_SCREENSHOT_DELAY_MS = "abc";
     expect(getAutoScreenshotDelayMs("gesture-tap")).toBe(1500);
   });
+
+  // N1: a read-only `find` shouldn't over-wait the tap settle for an unchanged
+  // screen — it uses describe's short delay, keyed off args.action.
+  it("uses the describe delay for a read-only find action", () => {
+    for (const action of ["exists", "get-text", "get-attrs", "wait"]) {
+      expect(getAutoScreenshotDelayMs("find", { action })).toBe(
+        AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["describe"]
+      );
+    }
+    // normalized names work too
+    expect(getAutoScreenshotDelayMs("mcp__argent__find", { action: "exists" })).toBe(
+      AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["describe"]
+    );
+  });
+
+  it("keeps the tap settle delay for a find tapping action (or omitted action → default tap)", () => {
+    expect(getAutoScreenshotDelayMs("find", { action: "tap" })).toBe(
+      AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"]
+    );
+    expect(getAutoScreenshotDelayMs("find", { action: "fill", text: "x" })).toBe(
+      AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"]
+    );
+    // omitted action defaults to tap → full settle; and no args at all
+    expect(getAutoScreenshotDelayMs("find", {})).toBe(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"]);
+    expect(getAutoScreenshotDelayMs("find")).toBe(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"]);
+  });
 });
 
 // ---------------------------------------------------------------------------
@@ -195,13 +221,21 @@ describe("shouldAutoScreenshot — unified surface", () => {
     for (const t of [
       "gesture-tap",
       "gesture-swipe",
+      "gesture-scroll",
+      "gesture-drag",
+      "gesture-custom",
+      "gesture-pinch",
+      "gesture-rotate",
       "button",
       "keyboard",
       "rotate",
       "launch-app",
       "restart-app",
       "open-url",
       "describe",
+      // `find`'s default action is a tap, which can trigger a transition worth
+      // capturing — so it must auto-screenshot like the other interaction tools.
+      "find",
       "run-sequence",
     ]) {
       expect(shouldAutoScreenshot(t)).toBe(true);

packages/skills/rules/argent.md

@@ -37,15 +37,20 @@ If argent is ABSENT, treat it as an expected state, not an error to retry. Do no
 
 <tapping_rule>
 <important>**Never** derive tap coordinates from a screenshot</important>
-Before **every** tap, you MUST call a discovery tool and extract coordinates from the result. This is not optional. Preferred tools are, in order:
+Before **every** coordinate tap, you MUST call a discovery tool and extract coordinates from the result. This is not optional. Preferred tools are, in order:
 
 - `describe` - native app-level components and safely targetable foreground apps (iOS and Android).
 - `native-describe-screen` - accessibility screen description via injected native devtools (iOS only)
 - `debugger-component-tree` - react-native specific components
 
 `native-user-interactable-view-at-point` / `native-view-at-point` are follow-up diagnostics once you already have a candidate point (iOS only).
 
-Whenever something changed YOU MUST first call `describe`, or another appropriate discovery tool so you do not hallucinate element positions. Do not guess coordinates if you can use discovery tool. Do not tap if you have not called a discovery tool in the current step. Screenshots alone are never sufficient for coordinates.
+Choose exactly one tap path for a target:
+
+- Unknown screen/layout or ambiguous target: `describe` / `native-describe-screen` / `debugger-component-tree` -> `gesture-tap` using the returned frame or tap point.
+- Known visible target by text/label/value/role/id and no current coordinates for it: `find` can locate and act in one call.
+- Recent discovery already exposed the same target frame/tap point: do **not** call `find` for that target; use `gesture-tap`.
+- UI changed after discovery (navigation, modal, list update, keyboard, animation, reload): the old coordinates are stale. Re-discover, or use `find` if the next target is known by text/label/id.
 
 If a **tap fails twice** at the same coordinates, **stop retrying**. Re-run the discovery tool.
 
@@ -73,7 +78,7 @@ Decision order:
 - All simulator/emulator interactions go through argent MCP tools — never use `xcrun simctl`,
   raw `curl` to simulator ports, or the simulator-server binary directly.
 - Before calling any gesture tool for the first time, use ToolSearch to load its schema.
-- Interaction tools (`gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot automatically.
+- Interaction tools (`find`, `gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot automatically.
   Call `screenshot` separately only for a baseline before any action or after a delay.
 - Always open apps with `launch-app` or `open-url` — never tap home screen icons.
 - Always use `run-sequence` when performing multiple sequential device actions where you don't need to observe the screen between steps. More in `argent-device-interact` skill.

packages/skills/skills/argent-create-flow/SKILL.md

@@ -12,7 +12,7 @@ A flow is a recorded sequence of MCP tool calls saved to a `.yaml` file in the `
 | Tool                     | Purpose                                                                    |
 | ------------------------ | -------------------------------------------------------------------------- |
 | `flow-start-recording`   | Start recording — takes a name and executionPrerequisite, creates the file |
-| `flow-add-step`          | Execute a tool call live and record it if it succeeds                      |
+| `flow-add-step`          | Execute a tool call live and record it when the call succeeds              |
 | `flow-add-echo`          | Add a label/comment that prints during replay                              |
 | `flow-finish-recording`  | Stop recording and get a summary                                           |
 | `flow-read-prerequisite` | Read a flow's execution prerequisite without running it                    |