diff --git a/packages/argent-mcp/src/auto-screenshot.ts b/packages/argent-mcp/src/auto-screenshot.ts index 041347ef8..367043e50 100644 --- a/packages/argent-mcp/src/auto-screenshot.ts +++ b/packages/argent-mcp/src/auto-screenshot.ts @@ -23,6 +23,7 @@ export const AUTO_SCREENSHOT_TOOLS = new Set([ "restart-app", "open-url", "describe", + "find", "run-sequence", ]); @@ -46,10 +47,27 @@ export const AUTO_SCREENSHOT_DELAY_MS_BY_TOOL: Record = { "rotate": 1000, "keyboard": 300, "describe": 100, + // `find`'s headline action is a tap (which can trigger a transition), so match + // gesture-tap's settle delay. Its read-only actions (exists/get-text/get-attrs/ + // wait) don't mutate the screen, so they use the shorter describe-style delay + // instead — see `getAutoScreenshotDelayMs`, which keys off `args.action`. + "find": 1500, }; const DEFAULT_DELAY_MS = 1400; +// `find` actions that don't touch the device — their auto-screenshot doesn't need +// the tap settle delay (it would just over-wait ~1.5s for an unchanged screen). +// Kept in sync with the tool's ACTIONS by name (argent-mcp must not import from +// tool-server); the default action is `tap`, so an omitted action is NOT read-only. +const READ_ONLY_FIND_ACTIONS = new Set(["exists", "get-text", "get-attrs", "wait"]); + +function isReadOnlyFindAction(args: unknown): boolean { + if (!args || typeof args !== "object") return false; + const action = (args as { action?: unknown }).action; + return typeof action === "string" && READ_ONLY_FIND_ACTIONS.has(action); +} + // Auto-screenshot is on by default; the opt-out is the off-by-default // `disable-auto-screenshot` flag. `options` mirrors isFlagEnabled so tests can // point storage at a temp dir. @@ -83,9 +101,12 @@ export function shouldAutoScreenshot(toolName: string): boolean { return canonical !== "screenshot" && AUTO_SCREENSHOT_TOOLS.has(canonical); } -export function getAutoScreenshotDelayMs(toolName: string): number { +export function getAutoScreenshotDelayMs(toolName: string, args?: unknown): number { const canonical = normalizeToolName(toolName); - const base = AUTO_SCREENSHOT_DELAY_MS_BY_TOOL[canonical] ?? DEFAULT_DELAY_MS; + const base = + canonical === "find" && isReadOnlyFindAction(args) + ? AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["describe"]! + : (AUTO_SCREENSHOT_DELAY_MS_BY_TOOL[canonical] ?? DEFAULT_DELAY_MS); const envOverride = process.env.ARGENT_AUTO_SCREENSHOT_DELAY_MS; if (envOverride) { const envMs = parseInt(envOverride, 10); diff --git a/packages/argent-mcp/src/mcp-server.ts b/packages/argent-mcp/src/mcp-server.ts index 118241ff2..c8558a600 100644 --- a/packages/argent-mcp/src/mcp-server.ts +++ b/packages/argent-mcp/src/mcp-server.ts @@ -304,7 +304,7 @@ export async function startMcpServer(options: StartMcpServerOptions): Promise 0) await new Promise((r) => setTimeout(r, delayMs)); try { diff --git a/packages/argent-mcp/test/auto-screenshot.test.ts b/packages/argent-mcp/test/auto-screenshot.test.ts index bfbb90117..3fe410fe4 100644 --- a/packages/argent-mcp/test/auto-screenshot.test.ts +++ b/packages/argent-mcp/test/auto-screenshot.test.ts @@ -180,6 +180,32 @@ describe("getAutoScreenshotDelayMs", () => { process.env.ARGENT_AUTO_SCREENSHOT_DELAY_MS = "abc"; expect(getAutoScreenshotDelayMs("gesture-tap")).toBe(1500); }); + + // N1: a read-only `find` shouldn't over-wait the tap settle for an unchanged + // screen — it uses describe's short delay, keyed off args.action. + it("uses the describe delay for a read-only find action", () => { + for (const action of ["exists", "get-text", "get-attrs", "wait"]) { + expect(getAutoScreenshotDelayMs("find", { action })).toBe( + AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["describe"] + ); + } + // normalized names work too + expect(getAutoScreenshotDelayMs("mcp__argent__find", { action: "exists" })).toBe( + AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["describe"] + ); + }); + + it("keeps the tap settle delay for a find tapping action (or omitted action → default tap)", () => { + expect(getAutoScreenshotDelayMs("find", { action: "tap" })).toBe( + AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"] + ); + expect(getAutoScreenshotDelayMs("find", { action: "fill", text: "x" })).toBe( + AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"] + ); + // omitted action defaults to tap → full settle; and no args at all + expect(getAutoScreenshotDelayMs("find", {})).toBe(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"]); + expect(getAutoScreenshotDelayMs("find")).toBe(AUTO_SCREENSHOT_DELAY_MS_BY_TOOL["find"]); + }); }); // --------------------------------------------------------------------------- @@ -195,6 +221,11 @@ describe("shouldAutoScreenshot — unified surface", () => { for (const t of [ "gesture-tap", "gesture-swipe", + "gesture-scroll", + "gesture-drag", + "gesture-custom", + "gesture-pinch", + "gesture-rotate", "button", "keyboard", "rotate", @@ -202,6 +233,9 @@ describe("shouldAutoScreenshot — unified surface", () => { "restart-app", "open-url", "describe", + // `find`'s default action is a tap, which can trigger a transition worth + // capturing — so it must auto-screenshot like the other interaction tools. + "find", "run-sequence", ]) { expect(shouldAutoScreenshot(t)).toBe(true); diff --git a/packages/skills/rules/argent.md b/packages/skills/rules/argent.md index d4960a3d3..f2cf20db5 100644 --- a/packages/skills/rules/argent.md +++ b/packages/skills/rules/argent.md @@ -37,7 +37,7 @@ If argent is ABSENT, treat it as an expected state, not an error to retry. Do no **Never** derive tap coordinates from a screenshot -Before **every** tap, you MUST call a discovery tool and extract coordinates from the result. This is not optional. Preferred tools are, in order: +Before **every** coordinate tap, you MUST call a discovery tool and extract coordinates from the result. This is not optional. Preferred tools are, in order: - `describe` - native app-level components and safely targetable foreground apps (iOS and Android). - `native-describe-screen` - accessibility screen description via injected native devtools (iOS only) @@ -45,7 +45,12 @@ Before **every** tap, you MUST call a discovery tool and extract coordinates fro `native-user-interactable-view-at-point` / `native-view-at-point` are follow-up diagnostics once you already have a candidate point (iOS only). -Whenever something changed YOU MUST first call `describe`, or another appropriate discovery tool so you do not hallucinate element positions. Do not guess coordinates if you can use discovery tool. Do not tap if you have not called a discovery tool in the current step. Screenshots alone are never sufficient for coordinates. +Choose exactly one tap path for a target: + +- Unknown screen/layout or ambiguous target: `describe` / `native-describe-screen` / `debugger-component-tree` -> `gesture-tap` using the returned frame or tap point. +- Known visible target by text/label/value/role/id and no current coordinates for it: `find` can locate and act in one call. +- Recent discovery already exposed the same target frame/tap point: do **not** call `find` for that target; use `gesture-tap`. +- UI changed after discovery (navigation, modal, list update, keyboard, animation, reload): the old coordinates are stale. Re-discover, or use `find` if the next target is known by text/label/id. If a **tap fails twice** at the same coordinates, **stop retrying**. Re-run the discovery tool. @@ -73,7 +78,7 @@ Decision order: - All simulator/emulator interactions go through argent MCP tools — never use `xcrun simctl`, raw `curl` to simulator ports, or the simulator-server binary directly. - Before calling any gesture tool for the first time, use ToolSearch to load its schema. -- Interaction tools (`gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot automatically. +- Interaction tools (`find`, `gesture-tap`, `gesture-swipe`, `gesture-pinch`, `gesture-rotate`, `gesture-custom`, `launch-app`, etc.) return a screenshot automatically. Call `screenshot` separately only for a baseline before any action or after a delay. - Always open apps with `launch-app` or `open-url` — never tap home screen icons. - Always use `run-sequence` when performing multiple sequential device actions where you don't need to observe the screen between steps. More in `argent-device-interact` skill. diff --git a/packages/skills/skills/argent-create-flow/SKILL.md b/packages/skills/skills/argent-create-flow/SKILL.md index 40dc8aa46..b630a71e7 100644 --- a/packages/skills/skills/argent-create-flow/SKILL.md +++ b/packages/skills/skills/argent-create-flow/SKILL.md @@ -12,7 +12,7 @@ A flow is a recorded sequence of MCP tool calls saved to a `.yaml` file in the ` | Tool | Purpose | | ------------------------ | -------------------------------------------------------------------------- | | `flow-start-recording` | Start recording — takes a name and executionPrerequisite, creates the file | -| `flow-add-step` | Execute a tool call live and record it if it succeeds | +| `flow-add-step` | Execute a tool call live and record it when the call succeeds | | `flow-add-echo` | Add a label/comment that prints during replay | | `flow-finish-recording` | Stop recording and get a summary | | `flow-read-prerequisite` | Read a flow's execution prerequisite without running it | diff --git a/packages/skills/skills/argent-device-interact/SKILL.md b/packages/skills/skills/argent-device-interact/SKILL.md index 1f73007c9..c741d6a2b 100644 --- a/packages/skills/skills/argent-device-interact/SKILL.md +++ b/packages/skills/skills/argent-device-interact/SKILL.md @@ -32,7 +32,7 @@ Use `list-devices` to get a target id. Results are tagged with `platform` (`ios` 3. **Use `gesture-swipe` for lists/scrolling**, not `gesture-custom`, unless you need non-linear movement. On Chromium use `gesture-scroll` instead — `gesture-swipe` is touch-only. Consider whether you need multiple swipes, if yes - use `run-sequence`. 4. **Tap a text field before typing**, then use `keyboard` to enter text. 5. **Coordinates are normalized** — always 0.0–1.0, not pixels. -6. **For app navigation, prefer `describe` first.** It works on any screen without app restart. Do not navigate from screenshots on regular in-app screens unless `describe` failed to expose a reliable target. Use `native-describe-screen` only when you need app-scoped UIKit properties. +6. **For app navigation, choose one tap path.** Unknown screen/layout: `describe` -> `gesture-tap` using the returned frame. Known visible target by text/label/id and no current coordinates: use `find` directly. Do not call `find` immediately after `describe` for the same target. ## 3. Opening Apps @@ -56,34 +56,42 @@ Common schemes: `messages://`, `settings://`, `maps://?q=`, `tel://