diff --git a/README.md b/README.md index 320d39707..1dc799c9d 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,65 @@ Argent drives a growing set of targets through a single toolkit, each with the r --- +## Physical iOS devices (experimental) + +Argent can drive a **physical iPhone** — no app installed on the device — over Apple's +CoreDevice "remote control" services (the same path Xcode's device window uses), via +[`pymobiledevice3`](https://github.com/doronz88/pymobiledevice3). Supported interactions: +`screenshot`, `gesture-tap`, `gesture-swipe`, `button`, `launch-app`, and `describe` (the +live on-screen accessibility tree — see the note below). The device shows up in `list-devices` with +`kind: "device"`. Interactions run through one persistent `pymobiledevice3` helper per +device (connected once), so a tap/screenshot costs a socket write rather than a fresh +Python cold-start. + +**Requirements** + +- **iOS 27 or later for tap/swipe** — Apple gates host-driven touch input to iOS 27+; on + earlier versions those commands report `CoreDeviceError 9021`. Screenshot and hardware + buttons work on earlier iOS versions too. +- macOS with Xcode, and `pymobiledevice3` installed (e.g. `pipx install pymobiledevice3`). +- The iPhone connected, unlocked, trusted, with **Developer Mode** on. + +**Setup** + +1. Enable the feature flag: + ```sh + argent enable physical-ios-devices + ``` +2. Connect the iPhone (unlocked, trusted, Developer Mode on). + +`list-devices` then includes the iPhone, and the supported tools work against its UDID. +The first interaction (or `boot-device`) starts the required CoreDevice tunnel +automatically: Argent shows a standard macOS authorization prompt (Touch ID / password) +to launch `pymobiledevice3 remote tunneld` as root (creating the tunnel interface needs +root once; every other command runs unprivileged). No manual `sudo`. When the signed +`argent-device-auth` helper is installed, the prompt is branded as Argent; otherwise it's +the system's default admin prompt. + +If the prompt is declined or there's no GUI session (headless), start the tunnel manually +and leave it running: `sudo pymobiledevice3 remote tunneld`. + +**Limitations / notes** + +- `describe` returns the device's **live on-screen accessibility tree** — the frontmost app's + elements (or the home screen), read app-free via the iOS-26+ accessibility-audit service over + CoreDevice. Element labels, values, traits (roles) and reading order are exact. Frames are exact + for the elements the accessibility audit reports and **interpolated** from reading-order + neighbours for the rest (Apple doesn't expose per-element geometry on a physical device), so + they're good enough to tap a row in a vertical list — but confirm with `screenshot` before a + precise tap, especially for controls like toggles. (This needs the RSDCheckin handshake iOS 26 + added; the helper performs it. For pixel-exact in-app frames + taps you'd need an on-device + XCUITest runner, which requires code-signing.) +- Not supported yet (return a clear "not supported" error): keyboard/typing, pinch & rotate + (multi-touch), `open-url`, `reinstall-app`, `restart-app`, and the native inspection / + profiling tools (`native-*`, `native-profiler-*`, `screenshot-diff`). `launch-app` (via + `devicectl`) works independently of the CoreDevice tunnel — it can succeed even before the + tunnel setup above has run. +- Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT` + (defaults to `49151`). + +--- + ## Installation #### Prerequisites diff --git a/packages/argent/scripts/bundle-tools.cjs b/packages/argent/scripts/bundle-tools.cjs index 1913bf7ab..2e77a488a 100644 --- a/packages/argent/scripts/bundle-tools.cjs +++ b/packages/argent/scripts/bundle-tools.cjs @@ -104,6 +104,17 @@ const AX_TCP_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/tcp/ax-service"); const BIN_DIR = path.resolve(__dirname, "../bin"); const AX_BIN_DEST = path.resolve(BIN_DIR, "darwin/ax-service"); const AX_TCP_BIN_DEST = path.resolve(BIN_DIR, "darwin/tcp/ax-service"); +// argent-device-auth: macOS host helper for the branded physical-iOS tunnel +// auth prompt. Best-effort — only present once argent-private publishes the +// signed binary; until then physical iOS falls back to the osascript prompt. +const DEVICE_AUTH_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/argent-device-auth"); +const DEVICE_AUTH_BIN_DEST = path.resolve(BIN_DIR, "darwin/argent-device-auth"); +// Argent icon shown in that prompt (committed under native-devtools-ios/assets). +const DEVICE_ICON_SRC = path.resolve( + WORKSPACE_ROOT, + "packages/native-devtools-ios/assets/argent-icon.png" +); +const DEVICE_ICON_DEST = path.resolve(__dirname, "../assets/argent-icon.png"); // tvOS control binaries. Both are macOS-only: tvos-ax-service runs inside an // appletvsimulator, tvos-hid-daemon runs on the host. Unix-socket only. const TVOS_AX_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/tvos-ax-service"); @@ -208,6 +219,27 @@ const ASSETS = [ copiedLabel: "ax-service (tcp) binary", missLabel: "ax-service (tcp) binary", }, + // macOS host helper for the branded physical-iOS tunnel auth prompt. + // Best-effort: present once argent-private publishes the signed binary; + // until then physical iOS falls back to the (unbranded) osascript prompt. + { + kind: "file", + src: DEVICE_AUTH_BIN_SRC, + dest: DEVICE_AUTH_BIN_DEST, + mode: 0o755, + required: false, + copiedLabel: "argent-device-auth binary", + missLabel: "argent-device-auth binary", + }, + // Argent icon shown in the device-auth prompt (committed; best-effort copy). + { + kind: "file", + src: DEVICE_ICON_SRC, + dest: DEVICE_ICON_DEST, + required: false, + copiedLabel: "device-auth icon", + missLabel: "device-auth icon", + }, // tvOS AX reader — spawned inside an appletvsimulator via simctl to read // the focus-engine accessibility tree. macOS-only, unix-socket transport. { diff --git a/packages/configuration-core/src/flags.ts b/packages/configuration-core/src/flags.ts index b9e61fe3b..87a8b0c70 100644 --- a/packages/configuration-core/src/flags.ts +++ b/packages/configuration-core/src/flags.ts @@ -57,6 +57,11 @@ export const FLAG_REGISTRY: readonly FlagDefinition[] = [ name: "artifacts-list-endpoint", description: "Expose GET /artifacts for remote artifact inventory consumers.", }, + { + name: "physical-ios-devices", + description: + "Discover and control physical iOS devices (iOS 27+) over Apple's CoreDevice tunnel via pymobiledevice3. The required tunnel is auto-started via a macOS admin prompt (or run `sudo pymobiledevice3 remote tunneld` manually). Supports screenshot, tap, swipe, hardware buttons, and launch-app.", + }, { name: "tool-server-event-log", description: "Write structured tool-server lifecycle events to a JSONL file.", diff --git a/packages/native-devtools-ios/assets/argent-icon.png b/packages/native-devtools-ios/assets/argent-icon.png new file mode 100644 index 000000000..dc54076a3 Binary files /dev/null and b/packages/native-devtools-ios/assets/argent-icon.png differ diff --git a/packages/native-devtools-ios/src/index.ts b/packages/native-devtools-ios/src/index.ts index 4be5101b9..75e39e599 100644 --- a/packages/native-devtools-ios/src/index.ts +++ b/packages/native-devtools-ios/src/index.ts @@ -9,6 +9,9 @@ import * as fs from "node:fs"; const DYLIB_DIR = process.env.ARGENT_NATIVE_DEVTOOLS_DIR ?? path.join(__dirname, "..", "dylibs"); const BIN_DIR = process.env.ARGENT_SIMULATOR_SERVER_DIR ?? path.join(__dirname, "..", "bin"); const DYLIB_TCP_DIR = process.env.ARGENT_NATIVE_DEVTOOLS_TCP_DIR ?? path.join(DYLIB_DIR, "tcp"); +// Committed static assets (e.g. the Argent icon shown in the device-auth prompt). +const ASSETS_DIR = + process.env.ARGENT_NATIVE_DEVTOOLS_ASSETS_DIR ?? path.join(__dirname, "..", "assets"); const DYLIB_TVOS_DIR = path.join(DYLIB_DIR, "tvos"); // The *local* and *tvos* accessors are darwin-only: the dylibs they return get @@ -146,6 +149,27 @@ export function axServiceBinaryPathTcp(): string { return requireBinIn(platformTcpBinDir(), "ax-service"); } +// argent-device-auth is the macOS host helper that pops the branded macOS +// authorization modal (password / Touch ID) and runs a command as root — +// Argent uses it to start the physical-iOS CoreDevice tunnel without a manual +// `sudo`. Unlike the resolvers above it returns null (rather than throwing) +// when absent, so callers can fall back to a less-branded escalation path. +// Override the binary with ARGENT_DEVICE_AUTH_HELPER (absolute path). +export function deviceAuthHelperPath(): string | null { + if (process.platform !== "darwin") return null; + const override = process.env.ARGENT_DEVICE_AUTH_HELPER; + if (override) return fs.existsSync(override) ? override : null; + const p = path.join(platformBinDir(), "argent-device-auth"); + return fs.existsSync(p) ? p : null; +} + +// Path to the Argent icon shown in the device-auth prompt, or null if missing. +// Override with ARGENT_DEVICE_ICON (absolute path to a PNG/icns). +export function argentIconPath(): string | null { + const p = process.env.ARGENT_DEVICE_ICON ?? path.join(ASSETS_DIR, "argent-icon.png"); + return fs.existsSync(p) ? p : null; +} + // tvOS control binaries. tvos-ax-service is `simctl spawn`d into an // appletvsimulator to read the focus-engine AX state; tvos-hid-daemon runs on // the host and injects Siri-remote HID via SimulatorKit. Both are darwin-only. diff --git a/packages/registry/src/errors.ts b/packages/registry/src/errors.ts index 1ace1a0fe..8e73eee6a 100644 --- a/packages/registry/src/errors.ts +++ b/packages/registry/src/errors.ts @@ -26,11 +26,13 @@ export const FAILURE_COMMANDS = [ "emulator", "vega", "xcrun_simctl", + "xcrun_devicectl", "xctrace", "native_devtools", "android_devtools", "ax_service", "simulator_server", + "pymobiledevice3", "cdp", "electron", "npm", diff --git a/packages/registry/src/failure-codes.ts b/packages/registry/src/failure-codes.ts index 6212304c2..98e7f83d7 100644 --- a/packages/registry/src/failure-codes.ts +++ b/packages/registry/src/failure-codes.ts @@ -79,6 +79,7 @@ export const FAILURE_CODES = { SIMULATOR_SERVER_READY_TIMEOUT: "SIMULATOR_SERVER_READY_TIMEOUT", SIMULATOR_SERVER_PROCESS_ERROR: "SIMULATOR_SERVER_PROCESS_ERROR", SIMULATOR_SERVER_TERMINATED: "SIMULATOR_SERVER_TERMINATED", + SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED: "SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED", AX_QUERY_TIMEOUT: "AX_QUERY_TIMEOUT", AX_DAEMON_READY_TIMEOUT: "AX_DAEMON_READY_TIMEOUT", @@ -86,6 +87,7 @@ export const FAILURE_CODES = { AX_DAEMON_PROCESS_ERROR: "AX_DAEMON_PROCESS_ERROR", AX_FACTORY_OPTIONS_MISSING: "AX_FACTORY_OPTIONS_MISSING", AX_WRONG_PLATFORM: "AX_WRONG_PLATFORM", + AX_PHYSICAL_DEVICE_UNSUPPORTED: "AX_PHYSICAL_DEVICE_UNSUPPORTED", AX_DEVICE_ID_INVALID: "AX_DEVICE_ID_INVALID", AX_DESCRIBE_ERROR: "AX_DESCRIBE_ERROR", AX_QUERY_FAILED: "AX_QUERY_FAILED", @@ -96,9 +98,22 @@ export const FAILURE_CODES = { ANDROID_REINSTALL_INSTALL_FAILED: "ANDROID_REINSTALL_INSTALL_FAILED", ANDROID_RESTART_FAILED: "ANDROID_RESTART_FAILED", IOS_LAUNCH_SIMCTL_FAILED: "IOS_LAUNCH_SIMCTL_FAILED", + IOS_LAUNCH_DEVICECTL_FAILED: "IOS_LAUNCH_DEVICECTL_FAILED", IOS_OPEN_URL_FAILED: "IOS_OPEN_URL_FAILED", IOS_REINSTALL_INSTALL_FAILED: "IOS_REINSTALL_INSTALL_FAILED", IOS_RESTART_LAUNCH_FAILED: "IOS_RESTART_LAUNCH_FAILED", + + CORE_DEVICE_FLAG_DISABLED: "CORE_DEVICE_FLAG_DISABLED", + CORE_DEVICE_FACTORY_OPTIONS_MISSING: "CORE_DEVICE_FACTORY_OPTIONS_MISSING", + CORE_DEVICE_WRONG_DEVICE: "CORE_DEVICE_WRONG_DEVICE", + CORE_DEVICE_PMD3_NOT_FOUND: "CORE_DEVICE_PMD3_NOT_FOUND", + CORE_DEVICE_TUNNEL_UNREACHABLE: "CORE_DEVICE_TUNNEL_UNREACHABLE", + CORE_DEVICE_TUNNEL_NOT_REGISTERED: "CORE_DEVICE_TUNNEL_NOT_REGISTERED", + CORE_DEVICE_TUNNEL_AUTH_DECLINED: "CORE_DEVICE_TUNNEL_AUTH_DECLINED", + CORE_DEVICE_TUNNEL_TIMEOUT: "CORE_DEVICE_TUNNEL_TIMEOUT", + CORE_DEVICE_IOS_VERSION_TOO_OLD: "CORE_DEVICE_IOS_VERSION_TOO_OLD", + CORE_DEVICE_COMMAND_FAILED: "CORE_DEVICE_COMMAND_FAILED", + NATIVE_DEVTOOLS_DESCRIBE_ERROR: "NATIVE_DEVTOOLS_DESCRIBE_ERROR", NATIVE_DEVTOOLS_VIEW_AT_POINT_ERROR: "NATIVE_DEVTOOLS_VIEW_AT_POINT_ERROR", NATIVE_DEVTOOLS_USER_INTERACTABLE_VIEW_AT_POINT_ERROR: diff --git a/packages/tool-server/src/blueprints/ax-service.ts b/packages/tool-server/src/blueprints/ax-service.ts index 7e7a48116..0b986ef36 100644 --- a/packages/tool-server/src/blueprints/ax-service.ts +++ b/packages/tool-server/src/blueprints/ax-service.ts @@ -235,6 +235,24 @@ export const axServiceBlueprint: ServiceBlueprint = { } ); } + if (device.kind === "device") { + // ax-service uses `xcrun simctl spawn`, which only works on simulators. + // Physical iPhones are driven over CoreDevice; their in-app accessibility + // tree is Apple-gated (the axAuditDaemon rejects trusted/AppleInternal + // callers — see describe/platforms/ios/index.ts). `describe` handles a + // physical device itself (via the SpringBoard home-screen layout) and + // never resolves ax-service for it — this is a backstop for any other + // direct ax-service resolution. + throw new FailureError( + `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; its in-app accessibility tree is Apple-gated, so use describe (SpringBoard home screen) or screenshot instead.`, + { + error_code: FAILURE_CODES.AX_PHYSICAL_DEVICE_UNSUPPORTED, + failure_stage: "ax_service_factory_platform", + failure_area: "tool_server", + error_kind: "unsupported", + } + ); + } // Reject before spawning. An undefined `device.id` slips through when an // inner tool is invoked via a wrapper that doesn't re-validate the inner // schema. Without this guard `getSocketPath(undefined).slice` would crash diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts new file mode 100644 index 000000000..882b52e03 --- /dev/null +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -0,0 +1,619 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { existsSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { tmpdir, homedir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { + TypedEventEmitter, + FAILURE_CODES, + FailureError, + type DeviceInfo, + type ServiceBlueprint, + type ServiceInstance, + type ServiceEvents, +} from "@argent/registry"; +import { isFlagEnabled } from "@argent/configuration-core"; +import { deviceAuthHelperPath, argentIconPath } from "@argent/native-devtools-ios"; +import { + CoreDeviceAgent, + CoreDeviceAgentError, + materializeAgentScript, + resolvePmd3Python, +} from "./coredevice-agent"; + +const execFileAsync = promisify(execFile); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export const CORE_DEVICE_NAMESPACE = "CoreDevice"; + +// Opt-in flag (also gates discovery in list-devices). The privileged tunnel +// start must not be reachable unless the user enabled the experimental feature. +const PHYSICAL_IOS_FLAG = "physical-ios-devices"; + +/** + * Throw the standard "enable the flag" error unless physical-iOS support is on. + * The single gate for every physical-iOS operation — the CoreDevice factory and + * tunnel start funnel through it, and `launch-app` (which drives a real device + * via `devicectl` rather than the CoreDevice backend) calls it directly so it + * can't bypass the opt-in. + */ +export function assertPhysicalIosEnabled(): void { + if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { + throw new FailureError( + `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}`, + { + error_code: FAILURE_CODES.CORE_DEVICE_FLAG_DISABLED, + failure_stage: "core_device_flag_gate", + failure_area: "tool_server", + error_kind: "unsupported", + } + ); + } +} + +// The registry's `ServiceRef.options` is typed as `Record`; +// the intersection adds the implicit string index signature an interface lacks. +type CoreDeviceFactoryOptions = Record & { device: DeviceInfo }; + +/** + * Backend for a physical iOS device, driven over Apple's CoreDevice "remote + * control" services via the `pymobiledevice3` CLI — no app installed on the + * device. This is a separate blueprint from the simulator-server because real + * iPhones speak an entirely different transport (the iOS-17+ RemoteXPC tunnel), + * mirroring how physical Android uses its own `android_device` controller. + * + * Requirements (all surfaced as actionable errors): iOS 27+ (Apple gates the + * touch/"remote control" services to 27.0+), `pymobiledevice3` installed, and a + * running CoreDevice tunnel (`sudo pymobiledevice3 remote tunneld`, which needs + * root to create the tunnel interface — every command here then runs unprivileged). + */ +export interface CoreDeviceAxElement { + /** VoiceOver caption: label + value + traits, e.g. "Wi-Fi, 1, Button, Toggle". */ + caption: string; + /** Per-element identifier (hex) — stable within a single snapshot. */ + id: string; + /** On-screen rect "{{x, y}, {w, h}}" in points, when the audit reported one. */ + rect?: string; +} + +export interface CoreDeviceAxTree { + elements: CoreDeviceAxElement[]; + /** Screen size in points, for normalizing rects to 0..1. */ + screen?: { w?: number; h?: number }; +} + +export interface CoreDeviceApi { + /** Capture a PNG to a temp file and return its path. */ + screenshot(): Promise<{ path: string }>; + /** Tap at normalized (x, y) in 0..1. */ + tap(x: number, y: number): Promise; + /** Swipe/drag from (fromX, fromY) to (toX, toY), all normalized 0..1. */ + swipe(fromX: number, fromY: number, toX: number, toY: number, durationMs: number): Promise; + /** Press a hardware button by its pymobiledevice3 name (home/lock/volume-up/volume-down/...). */ + button(name: string): Promise; + /** + * The on-screen accessibility tree of the frontmost app (or the home screen), + * via the iOS-26+ axAudit service — rich semantic captions + reading order for + * every element, plus on-screen rects for the subset the accessibility audit + * flags (Apple doesn't expose per-element geometry on hardware). Backs `describe`. + */ + axtree(): Promise; +} + +export function coreDeviceRef(device: DeviceInfo): { + urn: string; + options: CoreDeviceFactoryOptions; +} { + return { + urn: `${CORE_DEVICE_NAMESPACE}:${device.id}`, + options: { device }, + }; +} + +interface Rsd { + address: string; + port: number; +} + +function tunneldPort(): number { + const raw = process.env.ARGENT_PMD3_TUNNELD_PORT; + const n = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isInteger(n) && n > 0 && n <= 65535 ? n : 49151; +} + +/** Resolve the pymobiledevice3 executable: env override, common install dirs, then PATH. */ +function resolvePmd3(): string { + const override = process.env.ARGENT_PYMOBILEDEVICE3; + if (override) return override; + const candidates = [ + join(homedir(), ".local", "bin", "pymobiledevice3"), + "/opt/homebrew/bin/pymobiledevice3", + "/usr/local/bin/pymobiledevice3", + ]; + for (const c of candidates) if (existsSync(c)) return c; + return "pymobiledevice3"; +} + +/** + * Fail fast with an install hint when pymobiledevice3 is missing, rather than + * surfacing a raw `spawn ENOENT` from the first interaction. Runs `version` (a + * cheap subcommand); a missing binary throws ENOENT, anything else is tolerated. + */ +async function verifyPmd3Available(pmd3: string): Promise { + try { + await execFileAsync(pmd3, ["version"], { timeout: 10_000 }); + } catch (err) { + if ((err as { code?: string })?.code === "ENOENT") { + throw new FailureError( + `pymobiledevice3 was not found (tried "${pmd3}"). Physical iOS control needs it — ` + + `install it (e.g. \`pipx install pymobiledevice3\`) or set ARGENT_PYMOBILEDEVICE3 to its path.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_PMD3_NOT_FOUND, + failure_stage: "core_device_verify_pmd3", + failure_area: "tool_server", + error_kind: "dependency_missing", + failure_command: "pymobiledevice3", + }, + { cause: err instanceof Error ? err : new Error(String(err)) } + ); + } + // Non-ENOENT (odd `version` failure): don't block; a real command will report. + } +} + +/** Resolve pymobiledevice3 to an absolute path — the privileged root shell does + * not inherit the user's PATH (so `~/.local/bin` is invisible to it). */ +async function resolvePmd3Absolute(): Promise { + const p = resolvePmd3(); + if (p.startsWith("/")) return p; + try { + const { stdout } = await execFileAsync("/usr/bin/which", [p], { timeout: 5_000 }); + const abs = stdout.trim(); + if (abs.startsWith("/")) return abs; + } catch { + // fall through to the install hint + } + throw new FailureError( + `pymobiledevice3 was not found on PATH. Install it (e.g. \`pipx install pymobiledevice3\`) ` + + `or set ARGENT_PYMOBILEDEVICE3 to its absolute path.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_PMD3_NOT_FOUND, + failure_stage: "core_device_resolve_pmd3_absolute", + failure_area: "tool_server", + error_kind: "dependency_missing", + failure_command: "pymobiledevice3", + } + ); +} + +/** Single-quote a string for safe use as one /bin/sh word. */ +function shellSingleQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'`; +} + +/** Escape a /bin/sh command string for embedding inside an AppleScript double-quoted literal. */ +export function appleScriptQuote(cmd: string): string { + return cmd.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); +} + +/** + * The /bin/sh command (run as root) that starts a daemonized tunneld on `port`. + * HOME is pinned to root's home so pymobiledevice3 finds its RemoteXPC pairing + * records: the privileged-exec environment (Authorization Services) doesn't + * inherit a usable HOME, which otherwise leaves tunneld unable to form the + * device tunnel (it starts but never registers one). sudo set HOME=/var/root + * implicitly; here we set it explicitly. + * + * The log goes under /var/root (root's home, mode 0700) rather than a predictable + * path in world-writable /tmp: this command runs as root, and a root `>` redirect + * to a /tmp path a non-root user can pre-symlink is a classic privileged-tmp + * clobber (CWE-59). + */ +export function tunneldStartCommand(pmd3Abs: string, port: number): string { + return `HOME=/var/root ${shellSingleQuote(pmd3Abs)} remote tunneld --port ${port} -d > /var/root/argent-coredevice-tunneld.log 2>&1`; +} + +/** + * Start `pymobiledevice3 remote tunneld` as root via the standard macOS + * authorization dialog (password / Touch ID where the OS offers it) — so users + * never run sudo by hand. AppleScript's `with administrator privileges` routes + * through Authorization Services and shows the system modal in the active GUI + * session; `-d` daemonizes so the privileged shell returns immediately, leaving + * tunneld running as root for the rest of the session. Throws if the user + * cancels or no GUI session is available (headless) — callers fall back to the + * manual `sudo` instructions. + */ +async function startTunneldWithPrivilege(pmd3Abs: string, port: number): Promise { + const shellCmd = tunneldStartCommand(pmd3Abs, port); + // Generous timeout: the user has to see and approve the modal. + const timeout = 120_000; + + // Preferred: the signed host helper, which shows the modal branded as + // "Argent" with the Argent icon + a clear message via Authorization Services. + const helper = deviceAuthHelperPath(); + if (helper) { + const icon = argentIconPath() ?? ""; + const prompt = "Argent needs administrator access to connect to a physical iOS device."; + try { + await execFileAsync(helper, [icon, prompt, "/bin/sh", "-c", shellCmd], { timeout }); + return; + } catch (err) { + // Exit 3 = the user explicitly cancelled the prompt — respect that and + // don't pop a second (osascript) prompt. Any other failure (a broken, + // unsigned, quarantined, or 0-byte helper binary) degrades to the + // osascript admin prompt below rather than hard-failing. + if ((err as { code?: unknown }).code === 3) throw err; + } + } + + // Fallback when the helper isn't installed (e.g. a dev tree without the signed + // binary), is unusable, or there's no GUI: the generic osascript admin prompt. + // Functional but unbranded ("osascript wants to make changes"). + const appleScript = `do shell script "${appleScriptQuote(shellCmd)}" with administrator privileges`; + await execFileAsync("osascript", ["-e", appleScript], { timeout }); +} + +// Dedupe concurrent escalation prompts: parallel interactions share one modal. +// Cleared after settle so a later call can retry if the user cancelled. +let tunnelStartInFlight: Promise | null = null; + +function tunnelHelp(udid: string, reason: string): string { + const port = tunneldPort(); + // Echo the custom port in the manual command — otherwise a user who set + // ARGENT_PMD3_TUNNELD_PORT would start tunneld on pmd3's default (49151) and + // discovery, which probes the custom port, would never find it. + const portFlag = port === 49151 ? "" : ` --port ${port}`; + return ( + `Physical iOS control needs a CoreDevice tunnel for ${udid}, but ${reason} ` + + `(checked tunneld at 127.0.0.1:${port}). Argent tries to start it automatically via the macOS ` + + `authorization prompt; if that was declined or no GUI session is available, start it manually ` + + `and leave it running:\n sudo pymobiledevice3 remote tunneld${portFlag}\n` + + `Also ensure the iPhone is on iOS 27+, unlocked, and trusted. ` + + `(Override the port with ARGENT_PMD3_TUNNELD_PORT.)` + ); +} + +/** + * Look up the device's RSD endpoint from a running `pymobiledevice3 remote + * tunneld` (its local REST API at 127.0.0.1:). Re-resolved per command so + * a tunneld restart mid-session is picked up without re-creating the service. + */ +async function resolveTunnel(udid: string): Promise { + const port = tunneldPort(); + let payload: Record>; + try { + const res = await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(4000) }); + // Something other than tunneld could be bound to the port; don't parse an + // error page as a tunnel list. + if (!res.ok) throw new Error(`tunneld responded HTTP ${res.status}`); + payload = (await res.json()) as typeof payload; + } catch (err) { + throw new FailureError( + tunnelHelp(udid, "tunneld is not running"), + { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_UNREACHABLE, + failure_stage: "core_device_resolve_tunnel", + failure_area: "tool_server", + error_kind: "network", + failure_command: "pymobiledevice3", + }, + { cause: err instanceof Error ? err : new Error(String(err)) } + ); + } + const entry = payload?.[udid]; + const t = Array.isArray(entry) ? entry[0] : undefined; + // `!t["tunnel-port"]` (not `== null`) deliberately also rejects 0 — tunneld + // never assigns port 0 to a real tunnel, so treat it the same as missing, + // consistent with the tunnel-address check on the left. + if (!t?.["tunnel-address"] || !t["tunnel-port"]) { + throw new FailureError(tunnelHelp(udid, "no active tunnel is registered for it"), { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_NOT_REGISTERED, + failure_stage: "core_device_resolve_tunnel", + failure_area: "tool_server", + error_kind: "not_found", + failure_command: "pymobiledevice3", + }); + } + return { address: String(t["tunnel-address"]), port: Number(t["tunnel-port"]) }; +} + +/** + * Whether anything is serving on the tunneld port (any HTTP response counts). + * Distinguishes "tunneld not running → start it" from "tunneld running but this + * device's tunnel hasn't formed yet → just wait", so we don't pop a second, + * pointless root prompt (or fail trying to bind an already-used port). + */ +async function isTunneldReachable(port: number): Promise { + try { + await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(4000) }); + return true; + } catch { + return false; + } +} + +/** + * Return the device's RSD endpoint, auto-starting tunneld via the macOS auth + * modal when it isn't running — so the user never types sudo. Gated behind the + * physical-ios-devices flag (the privileged escalation is opt-in). If tunneld is + * already running but this device's tunnel hasn't registered (e.g. the iPhone is + * locked), it waits rather than popping a second, pointless prompt. + */ +export async function ensureCoreDeviceTunnel(udid: string): Promise { + assertPhysicalIosEnabled(); + try { + return await resolveTunnel(udid); + } catch (notRunning) { + if (process.platform !== "darwin") throw notRunning; + const port = tunneldPort(); + + // Only escalate (root prompt) when tunneld isn't running at all. If it IS + // running, the device just needs to be unlocked/trusted for the handshake — + // don't re-prompt, just poll. + const reachable = await isTunneldReachable(port); + if (!reachable) { + if (!tunnelStartInFlight) { + tunnelStartInFlight = (async () => { + const pmd3Abs = await resolvePmd3Absolute(); + await startTunneldWithPrivilege(pmd3Abs, port); + })(); + } + try { + await tunnelStartInFlight; + } catch (escalationErr) { + tunnelStartInFlight = null; // allow a later retry + throw new FailureError( + tunnelHelp(udid, "the authorization prompt was cancelled or unavailable"), + { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_AUTH_DECLINED, + failure_stage: "core_device_ensure_tunnel_escalation", + failure_area: "tool_server", + error_kind: "subprocess", + failure_command: "pymobiledevice3", + }, + { + cause: + escalationErr instanceof Error ? escalationErr : new Error(String(escalationErr)), + } + ); + } + tunnelStartInFlight = null; + } + + // Poll for this device's tunnel to register (handshake needs it unlocked & trusted). + for (let i = 0; i < 15; i++) { + await sleep(2_000); + try { + return await resolveTunnel(udid); + } catch { + // keep polling + } + } + throw new FailureError( + tunnelHelp( + udid, + reachable + ? "tunneld is running but the device tunnel did not form — is the iPhone unlocked and trusted?" + : "the tunnel did not come up after starting tunneld" + ), + { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_TIMEOUT, + failure_stage: "core_device_ensure_tunnel_poll", + failure_area: "tool_server", + error_kind: "timeout", + failure_command: "pymobiledevice3", + }, + { cause: notRunning instanceof Error ? notRunning : new Error(String(notRunning)) } + ); + } +} + +/** + * Map a CoreDevice agent failure to a FailureError. The agent flags Apple's + * host-input gate (CoreDeviceError 9021) explicitly via `gated9021`: on iOS + * 18-26 the "remote control" services exist but reject touch input, while + * `screen-capture` and hardware buttons (`hid button`) keep working — so this + * message is scoped to tap/swipe, hardware-verified on an iPhone 15 + * (iOS 18.7.8 vs 27.0). + */ +export function agentError(label: string, err: unknown): FailureError { + const cause = err instanceof Error ? err : new Error(String(err)); + if (err instanceof CoreDeviceAgentError && err.gated9021) { + return new FailureError( + `CoreDevice ${label} failed: this iPhone is on an iOS below 27; host-driven touch input ` + + `(tap/swipe) requires iOS 27+. Screenshot and hardware buttons work on earlier iOS ` + + `versions (Apple CoreDeviceError 9021).`, + { + error_code: FAILURE_CODES.CORE_DEVICE_IOS_VERSION_TOO_OLD, + failure_stage: "core_device_command", + failure_area: "tool_server", + error_kind: "unsupported", + failure_command: "pymobiledevice3", + }, + { cause } + ); + } + return new FailureError( + `CoreDevice ${label} failed: ${cause.message.slice(0, 240)}`, + { + error_code: FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED, + failure_stage: "core_device_command", + failure_area: "tool_server", + error_kind: "subprocess", + failure_command: "pymobiledevice3", + }, + { cause } + ); +} + +/** Normalized 0..1 → the device's 0..65535 HID coordinate space. */ +export function toHid(v: number): number { + return Math.max(0, Math.min(65535, Math.round(v * 65535))); +} + +/** + * Derive the pmd3 `drag` parameters for a swipe of `durationMs`. A drag must + * dwell to register, so degenerate inputs are clamped: 0ms is dropped by iOS + * like a zero-dwell tap, and a negative value would reach pmd3 as a flag-like + * "-0.100" arg; both floor to 50ms. A pathological value is capped at 60s so it + * can't pin the device for minutes. The command timeout scales with the drag so + * a long swipe isn't SIGTERM-killed mid-gesture (pmd3 runs for ~`dur` ms; the + * +15s buffer covers the interpreter's startup). + */ +export function swipeDragParams(durationMs: number): { + steps: number; + seconds: string; + timeoutMs: number; +} { + const dur = Math.max(50, Math.min(60_000, Math.round(durationMs))); + const steps = Math.max(2, Math.min(60, Math.round(dur / 16))); + return { steps, seconds: (dur / 1000).toFixed(3), timeoutMs: dur + 15_000 }; +} + +/** + * Ensure the personalized DeveloperDiskImage is mounted (the CoreDevice services + * live in it). Idempotent: when already mounted, pymobiledevice3 exits non-zero + * with "already mounted", which we treat as success. CoreDevice usually mounts + * it automatically when the device connects, so this is a best-effort fallback — + * a genuine mount failure surfaces later as an actionable per-command error. + */ +async function ensureMounted(pmd3: string, rsd: Rsd): Promise { + try { + await execFileAsync(pmd3, ["mounter", "auto-mount", "--rsd", rsd.address, String(rsd.port)], { + timeout: 60_000, + }); + } catch { + // Best-effort: when the DDI is already mounted pymobiledevice3 exits non-zero + // with "already mounted"; a genuine mount failure surfaces later as an + // actionable per-command error. Either way, don't fail service creation here. + } +} + +export const coreDeviceBlueprint: ServiceBlueprint = { + namespace: CORE_DEVICE_NAMESPACE, + getURN(device: DeviceInfo) { + return `${CORE_DEVICE_NAMESPACE}:${device.id}`; + }, + async factory(_deps, _payload, options) { + const opts = options as unknown as CoreDeviceFactoryOptions | undefined; + const device = opts?.device; + if (!device?.id) { + throw new FailureError( + `${CORE_DEVICE_NAMESPACE}.factory requires a resolved DeviceInfo via options.device. ` + + `Use coreDeviceRef(device) when registering the service ref.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_FACTORY_OPTIONS_MISSING, + failure_stage: "core_device_factory_options", + failure_area: "tool_server", + error_kind: "validation", + } + ); + } + if (device.platform !== "ios" || device.kind !== "device") { + throw new FailureError( + `${CORE_DEVICE_NAMESPACE} only drives physical iOS devices; got ${device.platform}/${device.kind}.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_WRONG_DEVICE, + failure_stage: "core_device_factory_platform", + failure_area: "tool_server", + error_kind: "validation", + } + ); + } + // Gate first — before any setup probe — so a flag-disabled user gets the + // "enable the flag" message rather than an incidental "install pymobiledevice3" + // (ensureCoreDeviceTunnel re-checks this; the duplicate keeps the message + // correct regardless of whether pmd3 happens to be installed). + assertPhysicalIosEnabled(); + + const pmd3 = resolvePmd3(); + const udid = device.id; + + // Surface setup problems up front, in the order a user fixes them: install + // pymobiledevice3, then ensure the tunnel (auto-started via the macOS auth + // modal if needed — no manual sudo), then mount the DDI. + await verifyPmd3Available(pmd3); + const rsd = await ensureCoreDeviceTunnel(udid); + await ensureMounted(pmd3, rsd); + + // Resolve the interpreter that has pymobiledevice3 importable (the CLI's own + // venv python) and materialize the agent program, then start ONE long-lived + // process for this device: it connects the RSD tunnel and opens the HID / + // screenshot services once, so each interaction is a socket write instead of + // a fresh ~0.8s Python cold-start (~0.5s of which is just the pmd3 import). + const python = await resolvePmd3Python(await resolvePmd3Absolute()); + const scriptPath = await materializeAgentScript(); + const agent = new CoreDeviceAgent(python, scriptPath, udid, tunneldPort()); + await agent.start(); + + const call = async ( + label: string, + op: string, + args: Record = {}, + timeoutMs = 30_000 + ): Promise> => { + try { + return (await agent.request(op, args, timeoutMs)) as Record; + } catch (err) { + throw agentError(label, err); + } + }; + + const events = new TypedEventEmitter(); + + const api: CoreDeviceApi = { + async screenshot() { + const res = await call("screenshot", "screenshot", {}, 30_000); + const b64 = typeof res.image_b64 === "string" ? res.image_b64 : ""; + const path = join(tmpdir(), `argent-ios-shot-${randomUUID()}.png`); + await writeFile(path, Buffer.from(b64, "base64")); + return { path }; + }, + async tap(x, y) { + // A zero-duration tap is dropped by iOS; the agent emits a short held + // dwell-drag. We hand it the 0..65535 HID coordinate. + await call("tap", "tap", { x: toHid(x), y: toHid(y) }, 15_000); + }, + async swipe(fromX, fromY, toX, toY, durationMs) { + const { steps, seconds, timeoutMs } = swipeDragParams(durationMs); + await call( + "swipe", + "swipe", + { + x1: toHid(fromX), + y1: toHid(fromY), + x2: toHid(toX), + y2: toHid(toY), + steps, + duration: Number(seconds), + }, + timeoutMs + ); + }, + async button(name) { + await call("button", "button", { name }, 15_000); + }, + async axtree() { + // The audit walk + rect pass runs on-device; give it room. + const res = await call("axtree", "axtree", {}, 45_000); + return { + elements: (res.elements as CoreDeviceAxTree["elements"]) ?? [], + screen: res.screen as CoreDeviceAxTree["screen"], + }; + }, + }; + + const instance: ServiceInstance = { + api, + // One persistent pymobiledevice3 process per device — tear it down. + dispose: async () => { + agent.dispose(); + }, + events, + }; + return instance; + }, +}; diff --git a/packages/tool-server/src/blueprints/coredevice-agent.py b/packages/tool-server/src/blueprints/coredevice-agent.py new file mode 100644 index 000000000..974e15b11 --- /dev/null +++ b/packages/tool-server/src/blueprints/coredevice-agent.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Persistent CoreDevice agent — one long-lived process per physical iPhone. + +Replaces per-call `pymobiledevice3` CLI spawns (each ~0.8s, ~0.5s just the +Python import) with a single process that connects the RSD tunnel once, holds +the touchscreen media-stream session + screenshot service open, and executes +newline-delimited JSON commands on stdin, replying with one JSON line each. + +Reuses pymobiledevice3's own CLI helpers so behaviour is identical to the +`developer core-device …` commands (dwell-drag tap, mainTouchscreen reports, +Indigo hardware buttons, screen-capture PNG). + +Protocol (one JSON object per line): + <- {"udid": "...", "port": 49151} (argv, not stdin) + -> {"ready": true} (or {"ready": false, "error": "..."}) + <- {"id": 1, "op": "screenshot"} + -> {"id": 1, "ok": true, "image_b64": "..."} + <- {"id": 2, "op": "tap", "x": 32768, "y": 20000} (x/y already 0..65535) + <- {"id": 3, "op": "swipe", "x1":.., "y1":.., "x2":.., "y2":.., "steps":19, "duration":0.3} + <- {"id": 4, "op": "button", "name": "home"} + <- {"id": 5, "op": "homescreen"} (springboard icon grid) + -> {"id": N, "ok": true, ...} | {"id": N, "error": "...", "gated_9021": bool} +""" +import asyncio +import base64 +import contextlib +import json +import sys + +from pymobiledevice3.remote.remote_service_discovery import RemoteServiceDiscoveryService +from pymobiledevice3.remote.core_device.hid_service import ( + touch_session, + IndigoHIDService, + DIGITIZER_SURFACE_MAIN_TOUCHSCREEN, +) +from pymobiledevice3.remote.core_device.screen_capture_service import ScreenCaptureService +from pymobiledevice3.services.springboard import SpringBoardServicesService +from pymobiledevice3.cli.developer.core_device import _do_drag, _send_button_press, _NAMED_BUTTONS +from pymobiledevice3.dtx_service_provider import DtxServiceProvider +from pymobiledevice3.dtx.connection import DTXConnection +from pymobiledevice3.services.accessibilityaudit import AccessibilityAudit, deserialize_object + +import urllib.request + + +# --- RSDCheckin fix for the iOS-26+ accessibility (axAudit) DTX service -------- +# In iOS 26 Apple re-plumbed axauditd onto RemoteServiceDiscovery; the +# `…axAuditDaemon.remoteserver.shim.remote` DTX daemon now requires the RSDCheckin +# handshake before it accepts DTX framing. pymobiledevice3's DtxServiceProvider +# opens RSD services with a raw connect (create_service_connection) and skips +# RSDCheckin, so iOS 26/27 drops the connection on the first byte. start_lockdown_service +# performs RSDCheckin — route RSD DTX opens through it. (ref: littledivy/iphone-mirroring-axdump) +_orig_open_dtx = DtxServiceProvider._open_dtx_connection + + +async def _open_dtx_with_checkin(self, service_name, *, strip_ssl=False): + if isinstance(self.lockdown, RemoteServiceDiscoveryService): + svc = await self.lockdown.start_lockdown_service(service_name) + return DTXConnection(svc.reader, svc.writer) + return await _orig_open_dtx(self, service_name, strip_ssl=strip_ssl) + + +DtxServiceProvider._open_dtx_connection = _open_dtx_with_checkin + + +def _resolve_rsd(udid: str, port: int): + payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=4)) + entry = payload.get(udid) or [] + t = entry[0] if entry else {} + addr, tport = t.get("tunnel-address"), t.get("tunnel-port") + if not addr or not tport: + raise RuntimeError(f"no active tunnel registered for {udid} on tunneld :{port}") + return addr, int(tport) + + +async def _maybe_await(v): + return await v if asyncio.iscoroutine(v) else v + + +class Agent: + def __init__(self, udid: str, port: int): + self.udid = udid + self.port = port + self.rsd = None + self.stack = contextlib.AsyncExitStack() + self.touch = None # UniversalHIDServiceService, held open (media stream stays warm) + + async def connect(self): + addr, tport = _resolve_rsd(self.udid, self.port) + self.rsd = RemoteServiceDiscoveryService((addr, tport)) + await self.rsd.connect() + + async def _ensure_touch(self): + # Lazily open (and keep) the touchscreen media-stream session — the auth + # gate backboardd needs for injected touches. Kept warm so taps don't pay + # the media-stream startup each time. + if self.touch is None: + self.touch = await self.stack.enter_async_context(touch_session(self.rsd)) + return self.touch + + async def op_screenshot(self, _): + # ScreenCaptureService delivers one PNG per open (the stream ends after + # the frame), so open a fresh one each call — cheap now the interpreter + # and tunnel are already warm. + async with ScreenCaptureService(self.rsd) as screen: + resp = await screen.capture_screenshot() + return {"image_b64": base64.b64encode(resp["image"]).decode("ascii")} + + async def op_tap(self, msg): + svc = await self._ensure_touch() + x, y = int(msg["x"]), int(msg["y"]) + # Zero-dwell taps are dropped by iOS; emit a short held drag with a tiny + # move away from the edge (mirrors core-device.ts / the CLI drag path). + y2 = y + 96 if y <= 65535 - 120 else y - 96 + await _do_drag(svc, x, y, x, y2, 3, 0.15, tsid=DIGITIZER_SURFACE_MAIN_TOUCHSCREEN) + return {} + + async def op_swipe(self, msg): + svc = await self._ensure_touch() + await _do_drag( + svc, int(msg["x1"]), int(msg["y1"]), int(msg["x2"]), int(msg["y2"]), + int(msg.get("steps", 19)), float(msg.get("duration", 0.3)), + tsid=DIGITIZER_SURFACE_MAIN_TOUCHSCREEN, + ) + return {} + + async def op_button(self, msg): + name = msg["name"] + if name not in _NAMED_BUTTONS: + raise RuntimeError(f"unknown button '{name}'") + usage_page, usage_code, hold = _NAMED_BUTTONS[name] + async with IndigoHIDService(self.rsd) as svc: + await _send_button_press(svc, usage_page, usage_code, "press", hold) + return {} + + async def op_homescreen(self, _): + sb = SpringBoardServicesService(lockdown=self.rsd) + icons = await _maybe_await(sb.get_icon_state()) + metrics = await _maybe_await(sb.get_homescreen_icon_metrics()) + return {"icon_state": icons, "metrics": metrics} + + async def op_axtree(self, msg): + """The on-screen accessibility tree of whatever app is frontmost (or the + home screen), via the iOS-26+ axAudit service (RSDCheckin-unlocked above). + + Returns each element's accessible caption (label + value + traits) in + VoiceOver reading order plus a stable element id, and — where the + accessibility audit reports one — its on-screen rect in points. Per-element + geometry isn't exposed on hardware, so only audited elements carry a rect; + the TS layer fills the gaps. Screen size (points) comes from SpringBoard. + """ + limit = int(msg.get("limit", 120)) + + # 1) Walk the elements (caption + reading order + element id). + walk = AccessibilityAudit(self.rsd) + elements = [] + try: + async for el in walk.iter_elements(): + elements.append( + {"caption": el.caption, "id": el.element.identifier.hex()} + ) + if len(elements) >= limit: + break + finally: + with contextlib.suppress(Exception): + await walk.close() + + # 2) Audit for rects, keyed by element id (the only hardware frame source). + rects = {} + audit = AccessibilityAudit(self.rsd) + try: + types = await audit.supported_audits_types() + await audit._ensure_ready() + await audit._invoke("deviceBeginAuditTypes:", types, expects_reply=False) + while True: + name, args = await audit._event_queue.get() + if name != "hostDeviceDidCompleteAuditCategoriesWithAuditIssues:": + continue + issues = deserialize_object(audit._extract_event_payload(args)) + # iOS 27 returns the AXAuditIssue list directly; older wrap it in [{"value":…}]. + if ( + isinstance(issues, list) + and issues + and isinstance(issues[0], dict) + and "value" in issues[0] + ): + issues = issues[0]["value"] + for iss in issues or []: + try: + eid = iss._fields["AuditElementValue_v1"]._fields[ + "PlatformElementValue_v1" + ].hex() + rect = iss._fields.get("ElementRectValue_v1") + if eid and rect and eid not in rects: + rects[eid] = rect + except Exception: + continue + break + except Exception: + pass # audit is best-effort; the tree still returns without rects + finally: + with contextlib.suppress(Exception): + await audit.close() + + for el in elements: + if el["id"] in rects: + el["rect"] = rects[el["id"]] + + # 3) Screen size in points, for normalizing the rects. + screen = None + with contextlib.suppress(Exception): + sb = SpringBoardServicesService(lockdown=self.rsd) + m = await _maybe_await(sb.get_homescreen_icon_metrics()) + screen = {"w": m.get("homeScreenWidth"), "h": m.get("homeScreenHeight")} + + return {"elements": elements, "screen": screen} + + async def op_ping(self, _): + return {"pong": True} + + async def dispatch(self, msg): + op = msg.get("op") + fn = getattr(self, f"op_{op}", None) + if fn is None: + raise RuntimeError(f"unknown op '{op}'") + return await fn(msg) + + async def close(self): + with contextlib.suppress(Exception): + await self.stack.aclose() + if self.rsd is not None: + with contextlib.suppress(Exception): + await self.rsd.close() + + +def _is_9021(text: str) -> bool: + import re + return bool(re.search(r"core\s*device\s*error\W*9021", text, re.I) or re.search(r"\b9021\b", text)) + + +async def main(): + udid = sys.argv[1] + port = int(sys.argv[2]) if len(sys.argv) > 2 else 49151 + agent = Agent(udid, port) + out = sys.stdout + + def emit(obj): + # default=str keeps a stray plist type (bytes/datetime in springboard + # icon state) from crashing serialization mid-session. + out.write(json.dumps(obj, default=str) + "\n") + out.flush() + + try: + await agent.connect() + except Exception as e: # noqa: BLE001 + emit({"ready": False, "error": str(e)}) + return + emit({"ready": True}) + + loop = asyncio.get_event_loop() + reader = asyncio.StreamReader() + await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(reader), sys.stdin) + + while True: + line = await reader.readline() + if not line: + break + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: # noqa: BLE001 + emit({"error": "bad json"}) + continue + mid = msg.get("id") + try: + result = await agent.dispatch(msg) + emit({"id": mid, "ok": True, **result}) + except Exception as e: # noqa: BLE001 + text = f"{type(e).__name__}: {e}" + emit({"id": mid, "error": text, "gated_9021": _is_9021(text)}) + + await agent.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/packages/tool-server/src/blueprints/coredevice-agent.ts b/packages/tool-server/src/blueprints/coredevice-agent.ts new file mode 100644 index 000000000..b44260bdd --- /dev/null +++ b/packages/tool-server/src/blueprints/coredevice-agent.ts @@ -0,0 +1,252 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { createHash } from "node:crypto"; +import { createInterface, type Interface } from "node:readline"; + +/** + * The persistent CoreDevice agent (a small pymobiledevice3 program) that a + * physical iPhone is driven through. It is base64-embedded rather than shipped + * as a loose `.py` so it survives esbuild bundling with zero build-pipeline + * wiring, and base64 (not a template literal) so its Python `\s`/`\n` escapes + * can't be mangled by JS string parsing. + * + * Source of truth: `coredevice-agent.py.b64` sits next to this file; regenerate + * with `base64 -i coredevice-agent.py | tr -d '\n'`. The decoded script speaks + * newline-delimited JSON on stdio (see AGENT_PROTOCOL below). + */ +const AGENT_SCRIPT_B64 = + "IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwoiIiJQZXJzaXN0ZW50IENvcmVEZXZpY2UgYWdlbnQg4oCUIG9uZSBsb25nLWxpdmVkIHByb2Nlc3MgcGVyIHBoeXNpY2FsIGlQaG9uZS4KClJlcGxhY2VzIHBlci1jYWxsIGBweW1vYmlsZWRldmljZTNgIENMSSBzcGF3bnMgKGVhY2ggfjAuOHMsIH4wLjVzIGp1c3QgdGhlClB5dGhvbiBpbXBvcnQpIHdpdGggYSBzaW5nbGUgcHJvY2VzcyB0aGF0IGNvbm5lY3RzIHRoZSBSU0QgdHVubmVsIG9uY2UsIGhvbGRzCnRoZSB0b3VjaHNjcmVlbiBtZWRpYS1zdHJlYW0gc2Vzc2lvbiArIHNjcmVlbnNob3Qgc2VydmljZSBvcGVuLCBhbmQgZXhlY3V0ZXMKbmV3bGluZS1kZWxpbWl0ZWQgSlNPTiBjb21tYW5kcyBvbiBzdGRpbiwgcmVwbHlpbmcgd2l0aCBvbmUgSlNPTiBsaW5lIGVhY2guCgpSZXVzZXMgcHltb2JpbGVkZXZpY2UzJ3Mgb3duIENMSSBoZWxwZXJzIHNvIGJlaGF2aW91ciBpcyBpZGVudGljYWwgdG8gdGhlCmBkZXZlbG9wZXIgY29yZS1kZXZpY2Ug4oCmYCBjb21tYW5kcyAoZHdlbGwtZHJhZyB0YXAsIG1haW5Ub3VjaHNjcmVlbiByZXBvcnRzLApJbmRpZ28gaGFyZHdhcmUgYnV0dG9ucywgc2NyZWVuLWNhcHR1cmUgUE5HKS4KClByb3RvY29sIChvbmUgSlNPTiBvYmplY3QgcGVyIGxpbmUpOgogIDwtIHsidWRpZCI6ICIuLi4iLCAicG9ydCI6IDQ5MTUxfSAgICAgICAgICAgICAgICAgKGFyZ3YsIG5vdCBzdGRpbikKICAtPiB7InJlYWR5IjogdHJ1ZX0gICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAob3IgeyJyZWFkeSI6IGZhbHNlLCAiZXJyb3IiOiAiLi4uIn0pCiAgPC0geyJpZCI6IDEsICJvcCI6ICJzY3JlZW5zaG90In0KICAtPiB7ImlkIjogMSwgIm9rIjogdHJ1ZSwgImltYWdlX2I2NCI6ICIuLi4ifQogIDwtIHsiaWQiOiAyLCAib3AiOiAidGFwIiwgIngiOiAzMjc2OCwgInkiOiAyMDAwMH0gICAgICAgICAgKHgveSBhbHJlYWR5IDAuLjY1NTM1KQogIDwtIHsiaWQiOiAzLCAib3AiOiAic3dpcGUiLCAieDEiOi4uLCAieTEiOi4uLCAieDIiOi4uLCAieTIiOi4uLCAic3RlcHMiOjE5LCAiZHVyYXRpb24iOjAuM30KICA8LSB7ImlkIjogNCwgIm9wIjogImJ1dHRvbiIsICJuYW1lIjogImhvbWUifQogIDwtIHsiaWQiOiA1LCAib3AiOiAiaG9tZXNjcmVlbiJ9ICAgICAgICAgICAgICAgICAgICAgICAgICAgIChzcHJpbmdib2FyZCBpY29uIGdyaWQpCiAgLT4geyJpZCI6IE4sICJvayI6IHRydWUsIC4uLn0gIHwgIHsiaWQiOiBOLCAiZXJyb3IiOiAiLi4uIiwgImdhdGVkXzkwMjEiOiBib29sfQoiIiIKaW1wb3J0IGFzeW5jaW8KaW1wb3J0IGJhc2U2NAppbXBvcnQgY29udGV4dGxpYgppbXBvcnQganNvbgppbXBvcnQgc3lzCgpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUucmVtb3RlX3NlcnZpY2VfZGlzY292ZXJ5IGltcG9ydCBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2UuaGlkX3NlcnZpY2UgaW1wb3J0ICgKICAgIHRvdWNoX3Nlc3Npb24sCiAgICBJbmRpZ29ISURTZXJ2aWNlLAogICAgRElHSVRJWkVSX1NVUkZBQ0VfTUFJTl9UT1VDSFNDUkVFTiwKKQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2Uuc2NyZWVuX2NhcHR1cmVfc2VydmljZSBpbXBvcnQgU2NyZWVuQ2FwdHVyZVNlcnZpY2UKZnJvbSBweW1vYmlsZWRldmljZTMuc2VydmljZXMuc3ByaW5nYm9hcmQgaW1wb3J0IFNwcmluZ0JvYXJkU2VydmljZXNTZXJ2aWNlCmZyb20gcHltb2JpbGVkZXZpY2UzLmNsaS5kZXZlbG9wZXIuY29yZV9kZXZpY2UgaW1wb3J0IF9kb19kcmFnLCBfc2VuZF9idXR0b25fcHJlc3MsIF9OQU1FRF9CVVRUT05TCmZyb20gcHltb2JpbGVkZXZpY2UzLmR0eF9zZXJ2aWNlX3Byb3ZpZGVyIGltcG9ydCBEdHhTZXJ2aWNlUHJvdmlkZXIKZnJvbSBweW1vYmlsZWRldmljZTMuZHR4LmNvbm5lY3Rpb24gaW1wb3J0IERUWENvbm5lY3Rpb24KZnJvbSBweW1vYmlsZWRldmljZTMuc2VydmljZXMuYWNjZXNzaWJpbGl0eWF1ZGl0IGltcG9ydCBBY2Nlc3NpYmlsaXR5QXVkaXQsIGRlc2VyaWFsaXplX29iamVjdAoKaW1wb3J0IHVybGxpYi5yZXF1ZXN0CgoKIyAtLS0gUlNEQ2hlY2tpbiBmaXggZm9yIHRoZSBpT1MtMjYrIGFjY2Vzc2liaWxpdHkgKGF4QXVkaXQpIERUWCBzZXJ2aWNlIC0tLS0tLS0tCiMgSW4gaU9TIDI2IEFwcGxlIHJlLXBsdW1iZWQgYXhhdWRpdGQgb250byBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5OyB0aGUKIyBg4oCmYXhBdWRpdERhZW1vbi5yZW1vdGVzZXJ2ZXIuc2hpbS5yZW1vdGVgIERUWCBkYWVtb24gbm93IHJlcXVpcmVzIHRoZSBSU0RDaGVja2luCiMgaGFuZHNoYWtlIGJlZm9yZSBpdCBhY2NlcHRzIERUWCBmcmFtaW5nLiBweW1vYmlsZWRldmljZTMncyBEdHhTZXJ2aWNlUHJvdmlkZXIKIyBvcGVucyBSU0Qgc2VydmljZXMgd2l0aCBhIHJhdyBjb25uZWN0IChjcmVhdGVfc2VydmljZV9jb25uZWN0aW9uKSBhbmQgc2tpcHMKIyBSU0RDaGVja2luLCBzbyBpT1MgMjYvMjcgZHJvcHMgdGhlIGNvbm5lY3Rpb24gb24gdGhlIGZpcnN0IGJ5dGUuIHN0YXJ0X2xvY2tkb3duX3NlcnZpY2UKIyBwZXJmb3JtcyBSU0RDaGVja2luIOKAlCByb3V0ZSBSU0QgRFRYIG9wZW5zIHRocm91Z2ggaXQuIChyZWY6IGxpdHRsZWRpdnkvaXBob25lLW1pcnJvcmluZy1heGR1bXApCl9vcmlnX29wZW5fZHR4ID0gRHR4U2VydmljZVByb3ZpZGVyLl9vcGVuX2R0eF9jb25uZWN0aW9uCgoKYXN5bmMgZGVmIF9vcGVuX2R0eF93aXRoX2NoZWNraW4oc2VsZiwgc2VydmljZV9uYW1lLCAqLCBzdHJpcF9zc2w9RmFsc2UpOgogICAgaWYgaXNpbnN0YW5jZShzZWxmLmxvY2tkb3duLCBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZSk6CiAgICAgICAgc3ZjID0gYXdhaXQgc2VsZi5sb2NrZG93bi5zdGFydF9sb2NrZG93bl9zZXJ2aWNlKHNlcnZpY2VfbmFtZSkKICAgICAgICByZXR1cm4gRFRYQ29ubmVjdGlvbihzdmMucmVhZGVyLCBzdmMud3JpdGVyKQogICAgcmV0dXJuIGF3YWl0IF9vcmlnX29wZW5fZHR4KHNlbGYsIHNlcnZpY2VfbmFtZSwgc3RyaXBfc3NsPXN0cmlwX3NzbCkKCgpEdHhTZXJ2aWNlUHJvdmlkZXIuX29wZW5fZHR4X2Nvbm5lY3Rpb24gPSBfb3Blbl9kdHhfd2l0aF9jaGVja2luCgoKZGVmIF9yZXNvbHZlX3JzZCh1ZGlkOiBzdHIsIHBvcnQ6IGludCk6CiAgICBwYXlsb2FkID0ganNvbi5sb2FkKHVybGxpYi5yZXF1ZXN0LnVybG9wZW4oZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fS8iLCB0aW1lb3V0PTQpKQogICAgZW50cnkgPSBwYXlsb2FkLmdldCh1ZGlkKSBvciBbXQogICAgdCA9IGVudHJ5WzBdIGlmIGVudHJ5IGVsc2Uge30KICAgIGFkZHIsIHRwb3J0ID0gdC5nZXQoInR1bm5lbC1hZGRyZXNzIiksIHQuZ2V0KCJ0dW5uZWwtcG9ydCIpCiAgICBpZiBub3QgYWRkciBvciBub3QgdHBvcnQ6CiAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKGYibm8gYWN0aXZlIHR1bm5lbCByZWdpc3RlcmVkIGZvciB7dWRpZH0gb24gdHVubmVsZCA6e3BvcnR9IikKICAgIHJldHVybiBhZGRyLCBpbnQodHBvcnQpCgoKYXN5bmMgZGVmIF9tYXliZV9hd2FpdCh2KToKICAgIHJldHVybiBhd2FpdCB2IGlmIGFzeW5jaW8uaXNjb3JvdXRpbmUodikgZWxzZSB2CgoKY2xhc3MgQWdlbnQ6CiAgICBkZWYgX19pbml0X18oc2VsZiwgdWRpZDogc3RyLCBwb3J0OiBpbnQpOgogICAgICAgIHNlbGYudWRpZCA9IHVkaWQKICAgICAgICBzZWxmLnBvcnQgPSBwb3J0CiAgICAgICAgc2VsZi5yc2QgPSBOb25lCiAgICAgICAgc2VsZi5zdGFjayA9IGNvbnRleHRsaWIuQXN5bmNFeGl0U3RhY2soKQogICAgICAgIHNlbGYudG91Y2ggPSBOb25lICAjIFVuaXZlcnNhbEhJRFNlcnZpY2VTZXJ2aWNlLCBoZWxkIG9wZW4gKG1lZGlhIHN0cmVhbSBzdGF5cyB3YXJtKQoKICAgIGFzeW5jIGRlZiBjb25uZWN0KHNlbGYpOgogICAgICAgIGFkZHIsIHRwb3J0ID0gX3Jlc29sdmVfcnNkKHNlbGYudWRpZCwgc2VsZi5wb3J0KQogICAgICAgIHNlbGYucnNkID0gUmVtb3RlU2VydmljZURpc2NvdmVyeVNlcnZpY2UoKGFkZHIsIHRwb3J0KSkKICAgICAgICBhd2FpdCBzZWxmLnJzZC5jb25uZWN0KCkKCiAgICBhc3luYyBkZWYgX2Vuc3VyZV90b3VjaChzZWxmKToKICAgICAgICAjIExhemlseSBvcGVuIChhbmQga2VlcCkgdGhlIHRvdWNoc2NyZWVuIG1lZGlhLXN0cmVhbSBzZXNzaW9uIOKAlCB0aGUgYXV0aAogICAgICAgICMgZ2F0ZSBiYWNrYm9hcmRkIG5lZWRzIGZvciBpbmplY3RlZCB0b3VjaGVzLiBLZXB0IHdhcm0gc28gdGFwcyBkb24ndCBwYXkKICAgICAgICAjIHRoZSBtZWRpYS1zdHJlYW0gc3RhcnR1cCBlYWNoIHRpbWUuCiAgICAgICAgaWYgc2VsZi50b3VjaCBpcyBOb25lOgogICAgICAgICAgICBzZWxmLnRvdWNoID0gYXdhaXQgc2VsZi5zdGFjay5lbnRlcl9hc3luY19jb250ZXh0KHRvdWNoX3Nlc3Npb24oc2VsZi5yc2QpKQogICAgICAgIHJldHVybiBzZWxmLnRvdWNoCgogICAgYXN5bmMgZGVmIG9wX3NjcmVlbnNob3Qoc2VsZiwgXyk6CiAgICAgICAgIyBTY3JlZW5DYXB0dXJlU2VydmljZSBkZWxpdmVycyBvbmUgUE5HIHBlciBvcGVuICh0aGUgc3RyZWFtIGVuZHMgYWZ0ZXIKICAgICAgICAjIHRoZSBmcmFtZSksIHNvIG9wZW4gYSBmcmVzaCBvbmUgZWFjaCBjYWxsIOKAlCBjaGVhcCBub3cgdGhlIGludGVycHJldGVyCiAgICAgICAgIyBhbmQgdHVubmVsIGFyZSBhbHJlYWR5IHdhcm0uCiAgICAgICAgYXN5bmMgd2l0aCBTY3JlZW5DYXB0dXJlU2VydmljZShzZWxmLnJzZCkgYXMgc2NyZWVuOgogICAgICAgICAgICByZXNwID0gYXdhaXQgc2NyZWVuLmNhcHR1cmVfc2NyZWVuc2hvdCgpCiAgICAgICAgcmV0dXJuIHsiaW1hZ2VfYjY0IjogYmFzZTY0LmI2NGVuY29kZShyZXNwWyJpbWFnZSJdKS5kZWNvZGUoImFzY2lpIil9CgogICAgYXN5bmMgZGVmIG9wX3RhcChzZWxmLCBtc2cpOgogICAgICAgIHN2YyA9IGF3YWl0IHNlbGYuX2Vuc3VyZV90b3VjaCgpCiAgICAgICAgeCwgeSA9IGludChtc2dbIngiXSksIGludChtc2dbInkiXSkKICAgICAgICAjIFplcm8tZHdlbGwgdGFwcyBhcmUgZHJvcHBlZCBieSBpT1M7IGVtaXQgYSBzaG9ydCBoZWxkIGRyYWcgd2l0aCBhIHRpbnkKICAgICAgICAjIG1vdmUgYXdheSBmcm9tIHRoZSBlZGdlIChtaXJyb3JzIGNvcmUtZGV2aWNlLnRzIC8gdGhlIENMSSBkcmFnIHBhdGgpLgogICAgICAgIHkyID0geSArIDk2IGlmIHkgPD0gNjU1MzUgLSAxMjAgZWxzZSB5IC0gOTYKICAgICAgICBhd2FpdCBfZG9fZHJhZyhzdmMsIHgsIHksIHgsIHkyLCAzLCAwLjE1LCB0c2lkPURJR0lUSVpFUl9TVVJGQUNFX01BSU5fVE9VQ0hTQ1JFRU4pCiAgICAgICAgcmV0dXJuIHt9CgogICAgYXN5bmMgZGVmIG9wX3N3aXBlKHNlbGYsIG1zZyk6CiAgICAgICAgc3ZjID0gYXdhaXQgc2VsZi5fZW5zdXJlX3RvdWNoKCkKICAgICAgICBhd2FpdCBfZG9fZHJhZygKICAgICAgICAgICAgc3ZjLCBpbnQobXNnWyJ4MSJdKSwgaW50KG1zZ1sieTEiXSksIGludChtc2dbIngyIl0pLCBpbnQobXNnWyJ5MiJdKSwKICAgICAgICAgICAgaW50KG1zZy5nZXQoInN0ZXBzIiwgMTkpKSwgZmxvYXQobXNnLmdldCgiZHVyYXRpb24iLCAwLjMpKSwKICAgICAgICAgICAgdHNpZD1ESUdJVElaRVJfU1VSRkFDRV9NQUlOX1RPVUNIU0NSRUVOLAogICAgICAgICkKICAgICAgICByZXR1cm4ge30KCiAgICBhc3luYyBkZWYgb3BfYnV0dG9uKHNlbGYsIG1zZyk6CiAgICAgICAgbmFtZSA9IG1zZ1sibmFtZSJdCiAgICAgICAgaWYgbmFtZSBub3QgaW4gX05BTUVEX0JVVFRPTlM6CiAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcihmInVua25vd24gYnV0dG9uICd7bmFtZX0nIikKICAgICAgICB1c2FnZV9wYWdlLCB1c2FnZV9jb2RlLCBob2xkID0gX05BTUVEX0JVVFRPTlNbbmFtZV0KICAgICAgICBhc3luYyB3aXRoIEluZGlnb0hJRFNlcnZpY2Uoc2VsZi5yc2QpIGFzIHN2YzoKICAgICAgICAgICAgYXdhaXQgX3NlbmRfYnV0dG9uX3ByZXNzKHN2YywgdXNhZ2VfcGFnZSwgdXNhZ2VfY29kZSwgInByZXNzIiwgaG9sZCkKICAgICAgICByZXR1cm4ge30KCiAgICBhc3luYyBkZWYgb3BfaG9tZXNjcmVlbihzZWxmLCBfKToKICAgICAgICBzYiA9IFNwcmluZ0JvYXJkU2VydmljZXNTZXJ2aWNlKGxvY2tkb3duPXNlbGYucnNkKQogICAgICAgIGljb25zID0gYXdhaXQgX21heWJlX2F3YWl0KHNiLmdldF9pY29uX3N0YXRlKCkpCiAgICAgICAgbWV0cmljcyA9IGF3YWl0IF9tYXliZV9hd2FpdChzYi5nZXRfaG9tZXNjcmVlbl9pY29uX21ldHJpY3MoKSkKICAgICAgICByZXR1cm4geyJpY29uX3N0YXRlIjogaWNvbnMsICJtZXRyaWNzIjogbWV0cmljc30KCiAgICBhc3luYyBkZWYgb3BfYXh0cmVlKHNlbGYsIG1zZyk6CiAgICAgICAgIiIiVGhlIG9uLXNjcmVlbiBhY2Nlc3NpYmlsaXR5IHRyZWUgb2Ygd2hhdGV2ZXIgYXBwIGlzIGZyb250bW9zdCAob3IgdGhlCiAgICAgICAgaG9tZSBzY3JlZW4pLCB2aWEgdGhlIGlPUy0yNisgYXhBdWRpdCBzZXJ2aWNlIChSU0RDaGVja2luLXVubG9ja2VkIGFib3ZlKS4KCiAgICAgICAgUmV0dXJucyBlYWNoIGVsZW1lbnQncyBhY2Nlc3NpYmxlIGNhcHRpb24gKGxhYmVsICsgdmFsdWUgKyB0cmFpdHMpIGluCiAgICAgICAgVm9pY2VPdmVyIHJlYWRpbmcgb3JkZXIgcGx1cyBhIHN0YWJsZSBlbGVtZW50IGlkLCBhbmQg4oCUIHdoZXJlIHRoZQogICAgICAgIGFjY2Vzc2liaWxpdHkgYXVkaXQgcmVwb3J0cyBvbmUg4oCUIGl0cyBvbi1zY3JlZW4gcmVjdCBpbiBwb2ludHMuIFBlci1lbGVtZW50CiAgICAgICAgZ2VvbWV0cnkgaXNuJ3QgZXhwb3NlZCBvbiBoYXJkd2FyZSwgc28gb25seSBhdWRpdGVkIGVsZW1lbnRzIGNhcnJ5IGEgcmVjdDsKICAgICAgICB0aGUgVFMgbGF5ZXIgZmlsbHMgdGhlIGdhcHMuIFNjcmVlbiBzaXplIChwb2ludHMpIGNvbWVzIGZyb20gU3ByaW5nQm9hcmQuCiAgICAgICAgIiIiCiAgICAgICAgbGltaXQgPSBpbnQobXNnLmdldCgibGltaXQiLCAxMjApKQoKICAgICAgICAjIDEpIFdhbGsgdGhlIGVsZW1lbnRzIChjYXB0aW9uICsgcmVhZGluZyBvcmRlciArIGVsZW1lbnQgaWQpLgogICAgICAgIHdhbGsgPSBBY2Nlc3NpYmlsaXR5QXVkaXQoc2VsZi5yc2QpCiAgICAgICAgZWxlbWVudHMgPSBbXQogICAgICAgIHRyeToKICAgICAgICAgICAgYXN5bmMgZm9yIGVsIGluIHdhbGsuaXRlcl9lbGVtZW50cygpOgogICAgICAgICAgICAgICAgZWxlbWVudHMuYXBwZW5kKAogICAgICAgICAgICAgICAgICAgIHsiY2FwdGlvbiI6IGVsLmNhcHRpb24sICJpZCI6IGVsLmVsZW1lbnQuaWRlbnRpZmllci5oZXgoKX0KICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGxlbihlbGVtZW50cykgPj0gbGltaXQ6CiAgICAgICAgICAgICAgICAgICAgYnJlYWsKICAgICAgICBmaW5hbGx5OgogICAgICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgICAgIGF3YWl0IHdhbGsuY2xvc2UoKQoKICAgICAgICAjIDIpIEF1ZGl0IGZvciByZWN0cywga2V5ZWQgYnkgZWxlbWVudCBpZCAodGhlIG9ubHkgaGFyZHdhcmUgZnJhbWUgc291cmNlKS4KICAgICAgICByZWN0cyA9IHt9CiAgICAgICAgYXVkaXQgPSBBY2Nlc3NpYmlsaXR5QXVkaXQoc2VsZi5yc2QpCiAgICAgICAgdHJ5OgogICAgICAgICAgICB0eXBlcyA9IGF3YWl0IGF1ZGl0LnN1cHBvcnRlZF9hdWRpdHNfdHlwZXMoKQogICAgICAgICAgICBhd2FpdCBhdWRpdC5fZW5zdXJlX3JlYWR5KCkKICAgICAgICAgICAgYXdhaXQgYXVkaXQuX2ludm9rZSgiZGV2aWNlQmVnaW5BdWRpdFR5cGVzOiIsIHR5cGVzLCBleHBlY3RzX3JlcGx5PUZhbHNlKQogICAgICAgICAgICB3aGlsZSBUcnVlOgogICAgICAgICAgICAgICAgbmFtZSwgYXJncyA9IGF3YWl0IGF1ZGl0Ll9ldmVudF9xdWV1ZS5nZXQoKQogICAgICAgICAgICAgICAgaWYgbmFtZSAhPSAiaG9zdERldmljZURpZENvbXBsZXRlQXVkaXRDYXRlZ29yaWVzV2l0aEF1ZGl0SXNzdWVzOiI6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlzc3VlcyA9IGRlc2VyaWFsaXplX29iamVjdChhdWRpdC5fZXh0cmFjdF9ldmVudF9wYXlsb2FkKGFyZ3MpKQogICAgICAgICAgICAgICAgIyBpT1MgMjcgcmV0dXJucyB0aGUgQVhBdWRpdElzc3VlIGxpc3QgZGlyZWN0bHk7IG9sZGVyIHdyYXAgaXQgaW4gW3sidmFsdWUiOuKApn1dLgogICAgICAgICAgICAgICAgaWYgKAogICAgICAgICAgICAgICAgICAgIGlzaW5zdGFuY2UoaXNzdWVzLCBsaXN0KQogICAgICAgICAgICAgICAgICAgIGFuZCBpc3N1ZXMKICAgICAgICAgICAgICAgICAgICBhbmQgaXNpbnN0YW5jZShpc3N1ZXNbMF0sIGRpY3QpCiAgICAgICAgICAgICAgICAgICAgYW5kICJ2YWx1ZSIgaW4gaXNzdWVzWzBdCiAgICAgICAgICAgICAgICApOgogICAgICAgICAgICAgICAgICAgIGlzc3VlcyA9IGlzc3Vlc1swXVsidmFsdWUiXQogICAgICAgICAgICAgICAgZm9yIGlzcyBpbiBpc3N1ZXMgb3IgW106CiAgICAgICAgICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgICAgICAgICBlaWQgPSBpc3MuX2ZpZWxkc1siQXVkaXRFbGVtZW50VmFsdWVfdjEiXS5fZmllbGRzWwogICAgICAgICAgICAgICAgICAgICAgICAgICAgIlBsYXRmb3JtRWxlbWVudFZhbHVlX3YxIgogICAgICAgICAgICAgICAgICAgICAgICBdLmhleCgpCiAgICAgICAgICAgICAgICAgICAgICAgIHJlY3QgPSBpc3MuX2ZpZWxkcy5nZXQoIkVsZW1lbnRSZWN0VmFsdWVfdjEiKQogICAgICAgICAgICAgICAgICAgICAgICBpZiBlaWQgYW5kIHJlY3QgYW5kIGVpZCBub3QgaW4gcmVjdHM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICByZWN0c1tlaWRdID0gcmVjdAogICAgICAgICAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgICAgICBicmVhawogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgIHBhc3MgICMgYXVkaXQgaXMgYmVzdC1lZmZvcnQ7IHRoZSB0cmVlIHN0aWxsIHJldHVybnMgd2l0aG91dCByZWN0cwogICAgICAgIGZpbmFsbHk6CiAgICAgICAgICAgIHdpdGggY29udGV4dGxpYi5zdXBwcmVzcyhFeGNlcHRpb24pOgogICAgICAgICAgICAgICAgYXdhaXQgYXVkaXQuY2xvc2UoKQoKICAgICAgICBmb3IgZWwgaW4gZWxlbWVudHM6CiAgICAgICAgICAgIGlmIGVsWyJpZCJdIGluIHJlY3RzOgogICAgICAgICAgICAgICAgZWxbInJlY3QiXSA9IHJlY3RzW2VsWyJpZCJdXQoKICAgICAgICAjIDMpIFNjcmVlbiBzaXplIGluIHBvaW50cywgZm9yIG5vcm1hbGl6aW5nIHRoZSByZWN0cy4KICAgICAgICBzY3JlZW4gPSBOb25lCiAgICAgICAgd2l0aCBjb250ZXh0bGliLnN1cHByZXNzKEV4Y2VwdGlvbik6CiAgICAgICAgICAgIHNiID0gU3ByaW5nQm9hcmRTZXJ2aWNlc1NlcnZpY2UobG9ja2Rvd249c2VsZi5yc2QpCiAgICAgICAgICAgIG0gPSBhd2FpdCBfbWF5YmVfYXdhaXQoc2IuZ2V0X2hvbWVzY3JlZW5faWNvbl9tZXRyaWNzKCkpCiAgICAgICAgICAgIHNjcmVlbiA9IHsidyI6IG0uZ2V0KCJob21lU2NyZWVuV2lkdGgiKSwgImgiOiBtLmdldCgiaG9tZVNjcmVlbkhlaWdodCIpfQoKICAgICAgICByZXR1cm4geyJlbGVtZW50cyI6IGVsZW1lbnRzLCAic2NyZWVuIjogc2NyZWVufQoKICAgIGFzeW5jIGRlZiBvcF9waW5nKHNlbGYsIF8pOgogICAgICAgIHJldHVybiB7InBvbmciOiBUcnVlfQoKICAgIGFzeW5jIGRlZiBkaXNwYXRjaChzZWxmLCBtc2cpOgogICAgICAgIG9wID0gbXNnLmdldCgib3AiKQogICAgICAgIGZuID0gZ2V0YXR0cihzZWxmLCBmIm9wX3tvcH0iLCBOb25lKQogICAgICAgIGlmIGZuIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcihmInVua25vd24gb3AgJ3tvcH0nIikKICAgICAgICByZXR1cm4gYXdhaXQgZm4obXNnKQoKICAgIGFzeW5jIGRlZiBjbG9zZShzZWxmKToKICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgYXdhaXQgc2VsZi5zdGFjay5hY2xvc2UoKQogICAgICAgIGlmIHNlbGYucnNkIGlzIG5vdCBOb25lOgogICAgICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgICAgIGF3YWl0IHNlbGYucnNkLmNsb3NlKCkKCgpkZWYgX2lzXzkwMjEodGV4dDogc3RyKSAtPiBib29sOgogICAgaW1wb3J0IHJlCiAgICByZXR1cm4gYm9vbChyZS5zZWFyY2gociJjb3JlXHMqZGV2aWNlXHMqZXJyb3JcVyo5MDIxIiwgdGV4dCwgcmUuSSkgb3IgcmUuc2VhcmNoKHIiXGI5MDIxXGIiLCB0ZXh0KSkKCgphc3luYyBkZWYgbWFpbigpOgogICAgdWRpZCA9IHN5cy5hcmd2WzFdCiAgICBwb3J0ID0gaW50KHN5cy5hcmd2WzJdKSBpZiBsZW4oc3lzLmFyZ3YpID4gMiBlbHNlIDQ5MTUxCiAgICBhZ2VudCA9IEFnZW50KHVkaWQsIHBvcnQpCiAgICBvdXQgPSBzeXMuc3Rkb3V0CgogICAgZGVmIGVtaXQob2JqKToKICAgICAgICAjIGRlZmF1bHQ9c3RyIGtlZXBzIGEgc3RyYXkgcGxpc3QgdHlwZSAoYnl0ZXMvZGF0ZXRpbWUgaW4gc3ByaW5nYm9hcmQKICAgICAgICAjIGljb24gc3RhdGUpIGZyb20gY3Jhc2hpbmcgc2VyaWFsaXphdGlvbiBtaWQtc2Vzc2lvbi4KICAgICAgICBvdXQud3JpdGUoanNvbi5kdW1wcyhvYmosIGRlZmF1bHQ9c3RyKSArICJcbiIpCiAgICAgICAgb3V0LmZsdXNoKCkKCiAgICB0cnk6CiAgICAgICAgYXdhaXQgYWdlbnQuY29ubmVjdCgpCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6ICAjIG5vcWE6IEJMRTAwMQogICAgICAgIGVtaXQoeyJyZWFkeSI6IEZhbHNlLCAiZXJyb3IiOiBzdHIoZSl9KQogICAgICAgIHJldHVybgogICAgZW1pdCh7InJlYWR5IjogVHJ1ZX0pCgogICAgbG9vcCA9IGFzeW5jaW8uZ2V0X2V2ZW50X2xvb3AoKQogICAgcmVhZGVyID0gYXN5bmNpby5TdHJlYW1SZWFkZXIoKQogICAgYXdhaXQgbG9vcC5jb25uZWN0X3JlYWRfcGlwZShsYW1iZGE6IGFzeW5jaW8uU3RyZWFtUmVhZGVyUHJvdG9jb2wocmVhZGVyKSwgc3lzLnN0ZGluKQoKICAgIHdoaWxlIFRydWU6CiAgICAgICAgbGluZSA9IGF3YWl0IHJlYWRlci5yZWFkbGluZSgpCiAgICAgICAgaWYgbm90IGxpbmU6CiAgICAgICAgICAgIGJyZWFrCiAgICAgICAgbGluZSA9IGxpbmUuc3RyaXAoKQogICAgICAgIGlmIG5vdCBsaW5lOgogICAgICAgICAgICBjb250aW51ZQogICAgICAgIHRyeToKICAgICAgICAgICAgbXNnID0ganNvbi5sb2FkcyhsaW5lKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246ICAjIG5vcWE6IEJMRTAwMQogICAgICAgICAgICBlbWl0KHsiZXJyb3IiOiAiYmFkIGpzb24ifSkKICAgICAgICAgICAgY29udGludWUKICAgICAgICBtaWQgPSBtc2cuZ2V0KCJpZCIpCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXN1bHQgPSBhd2FpdCBhZ2VudC5kaXNwYXRjaChtc2cpCiAgICAgICAgICAgIGVtaXQoeyJpZCI6IG1pZCwgIm9rIjogVHJ1ZSwgKipyZXN1bHR9KQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZTogICMgbm9xYTogQkxFMDAxCiAgICAgICAgICAgIHRleHQgPSBmInt0eXBlKGUpLl9fbmFtZV9ffToge2V9IgogICAgICAgICAgICBlbWl0KHsiaWQiOiBtaWQsICJlcnJvciI6IHRleHQsICJnYXRlZF85MDIxIjogX2lzXzkwMjEodGV4dCl9KQoKICAgIGF3YWl0IGFnZW50LmNsb3NlKCkKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgYXN5bmNpby5ydW4obWFpbigpKQo="; + +/** Decode the embedded agent program to its Python source. */ +export function coreDeviceAgentScript(): string { + return Buffer.from(AGENT_SCRIPT_B64, "base64").toString("utf8"); +} + +/** + * Materialize the agent program to a stable temp path (content-hashed, so a new + * build's script lands at a new path and an old one is never re-run). Written + * once; concurrent callers race harmlessly on an identical payload. + */ +export async function materializeAgentScript(): Promise { + const script = coreDeviceAgentScript(); + const hash = createHash("sha256").update(script).digest("hex").slice(0, 16); + const path = join(tmpdir(), `argent-coredevice-agent-${hash}.py`); + if (!existsSync(path)) { + await writeFile(path, script, { mode: 0o600 }); + } + return path; +} + +/** + * Resolve the Python interpreter that has pymobiledevice3 importable. The + * `pymobiledevice3` CLI is a thin launcher whose shebang points at its venv + * interpreter (pipx/venv installs), so read the shebang. Falls back to a + * sibling `python`/`python3` in the same bin dir. + */ +export async function resolvePmd3Python(pmd3CliPath: string): Promise { + try { + const head = (await readFile(pmd3CliPath, "utf8")).slice(0, 256); + const firstLine = head.split("\n", 1)[0] ?? ""; + if (firstLine.startsWith("#!")) { + const interp = firstLine.slice(2).trim().split(/\s+/)[0]; + if (interp && interp.startsWith("/") && existsSync(interp)) return interp; + } + } catch { + // not a readable script (e.g. a compiled shim) — fall through to siblings + } + const binDir = dirname(pmd3CliPath); + for (const name of ["python3", "python"]) { + const p = join(binDir, name); + if (existsSync(p)) return p; + } + throw new Error( + `could not resolve the pymobiledevice3 Python interpreter from "${pmd3CliPath}" ` + + `(its shebang and sibling python were both unusable)` + ); +} + +/** A response line from the agent, keyed by the request `id` we sent. */ +interface AgentResponse { + id?: number; + ok?: boolean; + error?: string; + gated_9021?: boolean; + [k: string]: unknown; +} + +/** Raised when the agent replies with an `error` (carries the 9021 hint). */ +export class CoreDeviceAgentError extends Error { + readonly gated9021: boolean; + constructor(message: string, gated9021: boolean) { + super(message); + this.name = "CoreDeviceAgentError"; + this.gated9021 = gated9021; + } +} + +/** + * One long-lived pymobiledevice3 process per physical iPhone. Connects the RSD + * tunnel and opens the HID/screenshot services once, then serves JSON commands + * on stdio — so each tap/screenshot/button costs a socket write, not a fresh + * ~0.8s Python cold-start (of which ~0.5s is just `import pymobiledevice3`). + * + * Requests are id-correlated; the caller may have several in flight, though the + * device serializes them anyway. + */ +export class CoreDeviceAgent { + private proc: ChildProcessWithoutNullStreams | null = null; + private rl: Interface | null = null; + private nextId = 1; + private readonly pending = new Map< + number, + { resolve: (r: AgentResponse) => void; reject: (e: Error) => void } + >(); + private startError: Error | null = null; + private exited = false; + + constructor( + private readonly python: string, + private readonly scriptPath: string, + private readonly udid: string, + private readonly tunneldPort: number, + private readonly startTimeoutMs = 30_000 + ) {} + + /** Spawn the agent and wait for its `{"ready":true}` handshake line. */ + async start(): Promise { + const proc = spawn(this.python, [this.scriptPath, this.udid, String(this.tunneldPort)], { + stdio: ["pipe", "pipe", "pipe"], + }); + this.proc = proc; + + let stderr = ""; + proc.stderr.on("data", (d: Buffer) => { + // pymobiledevice3 logs to stderr; keep a bounded tail for diagnostics. + stderr = (stderr + d.toString()).slice(-4000); + }); + + const rl = createInterface({ input: proc.stdout }); + this.rl = rl; + + const ready = new Promise((resolve, reject) => { + // `onLine` and `timer` reference each other; `timer` is only read from + // inside `onLine`, which never runs before `timer` is assigned below. + const onLine = (line: string): void => { + const trimmed = line.trim(); + if (!trimmed) return; + let msg: AgentResponse; + try { + msg = JSON.parse(trimmed) as AgentResponse; + } catch { + return; // ignore any non-JSON noise before the handshake + } + if ("ready" in msg) { + clearTimeout(timer); + rl.off("line", onLine); + if (msg.ready) { + rl.on("line", (l) => this.onResponse(l)); + resolve(); + } else { + reject(new Error(String(msg.error ?? "agent failed to connect"))); + } + } + }; + const timer = setTimeout(() => { + rl.off("line", onLine); + reject( + new Error( + `CoreDevice agent did not become ready within ${this.startTimeoutMs}ms` + + (stderr ? `; last stderr: ${stderr.slice(-300)}` : "") + ) + ); + }, this.startTimeoutMs); + rl.on("line", onLine); + }); + + proc.on("exit", (code, signal) => { + this.exited = true; + const err = new Error( + `CoreDevice agent exited (code=${code}, signal=${signal})` + + (stderr ? `; stderr: ${stderr.slice(-300)}` : "") + ); + this.startError ??= err; + for (const { reject } of this.pending.values()) reject(err); + this.pending.clear(); + }); + + try { + await ready; + } catch (err) { + this.dispose(); + throw err instanceof Error ? err : new Error(String(err)); + } + } + + private onResponse(line: string): void { + const trimmed = line.trim(); + if (!trimmed) return; + let msg: AgentResponse; + try { + msg = JSON.parse(trimmed) as AgentResponse; + } catch { + return; + } + if (typeof msg.id !== "number") return; + const waiter = this.pending.get(msg.id); + if (!waiter) return; + this.pending.delete(msg.id); + if (msg.error) { + waiter.reject(new CoreDeviceAgentError(String(msg.error), Boolean(msg.gated_9021))); + } else { + waiter.resolve(msg); + } + } + + /** Send one op and await its correlated response. */ + request( + op: string, + args: Record = {}, + timeoutMs = 30_000 + ): Promise { + if (this.exited || !this.proc) { + return Promise.reject(this.startError ?? new Error("CoreDevice agent is not running")); + } + const id = this.nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (this.pending.delete(id)) { + reject(new Error(`CoreDevice ${op} timed out after ${timeoutMs}ms`)); + } + }, timeoutMs); + this.pending.set(id, { + resolve: (r) => { + clearTimeout(timer); + resolve(r); + }, + reject: (e) => { + clearTimeout(timer); + reject(e); + }, + }); + this.proc!.stdin.write(JSON.stringify({ id, op, ...args }) + "\n"); + }); + } + + dispose(): void { + this.exited = true; + this.rl?.close(); + this.rl = null; + if (this.proc) { + this.proc.stdin.end(); + this.proc.kill("SIGTERM"); + this.proc = null; + } + for (const { reject } of this.pending.values()) { + reject(new Error("CoreDevice agent disposed")); + } + this.pending.clear(); + } +} diff --git a/packages/tool-server/src/blueprints/native-devtools.ts b/packages/tool-server/src/blueprints/native-devtools.ts index 72e0bee94..3150b7db7 100644 --- a/packages/tool-server/src/blueprints/native-devtools.ts +++ b/packages/tool-server/src/blueprints/native-devtools.ts @@ -311,6 +311,20 @@ export const nativeDevtoolsBlueprint: ServiceBlueprint((d) => { + if (d.platform === "ios") { + if (d.kind === "device") return []; + return [ + { + udid: d.udid, + name: d.name, + state: d.state, + runtime: d.runtime ?? "", + isAvailable: true, + platform: "ios", + }, + ]; + } + if (d.platform === "android") { + return [ + { + udid: d.serial, + name: d.avdName ?? d.model ?? d.serial, + state: d.state === "device" ? "Booted" : d.state, + runtime: d.sdkLevel != null ? `Android API ${d.sdkLevel}` : "Android", + isAvailable: true, + platform: "android", + }, + ]; + } + return []; + }); +} + export function createPreviewRouter(registry: Registry): Router { const router = express.Router(); @@ -93,7 +146,7 @@ export function createPreviewRouter(registry: Registry): Router { // udid, every other platform by its serial (a chromium entry has neither, so // it's skipped — it was never a valid preview target anyway). function deviceIdSet( - devices: ReadonlyArray<{ platform: string; udid?: string; serial?: string }> + devices: ReadonlyArray<{ platform: string; udid?: string; serial?: string | null }> ): Set { const ids = new Set(); for (const d of devices) { @@ -107,7 +160,7 @@ export function createPreviewRouter(registry: Registry): Router { // dedicated refresh below and the /simulators handler, which already fetches // the full list for its dropdown). function rememberDevices( - devices: ReadonlyArray<{ platform: string; udid?: string; serial?: string }> + devices: ReadonlyArray<{ platform: string; udid?: string; serial?: string | null }> ): void { knownDevices = { ids: deviceIdSet(devices), at: Date.now() }; } @@ -137,71 +190,19 @@ export function createPreviewRouter(registry: Registry): Router { router.get("/simulators", async (_req: Request, res: Response) => { try { - const data = await registry.invokeTool<{ - devices: Array< - | { platform: "ios"; udid: string; name: string; state: string; runtime: string } - | { - platform: "android"; - serial: string; - state: string; - avdName?: string; - model?: string; - sdkLevel?: number | null; - } - | { platform: "chromium"; id: string; title: string; port: number } - >; - }>(listDevicesTool.id); - // The preview UI keys off `udid` and `state === "Booted"`, which are - // iOS terminology. Map Android serials to the same shape so the same - // dropdown can target both platforms — `simulator-server/:udid` already - // accepts Android serials via `resolveDevice(udid)`. - // - // Chromium is intentionally excluded: the preview UI streams frames - // through simulator-server's WebSocket, which only exists for iOS / - // Android. Surfacing chromium entries would let the UI offer a target - // it can't actually drive. Chromium consumers should use the MCP tools - // (screenshot, describe, gesture-*) directly. - type PreviewEntry = { - udid: string; - name: string; - state: string; - runtime: string; - isAvailable: boolean; - platform: "ios" | "android"; - }; - const simulators = data.devices.flatMap((d) => { - if (d.platform === "ios") { - return [ - { - udid: d.udid, - name: d.name, - state: d.state, - runtime: d.runtime, - isAvailable: true, - platform: "ios", - }, - ]; - } - if (d.platform === "android") { - return [ - { - udid: d.serial, - name: d.avdName ?? d.model ?? d.serial, - state: d.state === "device" ? "Booted" : d.state, - runtime: d.sdkLevel != null ? `Android API ${d.sdkLevel}` : "Android", - isAvailable: true, - platform: "android", - }, - ]; - } - return []; - }); + // Reuse list-devices' own result type rather than a hand-rolled copy, so + // the preview's view of a device can't silently drift from what the tool + // actually returns (e.g. physical-iOS entries carrying `kind`/no runtime). + const data = await registry.invokeTool(listDevicesTool.id); // This is the authoritative fresh device list — prime the validation // cache so the immediately-following connect (/simulator-server/:udid) // and the describe poll loop hit a warm, correct set instead of each // re-running `list-devices`. rememberDevices(data.devices); - res.json({ simulators }); + // devicesToPreviewEntries maps iOS/Android into the UI's udid/Booted shape + // and excludes targets the preview can't stream — chromium (no + // simulator-server WebSocket) and physical iOS (kind: "device"). + res.json({ simulators: devicesToPreviewEntries(data.devices) }); } catch (err) { res.status(500).json({ error: err instanceof Error ? err.message : String(err) }); } diff --git a/packages/tool-server/src/tools/button/index.ts b/packages/tool-server/src/tools/button/index.ts index 8aa9db025..eb3043c56 100644 --- a/packages/tool-server/src/tools/button/index.ts +++ b/packages/tool-server/src/tools/button/index.ts @@ -1,12 +1,24 @@ import { z } from "zod"; import type { Platform, ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { UnsupportedOperationError } from "../../utils/capability"; import { sendCommand } from "../../utils/simulator-client"; import { ANDROID_BUTTON_KEYCODES, injectAndroidKeycode } from "../../utils/android-input"; import { ensureDep } from "../../utils/check-deps"; +// Argent button name → pymobiledevice3 CoreDevice HID button name. CoreDevice +// exposes the physical buttons only; appSwitch (a SpringBoard gesture) and the +// iPhone 15 Pro action button have no HID equivalent, so they are omitted and +// rejected with a clear error on physical iOS. +const COREDEVICE_BUTTON: Partial> = { + home: "home", + power: "lock", + volumeUp: "volume-up", + volumeDown: "volume-down", +}; + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const zodSchema = z.object({ @@ -58,6 +70,7 @@ export const buttonTool: ToolDefinition = { Supported buttons depend on the platform: home, back, power, volumeUp, volumeDown, appSwitch, actionButton — buttons not present on the target platform (e.g. 'back' on iOS, 'actionButton' on Android) are rejected with a clear error. Use when you need to trigger hardware button events. Returns { pressed: buttonName }. +On a physical iPhone, button presses route over CoreDevice (home, power, volumeUp, volumeDown). Fails if the device backend is not reachable — the simulator-server for iOS, or \`adb\` for Android (Android presses are injected with \`adb shell input keyevent\`).`, zodSchema, capability, @@ -69,6 +82,14 @@ Fails if the device backend is not reachable — the simulator-server for iOS, o // actually consumes it (mirrors the sibling `keyboard` tool's lazy services). services: (params): Record => { const device = resolveDevice(params.udid); + if (isPhysicalIos(device)) { + // A button with no CoreDevice HID equivalent (appSwitch/actionButton) is + // always rejected by execute() below, before it ever touches + // services.coreDevice — don't pay for resolving the CoreDevice service + // (tunnel setup, possibly a macOS admin prompt) just to reject anyway. + if (!COREDEVICE_BUTTON[params.button]) return {}; + return { coreDevice: coreDeviceRef(device) }; + } return device.platform === "android" ? {} : { simulatorServer: simulatorServerRef(device) }; }, async execute(services, params) { @@ -80,6 +101,19 @@ Fails if the device backend is not reachable — the simulator-server for iOS, o `button '${params.button}' is not available on ${device.platform}` ); } + if (isPhysicalIos(device)) { + const name = COREDEVICE_BUTTON[params.button]; + if (!name) { + throw new UnsupportedOperationError( + "button", + device, + `button '${params.button}' is not available on physical iOS (CoreDevice exposes home, power, volumeUp, volumeDown)` + ); + } + const coreDevice = services.coreDevice as CoreDeviceApi; + await coreDevice.button(name); + return { pressed: params.button }; + } if (device.platform === "android") { // Android presses go over `adb shell input keyevent`, not the // simulator-server's HID transport, which the guest silently drops on AVDs diff --git a/packages/tool-server/src/tools/describe/contract.ts b/packages/tool-server/src/tools/describe/contract.ts index cae35e884..fbc7dba28 100644 --- a/packages/tool-server/src/tools/describe/contract.ts +++ b/packages/tool-server/src/tools/describe/contract.ts @@ -76,7 +76,8 @@ export type DescribeSource = | "android-devtools" | "cdp-dom" | "vega-automation" - | "tv-focus"; + | "tv-focus" + | "coredevice-ax"; // Internal shape produced by the per-platform adapters. The `tree` is consumed // by the formatter in `format-tree.ts` and then dropped before the tool replies diff --git a/packages/tool-server/src/tools/describe/platforms/ios/index.ts b/packages/tool-server/src/tools/describe/platforms/ios/index.ts index 52addf05d..95e2eb319 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/index.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/index.ts @@ -6,12 +6,15 @@ import { nativeDevtoolsRef, NativeDevtoolsApi, } from "../../../../blueprints/native-devtools"; +import { isPhysicalIos } from "../../../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../../../blueprints/core-device"; import { resolveNativeTargetApp } from "../../../../utils/native-target-app"; import { isTvOsSimulator } from "../../../../utils/ios-devices"; import { parseNativeDescribeScreenResult } from "../../../native-devtools/native-describe-contract"; import { DescribeTreeData, parseDescribeResult, type DescribeNode } from "../../contract"; import { adaptAXDescribeToDescribeResult } from "./ios-ax-adapter"; import { adaptNativeDescribeToDescribeResult } from "./ios-native-adapter"; +import { adaptCoreDeviceAxToDescribeResult } from "./ios-coredevice-ax-adapter"; const DEGRADED_HINT = "This simulator was not booted through argent — system dialogs and native modals may not appear. You MUST call boot-device with force=true now to restart the simulator and apply full accessibility settings before continuing."; @@ -29,6 +32,19 @@ const TVOS_HINT = "(up/down/left/right/select/back/menu/home) to move focus, and `keyboard` to type. " + "See the argent-tv-interact skill."; +// Physical iPhones expose their real on-screen accessibility tree app-free via +// the iOS-26+ axAudit service (read over CoreDevice). Captions (label + value + +// traits) and reading order are exact for every element; frames are exact only +// for the subset the accessibility audit flags, and interpolated for the rest — +// this hint makes that precision boundary explicit. +const PHYSICAL_IOS_AX_HINT = + "This is the live accessibility tree of the frontmost app (or the home screen), read over " + + "CoreDevice. Element labels, values, traits and reading order are exact. Frames are exact for " + + "elements the accessibility audit reported and APPROXIMATE (interpolated from neighbours) for the " + + "rest — good enough to tap a row in a vertical list, but confirm with screenshot before a precise " + + "tap, especially for controls like toggles. Apple does not expose per-element geometry on a " + + "physical device, so screenshot remains authoritative for exact positions."; + // Apple system apps (`com.apple.*`) can never load argent's injected dylib, so // the native-devtools fallback can't read their view hierarchy and restarting // them would never help — returning `should_restart` here puts the agent in an @@ -75,6 +91,26 @@ export async function describeIos( params: DescribeIosParams, options: DescribeIosOptions = {} ): Promise { + // Physical iPhones are driven over CoreDevice. describe reads the device's real + // on-screen accessibility tree app-free via the iOS-26+ axAudit service (the + // `…axAuditDaemon.remoteserver.shim.remote` DTX daemon). This works in ANY app + // and on the home screen — the same VoiceOver-style walk. It needs the RSDCheckin + // handshake that iOS 26 added (the sidecar performs it); without it the daemon + // drops the connection on the first byte. Apple exposes no per-element geometry + // on hardware, so frames are exact only for elements the accessibility audit + // flags and interpolated for the rest (see the adapter + hint). The two + // simulator backends below can't run against hardware (they shell `simctl spawn`). + if (isPhysicalIos(device)) { + const ref = coreDeviceRef(device); + const coreDevice = await registry.resolveService(ref.urn, ref.options); + const axtree = await coreDevice.axtree(); + return { + tree: adaptCoreDeviceAxToDescribeResult(axtree), + source: "coredevice-ax", + hint: PHYSICAL_IOS_AX_HINT, + }; + } + // tvOS short-circuit: the focus-engine accessibility tree is served by the // tv-control daemons, not the iOS ax-service. Without this, describe would // try to spawn ax-service inside the Apple TV sim, time out on the daemon diff --git a/packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts b/packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts new file mode 100644 index 000000000..77abfc0a5 --- /dev/null +++ b/packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts @@ -0,0 +1,123 @@ +import type { CoreDeviceAxTree } from "../../../../blueprints/core-device"; +import { parseDescribeResult, type DescribeNode } from "../../contract"; + +/** + * Adapts a physical iPhone's on-screen accessibility tree (from the iOS-26+ + * axAudit service, read app-free over CoreDevice) into a describe tree. + * + * The audit gives a rich VoiceOver caption (label + value + traits) and reading + * order for EVERY on-screen element, but Apple doesn't expose per-element + * geometry on hardware — only the subset of elements the accessibility audit + * flags carry an on-screen rect. So real frames are used where present, and the + * rest are interpolated from their reading-order neighbours (good enough to tap + * a row in a vertical list; the tool's hint says to confirm with screenshot). + */ + +// AX trait tokens that appear (comma-separated) at the tail of a VoiceOver +// caption, mapped to a describe role. Order matters: the first structural trait +// found wins. +const TRAIT_ROLE: Array<[RegExp, string]> = [ + [/^Button$/i, "AXButton"], + [/^Link$/i, "AXLink"], + [/^Header$/i, "AXHeader"], + [/^(Toggle|Switch)$/i, "AXSwitch"], + [/^Adjustable$/i, "AXSlider"], + [/^Search Field$/i, "AXSearchField"], + [/^Text Field$/i, "AXTextField"], + [/^Tab$/i, "AXTab"], + [/^Image$/i, "AXImage"], +]; +// Trailing tokens that are traits/states (stripped from the label). +const TRAIT_TOKEN = + /^(Button|Link|Header|Toggle|Switch|Adjustable|Search Field|Text Field|Tab|Image|Selected|Not Selected|Dimmed|Disabled)$/i; + +function parseCaption(caption: string): { label: string; role: string } { + const tokens = caption.split(/,\s*/).filter((t) => t.length > 0); + let role = "AXStaticText"; + for (const [re, r] of TRAIT_ROLE) { + if (tokens.some((t) => re.test(t))) { + role = r; + break; + } + } + // Drop trailing trait/state tokens to get a cleaner label; keep the full + // caption if that would leave nothing. + let end = tokens.length; + while (end > 0 && TRAIT_TOKEN.test(tokens[end - 1])) end--; + const label = (end > 0 ? tokens.slice(0, end) : tokens).join(", ") || caption; + return { label, role }; +} + +const RECT_RE = /-?\d+(?:\.\d+)?/g; + +/** Parse "{{x, y}, {w, h}}" (points) → normalized frame, or null. */ +function parseRect(rect: string | undefined, sw: number, sh: number): DescribeNode["frame"] | null { + if (!rect || sw <= 0 || sh <= 0) return null; + const nums = rect.match(RECT_RE); + if (!nums || nums.length < 4) return null; + const [x, y, w, h] = nums.slice(0, 4).map(Number); + const clamp = (v: number) => Math.max(0, Math.min(1, v)); + return { + x: clamp(x / sw), + y: clamp(y / sh), + width: clamp(w / sw), + height: clamp(h / sh), + }; +} + +const MARGIN_X = 0.04; +const APPROX_HEIGHT = 0.05; + +/** Full-width approximate frame centred at normalized y. */ +function approxFrame(yCenter: number): DescribeNode["frame"] { + const y = Math.max(0, Math.min(1 - APPROX_HEIGHT, yCenter - APPROX_HEIGHT / 2)); + return { x: MARGIN_X, y, width: 1 - 2 * MARGIN_X, height: APPROX_HEIGHT }; +} + +/** + * Fill frames for elements the audit didn't give a rect: interpolate each gap's + * y-centres between the nearest real rects above and below (reading order), so a + * list row lands roughly where it should. Falls back to an even top-to-bottom + * spread when there are no anchoring rects. + */ +function fillFrames(frames: Array): DescribeNode["frame"][] { + const n = frames.length; + const yc = (f: DescribeNode["frame"]) => f.y + f.height / 2; + const out = frames.slice(); + for (let i = 0; i < n; i++) { + if (out[i]) continue; + let prev = i - 1; + while (prev >= 0 && !out[prev]) prev--; + let next = i + 1; + while (next < n && !out[next]) next++; + const top = prev >= 0 ? yc(out[prev]!) : 0.06; + const bottom = next < n ? yc(out[next]!) : 0.94; + const span = next < n ? next : n; // denominator for even spread in the run + const start = prev >= 0 ? prev : -1; + const frac = (i - start) / (span - start); + out[i] = approxFrame(top + (bottom - top) * frac); + } + return out as DescribeNode["frame"][]; +} + +export function adaptCoreDeviceAxToDescribeResult(tree: CoreDeviceAxTree): DescribeNode { + const sw = tree.screen?.w ?? 0; + const sh = tree.screen?.h ?? 0; + const els = tree.elements ?? []; + + const rectFrames = els.map((e) => parseRect(e.rect, sw, sh)); + const frames = fillFrames(rectFrames); + + const children: DescribeNode[] = els.map((e, i) => { + const { label, role } = parseCaption(e.caption ?? ""); + const node: DescribeNode = { role, frame: frames[i], children: [] }; + if (label) node.label = label; + return node; + }); + + return parseDescribeResult({ + role: "AXGroup", + frame: { x: 0, y: 0, width: 1, height: 1 }, + children, + }); +} diff --git a/packages/tool-server/src/tools/devices/boot-device.ts b/packages/tool-server/src/tools/devices/boot-device.ts index a0b66fb21..777b76eeb 100644 --- a/packages/tool-server/src/tools/devices/boot-device.ts +++ b/packages/tool-server/src/tools/devices/boot-device.ts @@ -31,7 +31,8 @@ import { import { ensureDep } from "../../utils/check-deps"; import { linuxBootDiagnostics } from "../../utils/linux-preflight"; import { listIosSimulators } from "../../utils/ios-devices"; -import { classifyDevice, stripRemotePrefix } from "../../utils/device-info"; +import { isPhysicalIosUdid, classifyDevice, stripRemotePrefix } from "../../utils/device-info"; +import { ensureCoreDeviceTunnel } from "../../blueprints/core-device"; import { simctlBoot as simRemoteBoot, simctlBootstatus as simRemoteBootstatus, @@ -477,6 +478,17 @@ async function bootIos( } ); } + + // A physical iPhone is already powered on — there is nothing to "boot". It is + // driven over CoreDevice (see core-device blueprint), not the simulator-server. + // Use this as the explicit "prepare" step: ensure the CoreDevice tunnel is up, + // auto-starting it via the macOS authorization prompt if needed (no manual + // sudo). Surfaces a clear error if the prompt is declined / unavailable. + if (isPhysicalIosUdid(udid)) { + await ensureCoreDeviceTunnel(udid); + return { platform: "ios", udid, booted: true }; + } + await ensureDep("xcrun"); const simMatch = await listIosSimulators() diff --git a/packages/tool-server/src/tools/devices/list-devices.ts b/packages/tool-server/src/tools/devices/list-devices.ts index 070b46047..1a750a5d1 100644 --- a/packages/tool-server/src/tools/devices/list-devices.ts +++ b/packages/tool-server/src/tools/devices/list-devices.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import type { ToolDefinition } from "@argent/registry"; +import { isFlagEnabled } from "@argent/configuration-core"; import { listAndroidDevices, listAvds, @@ -7,7 +8,7 @@ import { ADB_DEVICES_TIMEOUT_MS, } from "../../utils/adb"; import { listRunningVvdConsolePorts } from "../../utils/vega-process"; -import { listIosSimulators, type IosSimulator } from "../../utils/ios-devices"; +import { listIosSimulators, listIosDevices } from "../../utils/ios-devices"; import { simctlListDevices } from "../../utils/sim-remote"; import { withRemotePrefix } from "../../utils/device-info"; import { discoverChromiumDevices, type ChromiumDevice } from "../../utils/chromium-discovery"; @@ -16,7 +17,24 @@ import { filterVvdShadowsFromAndroid, type VegaDevice, } from "../../utils/vega-devices"; -type IosDevice = IosSimulator & { platform: "ios" }; + +const PHYSICAL_IOS_FLAG = "physical-ios-devices"; + +type IosDevice = { + platform: "ios"; + udid: string; + name: string; + state: string; + // "simulator" for an `xcrun simctl` simulator, "device" for a physical iPhone + // discovered via `xcrun devicectl` and driven over CoreDevice (pymobiledevice3). + kind: "simulator" | "device"; + // simulators only (the iOS runtime, e.g. "com.apple.CoreSimulator.SimRuntime.iOS-18-5") + runtime?: string; + // simulators only: "tv" for an Apple TV simulator, "mobile" for iOS/iPadOS. + runtimeKind?: "mobile" | "tv"; + // physical devices only (Apple product type, e.g. "iPhone15,4") + productType?: string | null; +}; type IosRemoteDevice = { platform: "ios-remote"; @@ -42,15 +60,18 @@ type AndroidDevice = { runtimeKind?: "mobile" | "tv"; }; -type ListDevicesResult = { +export type ListDevicesResult = { devices: Array; avds: Array<{ name: string }>; }; +// A simulator is ready when "Booted"; a physical device is ready when "connected". +const iosReady = (d: IosDevice): boolean => d.state === "Booted" || d.state === "connected"; + function sortIos(a: IosDevice, b: IosDevice): number { - const aBooted = a.state === "Booted" ? 0 : 1; - const bBooted = b.state === "Booted" ? 0 : 1; - if (aBooted !== bBooted) return aBooted - bBooted; + const aReady = iosReady(a) ? 0 : 1; + const bReady = iosReady(b) ? 0 : 1; + if (aReady !== bReady) return aReady - bReady; const aIpad = a.name.includes("iPad") ? 1 : 0; const bIpad = b.name.includes("iPad") ? 1 : 0; return aIpad - bIpad; @@ -70,6 +91,7 @@ function sortAndroid(a: AndroidDevice, b: AndroidDevice): number { function readinessRank( d: IosDevice | IosRemoteDevice | AndroidDevice | ChromiumDevice | VegaDevice ): number { + if (d.platform === "ios") return iosReady(d) ? 0 : 1; if (d.platform === "android") return d.state === "device" ? 0 : 1; if (d.platform === "vega") return d.state === "running" || d.state === "device" ? 0 : 1; if (d.platform === "chromium") return 0; // Chromium entries are only listed when their CDP is responsive @@ -196,6 +218,7 @@ export const listDevicesTool: ToolDefinition, ListDevicesR Use at the start of a session to pick a target id ('udid' for iOS entries, 'serial' for Android/Vega entries, 'id' for Chromium) to pass to interaction tools, and to see which targets are already running. Returns { devices, avds } where each device carries a 'platform' discriminator ('ios', 'android', 'chromium', or 'vega'); 'avds' lists Android AVDs bootable via boot-device. A Vega VVD is listed under 'devices' whether running or stopped (state 'running'/'stopped'); start a stopped one with boot-device using its 'vvdImage'. Android entries also carry a 'kind' ('emulator' for a local AVD, 'device' for a physical phone connected over USB / wireless adb) — physical phones are detected from \`adb devices\` (any serial that is not an \`emulator-*\` one) and are driven through the same interaction tools as emulators; they do not need boot-device (just connect the phone with USB debugging authorised). +iOS entries likewise carry a 'kind' ('simulator', or 'device' for a connected physical iPhone). Physical iOS devices require the 'physical-ios-devices' flag (\`argent enable physical-ios-devices\`), iOS 27+, and a running CoreDevice tunnel (\`sudo pymobiledevice3 remote tunneld\`); they support screenshot, gesture-tap, gesture-swipe, button, and launch-app. TV targets are tagged with runtimeKind 'tv' (Apple TV simulators on iOS, Android TV / leanback devices on Android) — these are focus-driven, not touch-driven: use \`describe\` to read focus, \`tv-remote\` for remote presses (up/down/left/right/select/back/menu/home), and \`keyboard\` to type, rather than the coordinate/gesture tools. Chromium apps are discovered by probing CDP debugging ports (default 9222; extend via the ARGENT_CHROMIUM_PORTS= env var). They must already be running with --remote-debugging-port= — use boot-device with electronAppPath to launch one. Booted/ready devices are listed first. Platforms whose CLI is unavailable are silently omitted — an empty result usually means xcode-select, Android platform-tools, or the Vega SDK is not installed.`, @@ -213,9 +236,15 @@ Booted/ready devices are listed first. Platforms whose CLI is unavailable are si // timer is cleared on the fast happy path). The deadline only substitutes a // fallback on *slowness*; a rejection still propagates exactly as before — so the // `.catch(() => [])` wrappers (and the lack of one on iOS/AVDs) are unchanged. - const [ios, iosRemote, android, avds, chromium, vega] = await Promise.all([ + const physicalIosEnabled = isFlagEnabled(PHYSICAL_IOS_FLAG); + const [ios, iosRemote, iosPhysical, android, avds, chromium, vega] = await Promise.all([ withDeadline(listIosSimulators(), [], "ios"), withDeadline(listRemoteIosSimulators(), [], "ios-remote"), + withDeadline( + physicalIosEnabled ? listIosDevices().catch(() => []) : Promise.resolve([]), + [], + "ios-physical" + ), withDeadline( // Opt into runtimeKind enrichment (list-devices surfaces TV vs mobile to // the agent, so the extra feature probe per device is warranted here — the @@ -240,7 +269,29 @@ Booted/ready devices are listed first. Platforms whose CLI is unavailable are si "vega" ), ]); - const iosTagged: IosDevice[] = ios.map((s) => ({ platform: "ios", ...s })); + const iosTagged: IosDevice[] = [ + ...ios.map( + (s): IosDevice => ({ + platform: "ios", + kind: "simulator", + udid: s.udid, + name: s.name, + state: s.state, + runtime: s.runtime, + runtimeKind: s.runtimeKind, + }) + ), + ...iosPhysical.map( + (d): IosDevice => ({ + platform: "ios", + kind: "device", + udid: d.udid, + name: d.name, + state: d.state, + productType: d.productType, + }) + ), + ]; iosTagged.sort(sortIos); iosRemote.sort(sortIosRemote); const androidTagged: AndroidDevice[] = android.map((d) => ({ diff --git a/packages/tool-server/src/tools/gesture-custom/index.ts b/packages/tool-server/src/tools/gesture-custom/index.ts index 14ea892a1..beedcad2e 100644 --- a/packages/tool-server/src/tools/gesture-custom/index.ts +++ b/packages/tool-server/src/tools/gesture-custom/index.ts @@ -49,7 +49,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, appleRemote: { simulator: true }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/gesture-pinch/index.ts b/packages/tool-server/src/tools/gesture-pinch/index.ts index 5efbcee38..715314b90 100644 --- a/packages/tool-server/src/tools/gesture-pinch/index.ts +++ b/packages/tool-server/src/tools/gesture-pinch/index.ts @@ -49,7 +49,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, appleRemote: { simulator: true }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/gesture-rotate/index.ts b/packages/tool-server/src/tools/gesture-rotate/index.ts index d93b75e6b..6c7ccb53c 100644 --- a/packages/tool-server/src/tools/gesture-rotate/index.ts +++ b/packages/tool-server/src/tools/gesture-rotate/index.ts @@ -39,7 +39,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, appleRemote: { simulator: true }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/gesture-swipe/index.ts b/packages/tool-server/src/tools/gesture-swipe/index.ts index fcc4a31a6..89d962384 100644 --- a/packages/tool-server/src/tools/gesture-swipe/index.ts +++ b/packages/tool-server/src/tools/gesture-swipe/index.ts @@ -1,7 +1,8 @@ import { z } from "zod"; -import type { ToolCapability, ToolDefinition } from "@argent/registry"; +import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { sendCommand } from "../../utils/simulator-client"; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -40,17 +41,27 @@ export const gestureSwipeTool: ToolDefinition = { description: `Execute a smooth swipe / drag touch gesture between two points on the device (iOS simulator or Android emulator). All from/to positions are normalized 0.0–1.0 (fractions of screen width/height, not pixels), same as gesture-tap. Generates interpolated Move events for a natural feel (~60fps). Swipe up (fromY > toY) to scroll content down. -Use when you need to scroll a list, dismiss a modal, drag an element, or navigate between pages. Not supported on Chromium — use gesture-scroll there instead. Returns { swiped: true, timestampMs }. Fails if the simulator-server / emulator backend is not reachable for the given device.`, +Use when you need to scroll a list, dismiss a modal, drag an element, or navigate between pages. Not supported on Chromium — use gesture-scroll there instead. Returns { swiped: true, timestampMs }. On a physical iPhone, swipes route over CoreDevice. Fails if the simulator-server / emulator backend is not reachable for the given device.`, alwaysLoad: true, searchHint: "swipe scroll drag pan gesture device simulator emulator touch move", zodSchema, capability, - services: (params) => ({ - simulatorServer: simulatorServerRef(resolveDevice(params.udid)), - }), + services: (params): Record => { + const device = resolveDevice(params.udid); + if (isPhysicalIos(device)) { + return { coreDevice: coreDeviceRef(device) }; + } + return { simulatorServer: simulatorServerRef(device) }; + }, async execute(services, params) { const duration = params.durationMs ?? 300; const timestampMs = Date.now(); + const device = resolveDevice(params.udid); + if (isPhysicalIos(device)) { + const coreDevice = services.coreDevice as CoreDeviceApi; + await coreDevice.swipe(params.fromX, params.fromY, params.toX, params.toY, duration); + return { swiped: true, timestampMs }; + } const api = services.simulatorServer as SimulatorServerApi; const steps = Math.max(1, Math.round(duration / 16)); diff --git a/packages/tool-server/src/tools/gesture-tap/index.ts b/packages/tool-server/src/tools/gesture-tap/index.ts index 0654b339d..3ba213f15 100644 --- a/packages/tool-server/src/tools/gesture-tap/index.ts +++ b/packages/tool-server/src/tools/gesture-tap/index.ts @@ -2,7 +2,8 @@ import { z } from "zod"; import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; import { chromiumCdpRef, type ChromiumCdpApi } from "../../blueprints/chromium-cdp"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { sendCommand } from "../../utils/simulator-client"; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -44,7 +45,7 @@ export const gestureTapTool: ToolDefinition = { description: `Press the device screen (iOS simulator, Android emulator, or Chromium app) at normalized coordinates: x and y are fractions of screen width and height in 0.0–1.0 (not pixels). Sends a Down event followed by an Up event at the same point. For Chromium, this dispatches a CDP mouse-press/release on the renderer. Use when you need to tap a button, link, or any tappable element on the screen. -Returns { tapped: true, timestampMs }. Fails if the simulator-server / emulator backend / Chromium CDP is not reachable for the given device. +Returns { tapped: true, timestampMs }. On a physical iPhone, taps route over CoreDevice. Fails if the simulator-server / emulator backend / Chromium CDP is not reachable for the given device. Before tapping, determine the correct coordinates by using discovery tools — pick by platform: iOS / Android use \`describe\`, \`native-describe-screen\`, or \`debugger-component-tree\`; Chromium uses \`describe\` (the DOM walker), since the native and RN-specific discovery tools don't apply. More information in \`argent-device-interact\` skill`, alwaysLoad: true, searchHint: "tap press button element device simulator emulator chromium touch down up click", @@ -55,6 +56,9 @@ Before tapping, determine the correct coordinates by using discovery tools — p if (device.platform === "chromium") { return { chromium: chromiumCdpRef(device) }; } + if (isPhysicalIos(device)) { + return { coreDevice: coreDeviceRef(device) }; + } return { simulatorServer: simulatorServerRef(device) }; }, async execute(services, params) { @@ -65,6 +69,11 @@ Before tapping, determine the correct coordinates by using discovery tools — p await tapChromium(chromium, params.x, params.y); return { tapped: true, timestampMs }; } + if (isPhysicalIos(device)) { + const coreDevice = services.coreDevice as CoreDeviceApi; + await coreDevice.tap(params.x, params.y); + return { tapped: true, timestampMs }; + } const api = services.simulatorServer as SimulatorServerApi; sendCommand(api, { cmd: "touch", diff --git a/packages/tool-server/src/tools/keyboard/index.ts b/packages/tool-server/src/tools/keyboard/index.ts index dbe252e83..78ddd9ba0 100644 --- a/packages/tool-server/src/tools/keyboard/index.ts +++ b/packages/tool-server/src/tools/keyboard/index.ts @@ -34,7 +34,7 @@ const zodSchema = z.object({ type Params = z.infer; const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, appleRemote: { simulator: true }, android: { emulator: true, device: true, unknown: true }, chromium: { app: true }, diff --git a/packages/tool-server/src/tools/launch-app/platforms/ios.ts b/packages/tool-server/src/tools/launch-app/platforms/ios.ts index 78900fb0d..cc26bbebb 100644 --- a/packages/tool-server/src/tools/launch-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/launch-app/platforms/ios.ts @@ -11,6 +11,7 @@ import { precheckNativeDevtools, type NativeDevtoolsApi, } from "../../../blueprints/native-devtools"; +import { assertPhysicalIosEnabled } from "../../../blueprints/core-device"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; import type { LaunchAppParams, LaunchAppResult } from "../types"; @@ -26,6 +27,44 @@ export function makeIosImpl( return { requires: ["xcrun"], handler: async (_services, params, device) => { + // Physical iPhones are driven via CoreDevice — launch through devicectl + // (the app must already be installed/signed on the device). The + // native-devtools precheck is simulator-only, so it is skipped here. + if (device.kind === "device") { + // Unlike the CoreDevice-routed tools, launch-app shells devicectl directly + // (no CoreDevice service), so it must enforce the opt-in flag itself — + // otherwise it would be the one physical-iOS operation reachable while the + // feature is disabled. + assertPhysicalIosEnabled(); + try { + await execFileAsync("xcrun", [ + "devicectl", + "device", + "process", + "launch", + "--device", + params.udid, + params.bundleId, + ]); + } catch (err) { + // The dominant failure here is "app not installed/signed on the device". + // Without this wrap, devicectl's verbose multi-line blob surfaces raw as + // a 500; emit a structured FailureError with a clean message + telemetry + // instead (mirroring the simctl branch below). + throw new FailureError( + `Failed to launch ${params.bundleId} on physical iOS device ${params.udid} via devicectl — the app must already be installed and signed on the device.`, + { + error_code: FAILURE_CODES.IOS_LAUNCH_DEVICECTL_FAILED, + failure_stage: "ios_launch_app_devicectl_launch", + failure_area: "tool_server", + error_kind: "subprocess", + ...subprocessFailureMetadata(err, "xcrun_devicectl"), + }, + { cause: err instanceof Error ? err : new Error(String(err)) } + ); + } + return { launched: true, bundleId: params.bundleId }; + } const ndRef = nativeDevtoolsRef(device); const nativeDevtools = await registry.resolveService( ndRef.urn, diff --git a/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts b/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts index 5c454ab4b..77c67b7c1 100644 --- a/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts +++ b/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts @@ -41,7 +41,7 @@ type Result = export const nativeDescribeScreenTool: ToolDefinition = { id: "native-describe-screen", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Read the running app's native accessibility screen description via injected native devtools. Returns a flat list of accessibility leaf elements with: @@ -62,6 +62,9 @@ If status is restart_required: call restart-app then retry.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts b/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts index 2400795eb..d82199ecb 100644 --- a/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts +++ b/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts @@ -30,7 +30,7 @@ type Result = export const nativeDevtoolsStatusTool: ToolDefinition = { id: "native-devtools-status", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Check whether native devtools are connected to a specific app and whether the next launch is prepared for injection. Use when you need to verify native devtools readiness before calling native-full-hierarchy, native-describe-screen, or native-network-logs. @@ -54,6 +54,9 @@ Fails if the simulator server is not running for the given UDID.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-find-views.ts b/packages/tool-server/src/tools/native-devtools/native-find-views.ts index 9533bfafa..41ce7e8c4 100644 --- a/packages/tool-server/src/tools/native-devtools/native-find-views.ts +++ b/packages/tool-server/src/tools/native-devtools/native-find-views.ts @@ -44,7 +44,7 @@ type Result = export const nativeFindViewsTool: ToolDefinition = { id: "native-find-views", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Search for specific UIViews in the running app by class name, accessibility identifier, label, tag, or React Native nativeID. Use when you need to locate a specific view by its properties without dumping the entire hierarchy. Returns { status: "ok", matches } with matching views including their frames, properties, optional ancestors, and optional children. Much more targeted than native-full-hierarchy. @@ -56,6 +56,9 @@ Fails if native devtools are not connected, the app is not running, or status is }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts b/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts index ab85f7f0b..c13550129 100644 --- a/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts +++ b/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts @@ -58,7 +58,7 @@ type Result = export const nativeFullHierarchyTool: ToolDefinition = { id: "native-full-hierarchy", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Get the complete UIKit view tree for the running app. WARNING: Output can be extremely large (100KB–500KB+) for complex apps, especially those built with SwiftUI. Prefer native-find-views for targeted queries. Use skipClasses / skipClassPrefixes to prune SwiftUI internal subtrees and reduce output size. Use the fields param to request only the properties you need. @@ -71,6 +71,9 @@ Fails if native devtools are not connected or the app is not running.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-network-logs.ts b/packages/tool-server/src/tools/native-devtools/native-network-logs.ts index f8cc9fd01..420b214c2 100644 --- a/packages/tool-server/src/tools/native-devtools/native-network-logs.ts +++ b/packages/tool-server/src/tools/native-devtools/native-network-logs.ts @@ -29,7 +29,7 @@ type Result = export const nativeNetworkLogsTool: ToolDefinition = { id: "native-network-logs", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Retrieve network requests captured at the native NSURLProtocol level. Unlike the JS-level network inspector (view-network-logs), this captures ALL network traffic from the app including native modules, Swift/Objective-C networking, and background transfers that bypass JS fetch. Use when you need to inspect native-level HTTP traffic that is invisible to JS fetch interception. @@ -41,6 +41,9 @@ Fails if native devtools are not connected or the app is not running.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts b/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts index c30ed7cf4..b886925e2 100644 --- a/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts +++ b/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts @@ -63,7 +63,7 @@ type Result = export const nativeUserInteractableViewAtPointTool: ToolDefinition = { id: "native-user-interactable-view-at-point", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Inspect the deepest UIView at a raw native window point that would actually receive touch input. Unlike native-view-at-point, this respects userInteractionEnabled and is closer to @@ -79,6 +79,9 @@ If status is restart_required: call restart-app then retry.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts b/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts index f5f21a136..2eb607494 100644 --- a/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts +++ b/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts @@ -63,7 +63,7 @@ type Result = export const nativeViewAtPointTool: ToolDefinition = { id: "native-view-at-point", - capability: { apple: { simulator: true, device: true }, appleRemote: { simulator: true } }, + capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Inspect the deepest visible UIView at a raw native window point. Unlike native-user-interactable-view-at-point, this ignores userInteractionEnabled, @@ -79,6 +79,9 @@ If status is restart_required: call restart-app then retry.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/open-url/platforms/ios.ts b/packages/tool-server/src/tools/open-url/platforms/ios.ts index 0bed1b6bd..720abfcff 100644 --- a/packages/tool-server/src/tools/open-url/platforms/ios.ts +++ b/packages/tool-server/src/tools/open-url/platforms/ios.ts @@ -2,13 +2,25 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { FAILURE_CODES, FailureError, subprocessFailureMetadata } from "@argent/registry"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; +import { UnsupportedOperationError } from "../../../utils/capability"; import type { OpenUrlParams, OpenUrlResult, OpenUrlServices } from "../types"; const execFileAsync = promisify(execFile); export const iosImpl: PlatformImpl = { requires: ["xcrun"], - handler: async (_services, params) => { + handler: async (_services, params, device) => { + if (device.kind === "device") { + // CoreDevice/devicectl has no deep-link/open-url surface for physical + // iOS; only screenshot, gesture-tap, gesture-swipe, button, and launch-app + // are supported there today. UnsupportedOperationError maps to a clean + // 400 (a plain Error would surface as a generic 500). + throw new UnsupportedOperationError( + "open-url", + device, + "CoreDevice has no deep-link/open-url surface for physical iOS" + ); + } try { await execFileAsync("xcrun", ["simctl", "openurl", params.udid, params.url]); } catch (err) { diff --git a/packages/tool-server/src/tools/paste/index.ts b/packages/tool-server/src/tools/paste/index.ts index c0ec32e21..afb66506d 100644 --- a/packages/tool-server/src/tools/paste/index.ts +++ b/packages/tool-server/src/tools/paste/index.ts @@ -19,7 +19,7 @@ interface Result { // Android serials with "Tool 'paste' is not supported on android". The handler // itself is iOS-only — no platforms/ split needed. const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, appleRemote: { simulator: true }, }; diff --git a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts index 182fe1420..36b099993 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts @@ -18,8 +18,12 @@ const zodSchema = z.object({ .describe("Target device id from `list-devices` (iOS UDID or Android serial)."), }); +// A session can never exist for physical iOS (native-profiler-start rejects it, +// same apple.device:false reasoning) — reject here too for a clean, consistent +// error rather than the confusing "no active session" a physical UDID would +// otherwise always hit. const capability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, } as const; diff --git a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts index bdde7fdfb..23f508e65 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts @@ -32,7 +32,7 @@ const zodSchema = z.object({ }); const capability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, } as const; diff --git a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts index 6d7d87cdd..e3a76e948 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts @@ -50,8 +50,12 @@ interface AndroidStopArtifacts { type StopResult = IosStopArtifacts | AndroidStopArtifacts; +// A session can never exist for physical iOS (native-profiler-start rejects it, +// same apple.device:false reasoning) — reject here too for a clean, consistent +// error rather than the confusing "no active session" a physical UDID would +// otherwise always hit. const capability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, } as const; diff --git a/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts b/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts index 4d81f91be..29d1f064e 100644 --- a/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts @@ -3,13 +3,25 @@ import { promisify } from "node:util"; import { resolve as resolvePath } from "node:path"; import { FAILURE_CODES, FailureError, subprocessFailureMetadata } from "@argent/registry"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; +import { UnsupportedOperationError } from "../../../utils/capability"; import type { ReinstallAppParams, ReinstallAppResult, ReinstallAppServices } from "../types"; const execFileAsync = promisify(execFile); export const iosImpl: PlatformImpl = { requires: ["xcrun"], - handler: async (_services, params) => { + handler: async (_services, params, device) => { + if (device.kind === "device") { + // Installing on a physical iPhone needs a device-signed .app and a + // provisioning profile; that is out of scope for the CoreDevice path. + // UnsupportedOperationError maps to a clean 400 (a plain Error would + // surface as a generic 500). + throw new UnsupportedOperationError( + "reinstall-app", + device, + "installing on physical iOS needs a device-signed .app and provisioning profile, which is out of scope for the CoreDevice path" + ); + } const { udid, bundleId, appPath } = params; const absolute = resolvePath(appPath); try { diff --git a/packages/tool-server/src/tools/restart-app/platforms/ios.ts b/packages/tool-server/src/tools/restart-app/platforms/ios.ts index 3d5b86bdc..c52ac9eca 100644 --- a/packages/tool-server/src/tools/restart-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/restart-app/platforms/ios.ts @@ -12,6 +12,7 @@ import { type NativeDevtoolsApi, } from "../../../blueprints/native-devtools"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; +import { UnsupportedOperationError } from "../../../utils/capability"; import type { RestartAppParams, RestartAppResult } from "../types"; const execFileAsync = promisify(execFile); @@ -27,6 +28,16 @@ export function makeIosImpl( requires: ["xcrun"], handler: async (_services, params, device) => { const { udid, bundleId } = params; + if (device.kind === "device") { + // Physical iOS is driven over CoreDevice and has no relaunch path (the + // native-devtools injection is simulator-only). UnsupportedOperationError + // maps to a clean 400 (a plain Error would surface as a generic 500). + throw new UnsupportedOperationError( + "restart-app", + device, + "physical iOS is driven over CoreDevice and has no relaunch path" + ); + } const ndRef = nativeDevtoolsRef(device); const nativeDevtools = await registry.resolveService( ndRef.urn, diff --git a/packages/tool-server/src/tools/rotate/index.ts b/packages/tool-server/src/tools/rotate/index.ts index 840f8f00a..37aef743e 100644 --- a/packages/tool-server/src/tools/rotate/index.ts +++ b/packages/tool-server/src/tools/rotate/index.ts @@ -18,7 +18,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, appleRemote: { simulator: true }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/run-sequence/index.ts b/packages/tool-server/src/tools/run-sequence/index.ts index 4502a2073..c74f0eb30 100644 --- a/packages/tool-server/src/tools/run-sequence/index.ts +++ b/packages/tool-server/src/tools/run-sequence/index.ts @@ -145,13 +145,15 @@ Stops on the first error (or unmet await-ui-element condition) and returns parti capability, // No eagerly-declared service: each step resolves its own services through // `invokeSubTool` below (simulator-server for iOS/Android, CDP for - // Chromium), so run-sequence itself needs none. An eager resolver can't be - // used here because a tvOS udid shape-classifies as `ios` (there is no - // `tvos` platform) — declaring simulator-server for it would spawn a - // controller it can't drive and hang on the ready timeout before any tv-* - // step could run. The sub-tool invocations still pay only their own - // first-step spawn cost, and `ctx` is threaded through so nested steps keep - // the outer request's telemetry attribution. + // Chromium, CoreDevice for physical iOS), so run-sequence itself needs + // none. An eager resolver can't be used here because a tvOS udid + // shape-classifies as `ios` (there is no `tvos` platform) — declaring + // simulator-server for it would spawn a controller it can't drive and + // hang on the ready timeout before any tv-* step could run; a physical + // iOS udid hits the same problem, since simulator-server's guard throws + // for kind === "device" before step 1 even runs. The sub-tool invocations + // still pay only their own first-step spawn cost, and `ctx` is threaded + // through so nested steps keep the outer request's telemetry attribution. services: () => ({}), async execute(_services, params, ctx?: ToolContext) { const { udid, steps } = params; diff --git a/packages/tool-server/src/tools/screenshot-diff/index.ts b/packages/tool-server/src/tools/screenshot-diff/index.ts index 974cbaa5f..7c2eb2c1f 100644 --- a/packages/tool-server/src/tools/screenshot-diff/index.ts +++ b/packages/tool-server/src/tools/screenshot-diff/index.ts @@ -75,7 +75,7 @@ export interface ScreenshotDiffResult { type CaptureScreenshot = typeof httpScreenshot; const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/screenshot/index.ts b/packages/tool-server/src/tools/screenshot/index.ts index f95e92391..1837c7f76 100644 --- a/packages/tool-server/src/tools/screenshot/index.ts +++ b/packages/tool-server/src/tools/screenshot/index.ts @@ -6,7 +6,8 @@ import { z } from "zod"; import type { Registry, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; import { chromiumCdpRef, type ChromiumCdpApi } from "../../blueprints/chromium-cdp"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { getScreenshotScale, httpScreenshot } from "../../utils/simulator-client"; import { isTvOsSimulator } from "../../utils/ios-devices"; import { captureVegaScreenshotPng } from "../../utils/vega-screen"; @@ -120,11 +121,12 @@ export async function tvTargetLongSide(file: string, scale: number): Promise { return { id: "screenshot", - description: `Capture a screenshot of the device screen (iOS simulator, Android emulator, Apple TV simulator, Vega, or Chromium app). Returns { image }; the MCP adapter renders it as a visible image unless the caller passed includeImageInContext: false. + description: `Capture a screenshot of the device screen (iOS simulator, physical iPhone, Android emulator, Apple TV simulator, Vega, or Chromium app). Returns { image }; the MCP adapter renders it as a visible image unless the caller passed includeImageInContext: false. Use when you need a baseline image before an interaction or to inspect the current screen state after a delay. Fails if the simulator-server / emulator backend / Chromium CDP is not reachable for the given device.`, alwaysLoad: true, - searchHint: "device simulator emulator chromium screen image capture baseline tvos apple tv", + searchHint: + "device simulator emulator chromium screen image capture baseline tvos apple tv vega fire tv", zodSchema, outputHint: "image", capability, @@ -150,6 +152,19 @@ Fails if the simulator-server / emulator backend / Chromium CDP is not reachable return { image }; } + // A physical iOS device captures over CoreDevice (pymobiledevice3), not + // the simulator-server. CoreDevice returns a full-resolution PNG; + // rotation/scale/downscaler are simulator/Chromium-only knobs and don't + // apply to the device. Checked before the (async, simulator-only) tvOS + // probe below since a physical udid isn't a simulator runtime at all. + if (isPhysicalIos(device)) { + const ref = coreDeviceRef(device); + const coreDevice = (await registry.resolveService(ref.urn, ref.options)) as CoreDeviceApi; + const { path: capturedPath } = await coreDevice.screenshot(); + const image = await requireArtifacts(ctx).register(capturedPath, { mimeType: "image/png" }); + return { image }; + } + // Distinguish tvOS from iOS by simulator runtime — shape alone can't. // tvOS has no simulator-server backend, so capture via xcrun instead. if (device.platform === "ios" && (await isTvOsSimulator(params.udid))) { diff --git a/packages/tool-server/src/utils/device-info.ts b/packages/tool-server/src/utils/device-info.ts index 89994d38a..cb6fcd8a3 100644 --- a/packages/tool-server/src/utils/device-info.ts +++ b/packages/tool-server/src/utils/device-info.ts @@ -11,6 +11,17 @@ import type { DeviceInfo, DeviceKind, Platform } from "@argent/registry"; const IOS_UDID_SHAPE = /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$/; +/** + * Physical iPhone/iPad UDID shape on Apple silicon devices (A12+/iOS 17+): + * an 8-hex ECID prefix, a single dash, then 16 hex — e.g. + * `00008120-000E6D0C0ABBA01E`. This is distinct from the simulator UUID + * (four dashes) so a real device can be told apart from a simulator by shape + * alone, the same way Android emulators vs phones are distinguished. Older + * 40-hex device UDIDs belong to pre-A12 hardware that tops out well below the + * iOS 27 floor for the CoreDevice control path, so they are intentionally not matched. + */ +const IOS_PHYSICAL_UDID_SHAPE = /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}$/; + /** * Prefix used on device ids that route through `sim-remote` to a remote iOS * simulator. The raw UUID after the prefix is the same RFC-4122 shape as a @@ -31,6 +42,11 @@ export function withRemotePrefix(udid: string): string { export const CHROMIUM_ID_PREFIX = "chromium-cdp-"; +/** Whether a udid is a physical iOS device (vs a simulator UUID), by shape. */ +export function isPhysicalIosUdid(udid: string): boolean { + return IOS_PHYSICAL_UDID_SHAPE.test(udid); +} + /** * Vega serial prefix. `vega device list` reports VVD / Fire-TV serials as * `amazon-` (e.g. `amazon-4a27df03c9777152`). No *known* Android adb serial @@ -49,7 +65,8 @@ export function classifyDevice(udid: string): Platform { if (udid.startsWith(REMOTE_PREFIX)) return "ios-remote"; if (udid.startsWith(VEGA_SERIAL_PREFIX)) return "vega"; if (udid.startsWith(CHROMIUM_ID_PREFIX)) return "chromium"; - return IOS_UDID_SHAPE.test(udid) ? "ios" : "android"; + if (IOS_UDID_SHAPE.test(udid) || IOS_PHYSICAL_UDID_SHAPE.test(udid)) return "ios"; + return "android"; } /** @@ -72,8 +89,9 @@ export function isAndroidEmulatorSerial(serial: string): boolean { /** * Build a `DeviceInfo` from a raw udid, by shape. Kind defaults per platform: - * 'simulator' for iOS / ios-remote, 'vvd' for Vega, 'emulator'/'device' for - * Android by serial shape, 'app' for Chromium — platform impls can enrich with + * 'simulator' for an iOS simulator or ios-remote ('device' for a physical + * iPhone/iPad by UDID shape), 'vvd' for Vega, 'emulator'/'device' for Android + * by serial shape, 'app' for Chromium — platform impls can enrich with * name/state/sdkLevel via simctl/adb/sim-remote if needed. * * Vega is VVD-only in v1: the tool-server does not connect to or detect physical @@ -86,18 +104,27 @@ export function isAndroidEmulatorSerial(serial: string): boolean { export function resolveDevice(udid: string): DeviceInfo { const platform = classifyDevice(udid); const kind: DeviceKind = - platform === "ios" || platform === "ios-remote" - ? "simulator" - : platform === "vega" - ? "vvd" - : platform === "android" - ? isAndroidEmulatorSerial(udid) - ? "emulator" - : "device" - : "app"; + platform === "ios" + ? isPhysicalIosUdid(udid) + ? "device" + : "simulator" + : platform === "ios-remote" + ? "simulator" + : platform === "vega" + ? "vvd" + : platform === "android" + ? isAndroidEmulatorSerial(udid) + ? "emulator" + : "device" + : "app"; return { id: udid, platform, kind }; } +/** A physical iOS device (driven via CoreDevice/pymobiledevice3, not the simulator-server). */ +export function isPhysicalIos(device: DeviceInfo): boolean { + return device.platform === "ios" && device.kind === "device"; +} + /** Parses the CDP port out of a chromium device id. Returns null if the id is malformed. */ export function parseChromiumCdpPort(udid: string): number | null { if (!udid.startsWith(CHROMIUM_ID_PREFIX)) return null; diff --git a/packages/tool-server/src/utils/ios-devices.ts b/packages/tool-server/src/utils/ios-devices.ts index a2879af33..1c67c0692 100644 --- a/packages/tool-server/src/utils/ios-devices.ts +++ b/packages/tool-server/src/utils/ios-devices.ts @@ -1,5 +1,10 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readFile, rm } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { isPhysicalIosUdid } from "./device-info"; const execFileAsync = promisify(execFile); @@ -11,6 +16,25 @@ export interface IosSimulator { runtimeKind?: "mobile" | "tv"; } +export interface IosPhysicalDevice { + udid: string; + name: string; + /** Apple product type, e.g. "iPhone15,4". Null when devicectl omits it. */ + productType: string | null; + /** Always "connected" — only currently-reachable devices are returned. */ + state: string; +} + +interface DevicectlDevice { + hardwareProperties?: { udid?: string; platform?: string; productType?: string }; + deviceProperties?: { name?: string }; + connectionProperties?: { transportType?: string; tunnelState?: string }; +} + +interface DevicectlOutput { + result?: { devices?: DevicectlDevice[] }; +} + interface SimctlDevice { udid: string; name: string; @@ -56,6 +80,63 @@ export async function listIosSimulators(): Promise { } } +/** + * List connected physical iOS devices via `xcrun devicectl list devices`. + * + * devicectl only emits stable machine output to a file (`--json-output`), never + * stdout, so we write to a temp file and parse it. We keep only devices that are + * actually reachable right now: iOS platform with a `connectionProperties.transportType` + * (wired/network). Paired-but-offline devices carry a `tunnelState: "unavailable"` + * and no `transportType`, and are dropped — listing them would invite taps that + * can't land. Returns an empty array on any failure so the rest of `list-devices` + * stays usable on non-mac hosts or without Xcode. + */ +export function parsePhysicalIosDevices(data: DevicectlOutput): IosPhysicalDevice[] { + const out: IosPhysicalDevice[] = []; + for (const d of data.result?.devices ?? []) { + const udid = d.hardwareProperties?.udid; + const platform = d.hardwareProperties?.platform; + const transport = d.connectionProperties?.transportType; + // Keep only iOS (skip watchOS/tvOS), with a physical ECID UDID, that is + // currently reachable. The `isPhysicalIosUdid` (8hex-16hex) check is + // load-bearing: `devicectl list devices` also enumerates the host's iOS + // *simulators*, which report `platform: "iOS"` with + // `transportType: "sameMachine"` (verified against real devicectl JSON) — + // without the shape gate every simulator surfaces as a phantom physical + // device. It also keeps discovery consistent with `classifyDevice`, which + // routes only this UDID shape to the CoreDevice backend. A reachable device + // reports a `transportType` (wired/network); paired-but-offline ones carry + // `tunnelState: "unavailable"` and no transport, and are dropped. + if (!udid || platform !== "iOS" || !isPhysicalIosUdid(udid) || !transport) continue; + if (d.connectionProperties?.tunnelState === "unavailable") continue; + out.push({ + udid, + name: d.deviceProperties?.name ?? "iPhone", + productType: d.hardwareProperties?.productType ?? null, + state: "connected", + }); + } + return out; +} + +export async function listIosDevices(): Promise { + if (process.platform !== "darwin") return []; + const outPath = join(tmpdir(), `argent-devicectl-${randomUUID()}.json`); + try { + await execFileAsync( + "xcrun", + ["devicectl", "list", "devices", "--quiet", "--json-output", outPath], + { timeout: 15_000 } + ); + const data: DevicectlOutput = JSON.parse(await readFile(outPath, "utf8")); + return parsePhysicalIosDevices(data); + } catch { + return []; + } finally { + await rm(outPath, { force: true }).catch(() => {}); + } +} + // A simulator's runtime kind is fixed at creation (an iOS sim can't become a // tvOS one), so memoize per-UDID to keep the hot describe/screenshot path from // paying the ~100ms `simctl list` cost on every call. Only successful lookups diff --git a/packages/tool-server/src/utils/setup-registry.ts b/packages/tool-server/src/utils/setup-registry.ts index 32a8253b9..5b6551299 100644 --- a/packages/tool-server/src/utils/setup-registry.ts +++ b/packages/tool-server/src/utils/setup-registry.ts @@ -1,6 +1,7 @@ import { Registry } from "@argent/registry"; import { isFlagEnabled } from "@argent/configuration-core"; import { simulatorServerBlueprint } from "../blueprints/simulator-server"; +import { coreDeviceBlueprint } from "../blueprints/core-device"; import { nativeDevtoolsBlueprint } from "../blueprints/native-devtools"; import { androidDevtoolsBlueprint } from "../blueprints/android-devtools"; import { axServiceBlueprint } from "../blueprints/ax-service"; @@ -93,6 +94,7 @@ export function createRegistry(): Registry { const registry = new Registry({ isFlagEnabled: (flag) => isFlagEnabled(flag) }); registry.registerBlueprint(simulatorServerBlueprint); + registry.registerBlueprint(coreDeviceBlueprint); registry.registerBlueprint(jsRuntimeDebuggerBlueprint); registry.registerBlueprint(networkInspectorBlueprint); registry.registerBlueprint(reactProfilerSessionBlueprint); diff --git a/packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts b/packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts new file mode 100644 index 000000000..c240b683f --- /dev/null +++ b/packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts @@ -0,0 +1,154 @@ +/** + * The native-devtools tools (native-describe-screen, native-find-views, …) + * support both local iOS simulators (driven via `xcrun simctl spawn`) and + * REMOTE iOS simulators (routed through `sim-remote`, no local xcrun needed) — + * their capability declares `appleRemote: { simulator: true }`. + * + * A regression once put a *global* `requires: ["xcrun"]` on these tools. The + * HTTP dispatcher probes `ToolDefinition.requires` unconditionally (before + * execution, regardless of the resolved device), so a remote sim on an + * xcrun-less host got a bogus 424 Failed Dependency even though it never needs + * xcrun. The correct gating is per device kind, done inside `execute` + * (`sim-remote` for remote sims, `xcrun` for local sims) — mirroring how + * `describe` declares its deps per branch. + * + * This pins that contract: remote sims are NOT xcrun-gated, local sims still + * are, and remote sims are gated on the RIGHT binary (sim-remote). + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import request from "supertest"; +import { Registry, TypedEventEmitter } from "@argent/registry"; + +const execFileMock = vi.fn(); +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { ...actual, execFile: (...args: unknown[]) => execFileMock(...args) }; +}); + +const resolveAndroidBinaryMock = vi.fn(); +vi.mock("../src/utils/android-binary", () => ({ + resolveAndroidBinary: (name: "adb" | "emulator") => resolveAndroidBinaryMock(name), + __resetAndroidBinaryCacheForTesting: () => {}, +})); + +import { createHttpApp } from "../src/http"; +import { __resetDepCacheForTests } from "../src/utils/check-deps"; +import { nativeDescribeScreenTool } from "../src/tools/native-devtools/native-describe-screen"; +import { nativeDevtoolsBlueprint, type NativeDevtoolsApi } from "../src/blueprints/native-devtools"; + +// Controls which host binaries the real dep-check treats as available: any dep +// in `missing` fails its `command -v` probe, everything else resolves. +function stubProbe(missing: readonly string[]): void { + execFileMock.mockImplementation( + ( + _cmd: string, + args: string[], + _opts: unknown, + cb: (err: Error | null, stdout?: string, stderr?: string) => void + ) => { + const script = args[1] ?? ""; + const dep = script.replace("command -v ", "").trim(); + if (missing.includes(dep)) cb(new Error(`not found: ${dep}`)); + else cb(null, `/usr/bin/${dep}\n`, ""); + } + ); + resolveAndroidBinaryMock.mockImplementation(async (name: string) => + missing.includes(name) ? null : `/usr/bin/${name}` + ); +} + +// Minimal fake NativeDevtools api so `execute` completes without any real +// simctl / sim-remote / socket I/O. `requiresAppRestart: true` makes the +// precheck return `restart_required`, a clean terminal result — the point is +// only that the request reaches execution instead of being 424'd at the gate. +function makeStubApi(): NativeDevtoolsApi { + return { + isEnvSetup: () => true, + socketPath: "/tmp/stub.sock", + ensureEnvReady: async () => {}, + reverifyEnv: async () => {}, + getInitFailure: () => null, + isConnected: () => false, + isAppRunning: async () => false, + listConnectedBundleIds: () => [], + requiresAppRestart: async () => true, + activateNetworkInspection: () => {}, + getNetworkLog: () => [], + clearNetworkLog: () => {}, + getAppState: async () => { + throw new Error("unused in this test"); + }, + detectFrontmostBundleId: async () => null, + queryViewHierarchy: async () => ({}), + }; +} + +function makeRegistry(): Registry { + const registry = new Registry(); + // Reuse the real blueprint's namespace/getURN but swap in a no-I/O factory so + // service resolution can't hang or shell out — the dep gating we're testing + // is entirely separate from the service factory. + registry.registerBlueprint({ + ...nativeDevtoolsBlueprint, + factory: async () => ({ + api: makeStubApi(), + dispose: async () => {}, + events: new TypedEventEmitter(), + }), + }); + registry.registerTool(nativeDescribeScreenTool); + return registry; +} + +const REMOTE_UDID = "remote:12345678-1234-1234-1234-123456789012"; +const LOCAL_UDID = "12345678-1234-1234-1234-123456789012"; +// Physical iPhone UDID shape (8-hex ECID, single dash, 16 hex). +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; + +describe("native-devtools tools — per-device-kind dependency gating", () => { + beforeEach(() => { + __resetDepCacheForTests(); + execFileMock.mockReset(); + resolveAndroidBinaryMock.mockReset(); + }); + + it("does NOT block a REMOTE sim on missing xcrun (routes via sim-remote)", async () => { + stubProbe(["xcrun"]); // xcrun absent, sim-remote present + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: REMOTE_UDID, bundleId: "com.example.app" }); + // Reaches execution instead of the xcrun preflight 424. + expect(res.status).not.toBe(424); + expect(res.status).toBe(200); + }); + + it("still gates a LOCAL sim on xcrun (via the in-execute dep check)", async () => { + stubProbe(["xcrun"]); + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: LOCAL_UDID, bundleId: "com.example.app" }); + expect(res.status).toBe(424); + expect(res.body.missing).toEqual(["xcrun"]); + }); + + it("gates a REMOTE sim on the RIGHT binary — sim-remote, not xcrun", async () => { + stubProbe(["xcrun", "sim-remote"]); // both absent + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: REMOTE_UDID, bundleId: "com.example.app" }); + expect(res.status).toBe(424); + expect(res.body.missing).toEqual(["sim-remote"]); + }); + + it("rejects a PHYSICAL iOS device at the capability gate (path unaffected)", async () => { + stubProbe([]); // all deps present — must still be rejected before deps + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: PHYSICAL_UDID, bundleId: "com.example.app" }); + expect(res.status).toBe(400); + }); +}); diff --git a/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts b/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts new file mode 100644 index 000000000..38b12f74f --- /dev/null +++ b/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts @@ -0,0 +1,164 @@ +/** + * Coverage for the physical-iOS (CoreDevice) backend after it moved from a + * per-call `pymobiledevice3` CLI spawn to a persistent stdio sidecar: + * + * - `adaptCoreDeviceAxToDescribeResult` — the axAudit accessibility tree → + * describe adapter that backs `describe` on a real iPhone. Pins caption→role + * parsing, label cleanup, rect normalization, and frame interpolation for the + * elements the audit didn't rect (every frame stays in [0,1]). + * - `agentError` — the 9021 (iOS-27 host-input gate) message mapping. + * - `CoreDeviceAgent` — the stdio JSON protocol: ready handshake, id-correlated + * request/response, and error propagation (via a stand-in node process). + */ +import { describe, it, expect } from "vitest"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { FAILURE_CODES, getFailureSignal } from "@argent/registry"; +import { adaptCoreDeviceAxToDescribeResult } from "../src/tools/describe/platforms/ios/ios-coredevice-ax-adapter"; +import { agentError } from "../src/blueprints/core-device"; +import { CoreDeviceAgent, CoreDeviceAgentError } from "../src/blueprints/coredevice-agent"; + +interface Node { + role: string; + frame: { x: number; y: number; width: number; height: number }; + children: Node[]; + label?: string; +} +function flatten(n: Node, out: Node[] = []): Node[] { + out.push(n); + for (const c of n.children) flatten(c, out); + return out; +} +const center = (f: Node["frame"]) => ({ x: f.x + f.width / 2, y: f.y + f.height / 2 }); + +// A realistic axAudit snapshot: some elements carry an audit rect (points on a +// 393x852 screen), others don't (interpolated by the adapter). +const AXTREE = { + screen: { w: 393, h: 852 }, + elements: [ + { caption: "Settings, Button", id: "a1", rect: "{{318, 63}, {55, 36}}" }, + { caption: "Wi-Fi, Header", id: "a2", rect: "{{32, 168}, {55, 26}}" }, + { caption: "Wi-Fi, 1, Button, Toggle", id: "a3" }, // no rect -> interpolated + { caption: "Other…, Button", id: "a4", rect: "{{16, 553}, {361, 52}}" }, + { caption: "Known networks will be joined automatically.", id: "a5" }, // static text + ], +}; + +describe("adaptCoreDeviceAxToDescribeResult", () => { + const tree = adaptCoreDeviceAxToDescribeResult(AXTREE); + const nodes = flatten(tree as Node); + const byLabel = (l: string) => nodes.find((n) => n.label === l); + + it("parses roles from caption traits and strips them from the label", () => { + expect(byLabel("Settings")?.role).toBe("AXButton"); + expect(byLabel("Wi-Fi")?.role).toBe("AXHeader"); + // Button trait wins the role; trailing Button/Toggle stripped from the label. + expect(byLabel("Wi-Fi, 1")?.role).toBe("AXButton"); + // No trait -> static text, full caption kept as label. + const stat = nodes.find((n) => n.label?.startsWith("Known networks")); + expect(stat?.role).toBe("AXStaticText"); + }); + + it("normalizes an audited rect (points) into a [0,1] frame", () => { + const other = byLabel("Other…")!; + // {{16, 553}, {361, 52}} on 393x852 + expect(other.frame.x).toBeCloseTo(16 / 393, 3); + expect(other.frame.y).toBeCloseTo(553 / 852, 3); + expect(other.frame.width).toBeCloseTo(361 / 393, 3); + }); + + it("interpolates a rect-less element between its neighbours (reading order)", () => { + const wifiHeader = center(byLabel("Wi-Fi")!.frame).y; // ~168/852 + const other = center(byLabel("Other…")!.frame).y; // ~553/852 + const toggle = center(byLabel("Wi-Fi, 1")!.frame).y; // no rect, between the two + expect(toggle).toBeGreaterThan(wifiHeader); + expect(toggle).toBeLessThan(other); + }); + + it("keeps every frame within the normalized [0,1] box", () => { + for (const n of nodes) { + const { x, y, width, height } = n.frame; + for (const v of [x, y, width, height]) { + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThanOrEqual(1); + } + expect(x + width).toBeLessThanOrEqual(1.0001); + expect(y + height).toBeLessThanOrEqual(1.0001); + } + }); + + it("does not throw on an empty / screen-less tree", () => { + expect(() => adaptCoreDeviceAxToDescribeResult({ elements: [] })).not.toThrow(); + expect(() => + adaptCoreDeviceAxToDescribeResult({ + elements: [{ caption: "x", id: "1" }], + }) + ).not.toThrow(); + }); +}); + +describe("agentError — iOS-27 host-input gate (CoreDeviceError 9021)", () => { + it("maps a gated agent error to the actionable iOS-27 message", () => { + const e = agentError("tap", new CoreDeviceAgentError("… CoreDeviceError 9021 …", true)); + expect(e.message).toContain("requires iOS 27+"); + expect(getFailureSignal(e)?.error_code).toBe(FAILURE_CODES.CORE_DEVICE_IOS_VERSION_TOO_OLD); + }); + + it("maps a non-gated agent error to a generic command failure", () => { + const e = agentError("swipe", new CoreDeviceAgentError("some other failure", false)); + expect(e.message).toContain("CoreDevice swipe failed"); + expect(e.message).not.toContain("iOS 27"); + expect(getFailureSignal(e)?.error_code).toBe(FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED); + }); + + it("maps a plain Error to a command failure", () => { + const e = agentError("button", new Error("boom")); + expect(getFailureSignal(e)?.error_code).toBe(FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED); + }); +}); + +describe("CoreDeviceAgent — stdio JSON protocol", () => { + // A stand-in for the python agent: emits the ready handshake, echoes ops, and + // returns a gated error for op "fail". Run with node so the test needs no + // device or pymobiledevice3. + const mock = join(tmpdir(), `argent-mock-coredevice-agent-${process.pid}.cjs`); + writeFileSync( + mock, + `const rl = require("readline").createInterface({ input: process.stdin }); +process.stdout.write(JSON.stringify({ ready: true }) + "\\n"); +rl.on("line", (l) => { + const m = JSON.parse(l); + if (m.op === "fail") { + process.stdout.write(JSON.stringify({ id: m.id, error: "CoreDeviceError 9021", gated_9021: true }) + "\\n"); + } else { + process.stdout.write(JSON.stringify({ id: m.id, ok: true, echo: m.op }) + "\\n"); + } +}); +` + ); + + it("handshakes, correlates responses by id, and propagates errors", async () => { + const agent = new CoreDeviceAgent(process.execPath, mock, "UDID", 49151, 5000); + await agent.start(); + try { + const r = await agent.request("ping"); + expect(r.ok).toBe(true); + expect(r.echo).toBe("ping"); + + await expect(agent.request("fail")).rejects.toMatchObject({ + name: "CoreDeviceAgentError", + gated9021: true, + }); + } finally { + agent.dispose(); + } + }); + + it("rejects a request made after dispose", async () => { + const agent = new CoreDeviceAgent(process.execPath, mock, "UDID", 49151, 5000); + await agent.start(); + agent.dispose(); + await expect(agent.request("ping")).rejects.toThrow(); + }); +}); diff --git a/packages/tool-server/test/physical-ios-followups.test.ts b/packages/tool-server/test/physical-ios-followups.test.ts new file mode 100644 index 000000000..17517e8a0 --- /dev/null +++ b/packages/tool-server/test/physical-ios-followups.test.ts @@ -0,0 +1,364 @@ +/** + * Follow-up coverage for the physical-iOS (CoreDevice) feature. These tests pin + * behaviors that the original physical-ios.test.ts left uncovered and that a + * regression could silently break: + * + * - discovery must NOT surface the host's iOS simulators as phantom physical + * devices (devicectl enumerates them with transportType "sameMachine"); + * - the `button` CoreDevice HID mapping + rejection of buttons with no HID; + * - the privileged-tunnel flag gate (root escalation must be opt-in); + * - tools that are unsupported on physical iOS reject with a 400-mapped + * UnsupportedOperationError, not a generic 500 (open-url/reinstall/restart, + * describe, native-profiler), while staying supported on simulators/Android; + * - run-sequence must not eagerly hold simulator-server for a physical iPhone; + * - swipe duration clamping + timeout scaling. + */ +import { describe, it, expect, vi } from "vitest"; + +// core-device is the only module under test that reads the feature flag; mock it +// so the flag gate can be exercised deterministically regardless of the host's +// ~/.argent/flags.json. (See variant-flag-gate.test.ts for the same pattern.) +vi.mock("@argent/configuration-core", () => ({ isFlagEnabled: vi.fn() })); +import { isFlagEnabled } from "@argent/configuration-core"; + +import { resolveDevice, isPhysicalIosUdid } from "../src/utils/device-info"; +import { parsePhysicalIosDevices } from "../src/utils/ios-devices"; +import { UnsupportedOperationError, assertSupported } from "../src/utils/capability"; +import { + swipeDragParams, + ensureCoreDeviceTunnel, + assertPhysicalIosEnabled, +} from "../src/blueprints/core-device"; +import { buttonTool } from "../src/tools/button"; +import { createRunSequenceTool } from "../src/tools/run-sequence"; +import { describeIos } from "../src/tools/describe/platforms/ios"; +import { iosImpl as openUrlIos } from "../src/tools/open-url/platforms/ios"; +import { iosImpl as reinstallIos } from "../src/tools/reinstall-app/platforms/ios"; +import { makeIosImpl as makeRestartAppIosImpl } from "../src/tools/restart-app/platforms/ios"; +import { makeIosImpl as makeLaunchAppIosImpl } from "../src/tools/launch-app/platforms/ios"; +import { gestureSwipeTool } from "../src/tools/gesture-swipe"; +import { gestureTapTool } from "../src/tools/gesture-tap"; +import { createKeyboardTool } from "../src/tools/keyboard"; +import { gesturePinchTool } from "../src/tools/gesture-pinch"; +import { screenshotDiffTool } from "../src/tools/screenshot-diff"; +import { nativeDescribeScreenTool } from "../src/tools/native-devtools/native-describe-screen"; +import { nativeProfilerStartTool } from "../src/tools/profiler/native-profiler/native-profiler-start"; + +const mockFlag = vi.mocked(isFlagEnabled); + +// Physical-iOS branches of both handlers throw/reject before ever touching +// `registry` (see the assertions below), so a stub registry is safe here. +const restartIos = makeRestartAppIosImpl({} as never); +const launchAppIos = makeLaunchAppIosImpl({} as never); +const keyboardTool = createKeyboardTool({} as never); + +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; +const SIM_UDID = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; + +describe("discovery does not surface simulators as physical devices", () => { + // Real `xcrun devicectl list devices` JSON also lists every host iOS + // simulator (platform "iOS", transportType "sameMachine"); without the + // UDID-shape gate those leak in as phantom physical devices. + const data = { + result: { + devices: [ + // The real, connected iPhone — KEPT. + { + hardwareProperties: { udid: PHYSICAL_UDID, platform: "iOS", productType: "iPhone15,4" }, + deviceProperties: { name: "Real iPhone" }, + connectionProperties: { transportType: "wired", tunnelState: "disconnected" }, + }, + // A booted iOS simulator (UUID shape, sameMachine transport) — DROPPED. + { + hardwareProperties: { udid: SIM_UDID, platform: "iOS", productType: "iPhone17,2" }, + deviceProperties: { name: "iPhone 16 Pro Max" }, + connectionProperties: { transportType: "sameMachine", tunnelState: "connected" }, + }, + // A shut-down simulator (UUID shape) — DROPPED (would otherwise be + // reported as a "connected" device). + { + hardwareProperties: { + udid: "39646432-58B6-4A21-923A-00F0EDE4FF81", + platform: "iOS", + productType: "iPhone17,3", + }, + deviceProperties: { name: "iPhone 16" }, + connectionProperties: { transportType: "sameMachine", tunnelState: "disconnected" }, + }, + // A paired-but-offline real device (physical shape, no transport) — DROPPED. + { + hardwareProperties: { + udid: "00008030-00096526219B802E", + platform: "iOS", + productType: "iPhone12,8", + }, + deviceProperties: { name: "Old iPhone" }, + connectionProperties: { tunnelState: "unavailable" }, + }, + ], + }, + }; + + it("returns only the real physical iPhone", () => { + const out = parsePhysicalIosDevices(data); + expect(out).toEqual([ + { udid: PHYSICAL_UDID, name: "Real iPhone", productType: "iPhone15,4", state: "connected" }, + ]); + }); + + it("every returned device has a physical-shape UDID", () => { + for (const d of parsePhysicalIosDevices(data)) { + expect(isPhysicalIosUdid(d.udid)).toBe(true); + } + }); +}); + +describe("button — CoreDevice HID mapping on physical iOS", () => { + const press = async (button: string) => { + const coreDevice = { button: vi.fn().mockResolvedValue(undefined) }; + const res = await buttonTool.execute( + { coreDevice } as never, + { + udid: PHYSICAL_UDID, + button, + } as never + ); + return { coreDevice, res }; + }; + + it("maps the four supported buttons to their pymobiledevice3 HID names", async () => { + for (const [argent, hid] of [ + ["home", "home"], + ["power", "lock"], + ["volumeUp", "volume-up"], + ["volumeDown", "volume-down"], + ]) { + const { coreDevice, res } = await press(argent); + expect(coreDevice.button).toHaveBeenCalledWith(hid); + expect(res).toEqual({ pressed: argent }); + } + }); + + it("rejects buttons with no CoreDevice HID equivalent", async () => { + // appSwitch / actionButton exist on iOS but have no HID button. + await expect(press("appSwitch")).rejects.toBeInstanceOf(UnsupportedOperationError); + await expect(press("actionButton")).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + it("does not resolve the CoreDevice service for a button with no HID equivalent", () => { + // services() runs before execute() (the registry resolves services first), + // so eagerly resolving coreDevice here would pay for tunnel setup — + // possibly a macOS admin prompt — just to reject the button afterward. + for (const button of ["appSwitch", "actionButton"]) { + const services = buttonTool.services!({ udid: PHYSICAL_UDID, button } as never); + expect(services.coreDevice).toBeUndefined(); + } + }); + + it("still resolves the CoreDevice service for a supported button", () => { + const services = buttonTool.services!({ udid: PHYSICAL_UDID, button: "home" } as never); + expect(services.coreDevice).toBeDefined(); + }); +}); + +describe("privileged tunnel start is gated behind the feature flag", () => { + it("rejects with the enable hint when physical-ios-devices is off", async () => { + mockFlag.mockReturnValue(false); + await expect(ensureCoreDeviceTunnel(PHYSICAL_UDID)).rejects.toThrow( + /Physical iOS support is disabled.*argent enable physical-ios-devices/s + ); + expect(mockFlag).toHaveBeenCalledWith("physical-ios-devices"); + }); +}); + +describe("launch-app enforces the physical-iOS flag (no bypass)", () => { + // launch-app drives a real device via `devicectl` directly (not the + // CoreDevice service), so unlike screenshot/tap/swipe it must enforce the + // opt-in itself — otherwise it would be the one physical-iOS operation + // reachable while the feature is disabled. + it("assertPhysicalIosEnabled throws when the flag is off, not when on", () => { + mockFlag.mockReturnValue(false); + expect(() => assertPhysicalIosEnabled()).toThrow(/Physical iOS support is disabled/); + mockFlag.mockReturnValue(true); + expect(() => assertPhysicalIosEnabled()).not.toThrow(); + }); + + it("launch-app rejects a physical iPhone when the flag is off (before shelling devicectl)", async () => { + mockFlag.mockReturnValue(false); + await expect( + launchAppIos.handler( + {} as never, + { udid: PHYSICAL_UDID, bundleId: "com.apple.Preferences" } as never, + resolveDevice(PHYSICAL_UDID) + ) + ).rejects.toThrow(/Physical iOS support is disabled.*argent enable physical-ios-devices/s); + }); +}); + +describe("gesture-swipe routes physical iOS to the CoreDevice backend", () => { + it("forwards normalized coords and duration to coreDevice.swipe", async () => { + const coreDevice = { swipe: vi.fn().mockResolvedValue(undefined) }; + const res = await gestureSwipeTool.execute( + { coreDevice } as never, + { + udid: PHYSICAL_UDID, + fromX: 0.5, + fromY: 0.7, + toX: 0.5, + toY: 0.3, + durationMs: 250, + } as never + ); + expect(coreDevice.swipe).toHaveBeenCalledWith(0.5, 0.7, 0.5, 0.3, 250); + expect(res).toMatchObject({ swiped: true }); + }); + + it("defaults the duration to 300ms when omitted", async () => { + const coreDevice = { swipe: vi.fn().mockResolvedValue(undefined) }; + await gestureSwipeTool.execute( + { coreDevice } as never, + { + udid: PHYSICAL_UDID, + fromX: 0.2, + fromY: 0.2, + toX: 0.8, + toY: 0.8, + } as never + ); + expect(coreDevice.swipe).toHaveBeenCalledWith(0.2, 0.2, 0.8, 0.8, 300); + }); +}); + +describe("tools unsupported on physical iOS reject with UnsupportedOperationError (400)", () => { + const device = resolveDevice(PHYSICAL_UDID); + + it("open-url", async () => { + await expect( + openUrlIos.handler({} as never, { udid: PHYSICAL_UDID, url: "https://x" } as never, device) + ).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + it("reinstall-app", async () => { + await expect( + reinstallIos.handler( + {} as never, + { udid: PHYSICAL_UDID, bundleId: "com.x", appPath: "/tmp/x.app" } as never, + device + ) + ).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + it("restart-app", async () => { + await expect( + restartIos.handler({} as never, { udid: PHYSICAL_UDID, bundleId: "com.x" } as never, device) + ).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + // describe is NOT in the unsupported set: on a physical iPhone it returns the + // real on-screen accessibility tree via the CoreDevice axtree() snapshot + // (works in-app and on the home screen). A stub service stands in here. + it("describe — returns the CoreDevice accessibility tree (source coredevice-ax), not a rejection", async () => { + const registry = { + resolveService: async () => ({ + axtree: async () => ({ + screen: { w: 393, h: 852 }, + elements: [ + { caption: "General, Button", id: "e1", rect: "{{16, 100}, {361, 44}}" }, + { caption: "Accessibility, Button", id: "e2" }, + ], + }), + }), + }; + const result = await describeIos(registry as never, device, {}); + expect(result.source).toBe("coredevice-ax"); + const flat = JSON.stringify(result.tree); + expect(flat).toContain("General"); + expect(flat).toContain("Accessibility"); + expect(result.hint).toContain("screenshot"); + }); +}); + +describe("run-sequence does not eagerly hold simulator-server for physical iOS", () => { + // run-sequence never eagerly declares any service (each step resolves its own + // via invokeSubTool) — a physical iOS udid must not eagerly hold + // simulator-server, which would throw on a `kind === "device"` target before + // step 1 even runs. Simulators go through the same lazy path. + const tool = createRunSequenceTool({} as never); + const params = (udid: string) => ({ udid, steps: [{ tool: "gesture-tap", args: {} }] }); + + it("holds no simulator-server service for a physical iPhone", () => { + const services = tool.services(params(PHYSICAL_UDID)); + expect(services.simulatorServer).toBeUndefined(); + expect(Object.keys(services)).toHaveLength(0); + }); + + it("holds no simulator-server service for a simulator either", () => { + const services = tool.services(params(SIM_UDID)); + expect(services.simulatorServer).toBeUndefined(); + expect(Object.keys(services)).toHaveLength(0); + }); +}); + +describe("capability matrix is honest about physical-iOS support (clean 400 at the gate)", () => { + const physical = resolveDevice(PHYSICAL_UDID); + const sim = resolveDevice(SIM_UDID); + const androidEmu = resolveDevice("emulator-5554"); + + it("supported tools accept a physical iPhone", () => { + expect(() => assertSupported("gesture-tap", gestureTapTool.capability, physical)).not.toThrow(); + expect(() => assertSupported("button", buttonTool.capability, physical)).not.toThrow(); + }); + + it("simulator-only tools reject a physical iPhone via the capability gate", () => { + for (const [id, cap] of [ + ["keyboard", keyboardTool.capability], + ["gesture-pinch", gesturePinchTool.capability], + ["screenshot-diff", screenshotDiffTool.capability], + ["native-describe-screen", nativeDescribeScreenTool.capability], + // native-profiler-start does LIVE capture via simulator-only simctl (the + // process enumeration mislabels a real iPhone as a "simulator"), so it + // rejects physical iOS at the gate. (Its post-capture sibling tools stay + // device-agnostic — see profiler-query-android-capability.test.ts.) + ["native-profiler-start", nativeProfilerStartTool.capability], + ] as const) { + expect(() => assertSupported(id, cap, physical)).toThrow(UnsupportedOperationError); + // ...but still work on a simulator (no regression to simulator support). + expect(() => assertSupported(id, cap, sim)).not.toThrow(); + } + }); + + it("native-profiler-start still accepts a physical Android device", () => { + expect(() => + assertSupported("native-profiler-start", nativeProfilerStartTool.capability, androidEmu) + ).not.toThrow(); + }); +}); + +describe("swipeDragParams — clamping and timeout scaling", () => { + it("clamps a typical swipe and scales the timeout past the drag duration", () => { + const p = swipeDragParams(300); + expect(p.seconds).toBe("0.300"); + expect(p.steps).toBe(19); + expect(p.timeoutMs).toBe(15_300); + }); + + it("a long swipe gets a timeout that outlasts the drag (the bug this fixes)", () => { + const p = swipeDragParams(20_000); + expect(p.seconds).toBe("20.000"); + expect(p.steps).toBe(60); // step count is capped + expect(p.timeoutMs).toBe(35_000); + expect(p.timeoutMs).toBeGreaterThan(20_000); + }); + + it("floors degenerate (zero/negative) durations to a real dwell", () => { + expect(swipeDragParams(0).seconds).toBe("0.050"); + expect(swipeDragParams(-100).seconds).toBe("0.050"); + expect(swipeDragParams(0).steps).toBeGreaterThanOrEqual(2); + }); + + it("caps a pathological duration", () => { + const p = swipeDragParams(10_000_000); + expect(p.seconds).toBe("60.000"); + expect(p.timeoutMs).toBe(75_000); + }); +}); diff --git a/packages/tool-server/test/physical-ios.test.ts b/packages/tool-server/test/physical-ios.test.ts new file mode 100644 index 000000000..39fbcbff9 --- /dev/null +++ b/packages/tool-server/test/physical-ios.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect } from "vitest"; +import { + classifyDevice, + resolveDevice, + isPhysicalIos, + isPhysicalIosUdid, +} from "../src/utils/device-info"; +import { parsePhysicalIosDevices } from "../src/utils/ios-devices"; +import { toHid, tunneldStartCommand, appleScriptQuote } from "../src/blueprints/core-device"; +import { createLaunchAppTool } from "../src/tools/launch-app"; +import { createRestartAppTool } from "../src/tools/restart-app"; +import { devicesToPreviewEntries } from "../src/preview"; +import type { ListDevicesResult } from "../src/tools/devices/list-devices"; + +// A real iPhone's UDID: 8-hex ECID, one dash, 16 hex. +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; +const SIM_UDID = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; + +describe("physical iOS classification", () => { + it("classifies a physical iPhone UDID as ios", () => { + expect(classifyDevice(PHYSICAL_UDID)).toBe("ios"); + }); + + it("resolves a physical iPhone UDID to ios + device", () => { + const d = resolveDevice(PHYSICAL_UDID); + expect(d.platform).toBe("ios"); + expect(d.kind).toBe("device"); + expect(d.id).toBe(PHYSICAL_UDID); + }); + + it("still resolves a simulator UUID to ios + simulator", () => { + const d = resolveDevice(SIM_UDID); + expect(d.platform).toBe("ios"); + expect(d.kind).toBe("simulator"); + }); + + it("isPhysicalIosUdid distinguishes device from simulator shapes", () => { + expect(isPhysicalIosUdid(PHYSICAL_UDID)).toBe(true); + expect(isPhysicalIosUdid(SIM_UDID)).toBe(false); + }); + + it("isPhysicalIos is true only for ios+device", () => { + expect(isPhysicalIos(resolveDevice(PHYSICAL_UDID))).toBe(true); + expect(isPhysicalIos(resolveDevice(SIM_UDID))).toBe(false); + expect(isPhysicalIos(resolveDevice("emulator-5554"))).toBe(false); + }); + + it("does not reclassify Android serials as physical iOS", () => { + // No 8hex-16hex Android serials in the wild; guard against regressions. + expect(classifyDevice("HT82A0203045")).toBe("android"); + expect(classifyDevice("192.168.1.5:5555")).toBe("android"); + expect(resolveDevice("HT82A0203045").kind).toBe("device"); + }); +}); + +describe("parsePhysicalIosDevices (devicectl JSON)", () => { + const sample = { + result: { + devices: [ + // Connected iPhone — kept. + { + hardwareProperties: { udid: PHYSICAL_UDID, platform: "iOS", productType: "iPhone15,4" }, + deviceProperties: { name: "iPhone 15" }, + connectionProperties: { transportType: "wired", tunnelState: "disconnected" }, + }, + // Paired but offline (no transportType, tunnelState unavailable) — dropped. + { + hardwareProperties: { + udid: "00008030-00096526219B802E", + platform: "iOS", + productType: "iPhone12,8", + }, + deviceProperties: { name: "Old iPhone" }, + connectionProperties: { tunnelState: "unavailable" }, + }, + // A connected Apple Watch — dropped (not iOS). + { + hardwareProperties: { + udid: "11111111-2222222222222222", + platform: "watchOS", + productType: "Watch7,1", + }, + deviceProperties: { name: "Watch" }, + connectionProperties: { transportType: "wired" }, + }, + ], + }, + }; + + it("keeps only connected iOS devices with the right fields", () => { + const out = parsePhysicalIosDevices(sample); + expect(out).toEqual([ + { udid: PHYSICAL_UDID, name: "iPhone 15", productType: "iPhone15,4", state: "connected" }, + ]); + }); + + it("returns [] for empty/missing input", () => { + expect(parsePhysicalIosDevices({})).toEqual([]); + expect(parsePhysicalIosDevices({ result: { devices: [] } })).toEqual([]); + }); +}); + +describe("toHid (normalized 0..1 → 0..65535)", () => { + it("maps endpoints and midpoint", () => { + expect(toHid(0)).toBe(0); + expect(toHid(1)).toBe(65535); + expect(toHid(0.5)).toBe(32768); + }); + + it("clamps out-of-range input", () => { + expect(toHid(-0.2)).toBe(0); + expect(toHid(1.5)).toBe(65535); + }); +}); + +describe("privileged tunnel start command", () => { + it("single-quotes the binary path, pins HOME, passes the port + daemonize flag", () => { + const cmd = tunneldStartCommand("/Users/me/.local/bin/pymobiledevice3", 49151); + expect(cmd).toContain("HOME=/var/root"); + expect(cmd).toContain("'/Users/me/.local/bin/pymobiledevice3' remote tunneld --port 49151 -d"); + }); + + it("escapes a single quote in the path so it can't break out of the sh word", () => { + const cmd = tunneldStartCommand("/Users/o'brien/pmd3", 5000); + // ' -> '\'' so the whole path stays one shell word + expect(cmd).toContain(`'/Users/o'\\''brien/pmd3' remote tunneld`); + }); + + it("escapes backslashes and double-quotes for the AppleScript string literal", () => { + expect(appleScriptQuote('a "b" c')).toBe('a \\"b\\" c'); + expect(appleScriptQuote("a\\b")).toBe("a\\\\b"); + }); +}); + +describe("lifecycle tools don't eagerly resolve native-devtools for local iOS", () => { + // Regression guard: the registry resolves a tool's services() BEFORE execute(), + // and the native-devtools service throws a simulator-only guard for physical + // devices. So eagerly resolving it for a physical iPhone would break launch-app + // (a supported tool) and mask restart-app's intended rejection message. Local + // iOS (simulator and physical alike) resolves native-devtools lazily inside the + // handler instead — only ios-remote declares it eagerly via services(). + const launchAppTool = createLaunchAppTool({} as never); + const restartAppTool = createRestartAppTool({} as never); + const params = (udid: string) => ({ udid, bundleId: "com.apple.Preferences" }); + + it("launch-app: omitted for both physical device and simulator", () => { + expect(launchAppTool.services(params(PHYSICAL_UDID)).nativeDevtools).toBeUndefined(); + expect(launchAppTool.services(params(SIM_UDID)).nativeDevtools).toBeUndefined(); + }); + + it("restart-app: omitted for both physical device and simulator", () => { + expect(restartAppTool.services(params(PHYSICAL_UDID)).nativeDevtools).toBeUndefined(); + expect(restartAppTool.services(params(SIM_UDID)).nativeDevtools).toBeUndefined(); + }); +}); + +describe("preview target list excludes targets it can't stream", () => { + // Regression guard: the preview / Lens UI streams frames over simulator-server, + // which refuses physical iOS (kind === "device", driven over CoreDevice) and + // never serves Chromium. Those must not leak into the selectable target list. + const devices: ListDevicesResult["devices"] = [ + { + platform: "ios", + udid: SIM_UDID, + name: "iPhone 16 Pro", + state: "Booted", + kind: "simulator", + runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-5", + }, + { + platform: "ios", + udid: PHYSICAL_UDID, + name: "iPhone 15", + state: "connected", + kind: "device", + productType: "iPhone15,4", + }, + { + platform: "android", + serial: "emulator-5554", + state: "device", + isEmulator: true, + kind: "emulator", + model: "Pixel 7", + avdName: "Pixel_7_API_33", + sdkLevel: 34, + }, + { + platform: "chromium", + id: "chromium-1", + title: "App", + port: 9222, + url: "about:blank", + browser: "Chrome/120", + state: "Running", + }, + ]; + + it("includes the simulator and the Android emulator", () => { + const entries = devicesToPreviewEntries(devices); + expect(entries.map((e) => e.udid).sort()).toEqual([SIM_UDID, "emulator-5554"].sort()); + }); + + it("excludes the physical iPhone and the Chromium app", () => { + const entries = devicesToPreviewEntries(devices); + expect(entries.some((e) => e.udid === PHYSICAL_UDID)).toBe(false); + expect(entries.some((e) => e.platform !== "ios" && e.platform !== "android")).toBe(false); + }); + + it("keeps `runtime` a string for a simulator that reports one", () => { + const [sim] = devicesToPreviewEntries(devices); + expect(typeof sim!.runtime).toBe("string"); + expect(sim!.runtime.length).toBeGreaterThan(0); + }); +}); diff --git a/scripts/download-native-binaries.sh b/scripts/download-native-binaries.sh index 534f5afac..181166f6a 100755 --- a/scripts/download-native-binaries.sh +++ b/scripts/download-native-binaries.sh @@ -63,6 +63,23 @@ gh release download "${TAG}" \ --clobber chmod +x "${IOS_BIN_DIR}/ax-service" +# argent-device-auth is the macOS host helper that shows the branded admin +# prompt to start the physical-iOS CoreDevice tunnel as root. Like ax-service it +# is a darwin-only binary, resolved at bin/darwin/argent-device-auth. +# Optional: until the argent-private build publishes it, the release won't carry +# it — physical-iOS then falls back to the (unbranded) osascript admin prompt, +# so a missing asset must not fail the whole download. +echo " Downloading argent-device-auth..." +if gh release download "${TAG}" \ + --repo "${REPO}" \ + --pattern "argent-device-auth" \ + --dir "${IOS_BIN_DIR}" \ + --clobber 2>/dev/null; then + chmod +x "${IOS_BIN_DIR}/argent-device-auth" +else + echo " (not in this release yet — physical iOS will use the osascript prompt fallback)" +fi + # tvOS binaries (Apple TV support). The three tvOS injection dylibs share their # filenames with the iOS dylibs, so the release ships them as a tarball # (native-devtools-ios-tvos-dylibs.tar.gz) that we extract into dylibs/tvos/ — @@ -223,6 +240,7 @@ if command -v codesign &>/dev/null; then "${TVOS_DYLIBS_DIR}"/*.dylib \ "${TCP_DYLIBS_DIR}"/*.dylib \ "${IOS_BIN_DIR}/ax-service" \ + "${IOS_BIN_DIR}/argent-device-auth" \ "${IOS_BIN_DIR}/tvos-ax-service" \ "${IOS_BIN_DIR}/tvos-hid-daemon" \ "${TCP_BIN_DIR}/ax-service"; do