From 6710eef2bd72b3641a4ac753d5566e883d674237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Thu, 18 Jun 2026 13:40:09 +0200 Subject: [PATCH 01/16] feat(ios): control physical iOS devices over CoreDevice (experimental) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive a real iPhone (iOS 27+) app-free via Apple's CoreDevice remote-control services through pymobiledevice3 — no on-device app, no WebDriverAgent. Gated behind the `physical-ios-devices` flag. - Discovery: classify physical UDIDs (8hex-16hex) as ios/device; list connected iPhones via `xcrun devicectl`, flag-gated. - CoreDevice backend (blueprints/core-device.ts): resolves the tunneld RSD endpoint, ensures the DDI is mounted, shells to pymobiledevice3 for screenshot, tap (dwell-drag — a zero-dwell tap is ignored by iOS), swipe, and buttons. - Branded privilege escalation: the tunnel needs root once to create its network interface. Instead of manual sudo, Argent auto-starts it via a signed macOS host helper (argent-device-auth) that shows the standard auth modal branded as "Argent" with the Argent icon (Authorization Services + dlsym of AuthorizationExecuteWithPrivileges). Falls back to the osascript admin prompt, then manual-sudo instructions (headless). HOME is pinned so tunneld finds its pairing records under the privileged-exec environment. - Wire screenshot/gesture-tap/gesture-swipe/button to the backend; launch-app via devicectl; boot-device starts the tunnel (branded prompt). Unsupported tools on physical iOS are cleanly rejected (simulator-server / ax-service / native-devtools guards; open-url/reinstall/restart handlers). - Helper resolvers + Argent icon in @argent/native-devtools-ios; download-native-binaries.sh fetches the signed helper (optional until published); bundle-tools copies it for distribution. Helper source + signing live in argent-private. - Tests for classification, devicectl parsing, coord mapping, and privileged command building. Overrides: ARGENT_PYMOBILEDEVICE3, ARGENT_PMD3_TUNNELD_PORT, ARGENT_DEVICE_AUTH_HELPER, ARGENT_DEVICE_ICON. --- README.md | 41 ++ packages/argent/scripts/bundle-tools.cjs | 32 ++ packages/configuration-core/src/flags.ts | 5 + .../assets/argent-icon.png | Bin 0 -> 4058 bytes packages/native-devtools-ios/src/index.ts | 24 + .../tool-server/src/blueprints/ax-service.ts | 8 + .../tool-server/src/blueprints/core-device.ts | 464 ++++++++++++++++++ .../src/blueprints/native-devtools.ts | 8 + .../src/blueprints/simulator-server.ts | 12 + .../tool-server/src/tools/button/index.ts | 39 +- .../src/tools/devices/boot-device.ts | 13 + .../src/tools/devices/list-devices.ts | 58 ++- .../src/tools/gesture-swipe/index.ts | 21 +- .../src/tools/gesture-tap/index.ts | 11 +- .../tool-server/src/tools/launch-app/index.ts | 8 +- .../src/tools/launch-app/platforms/ios.ts | 17 +- .../src/tools/open-url/platforms/ios.ts | 8 +- .../src/tools/reinstall-app/platforms/ios.ts | 7 +- .../src/tools/restart-app/index.ts | 8 +- .../src/tools/restart-app/platforms/ios.ts | 5 +- .../tool-server/src/tools/screenshot/index.ts | 14 +- packages/tool-server/src/utils/device-info.ts | 28 +- packages/tool-server/src/utils/ios-devices.ts | 73 +++ .../tool-server/src/utils/setup-registry.ts | 2 + .../tool-server/test/physical-ios.test.ts | 150 ++++++ scripts/download-native-binaries.sh | 20 +- 26 files changed, 1045 insertions(+), 31 deletions(-) create mode 100644 packages/native-devtools-ios/assets/argent-icon.png create mode 100644 packages/tool-server/src/blueprints/core-device.ts create mode 100644 packages/tool-server/test/physical-ios.test.ts diff --git a/README.md b/README.md index 969ee819b..a13e6b6cf 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,47 @@ npx @swmansion/argent init --- +## Physical iOS devices (experimental) + +Argent can drive a **physical iPhone** — no app installed on the device — over Apple's +CoreDevice "remote control" services (the same path Xcode's device window uses), via +[`pymobiledevice3`](https://github.com/doronz88/pymobiledevice3). Supported interactions: +`screenshot`, `gesture-tap`, `gesture-swipe`, `button`, and `launch-app`. The device shows +up in `list-devices` with `kind: "device"`. + +**Requirements** + +- **iOS 27 or later** — Apple gates host-driven input ("remote control") to iOS 27+; on + earlier versions the device reports `CoreDeviceError 9021` and only screenshots work. +- macOS with Xcode, and `pymobiledevice3` installed (e.g. `pipx install pymobiledevice3`). +- The iPhone connected, unlocked, trusted, with **Developer Mode** on. + +**Setup** + +1. Enable the feature flag: + ```sh + argent enable physical-ios-devices + ``` +2. Connect the iPhone (unlocked, trusted, Developer Mode on). + +`list-devices` then includes the iPhone, and the supported tools work against its UDID. +The first interaction (or `boot-device`) starts the required CoreDevice tunnel +automatically: Argent shows a standard macOS authorization prompt — branded as Argent, +with Touch ID / password — to launch `pymobiledevice3 remote tunneld` as root (creating the +tunnel interface needs root once; every other command runs unprivileged). No manual `sudo`. + +If the prompt is declined or there's no GUI session (headless), start the tunnel manually +and leave it running: `sudo pymobiledevice3 remote tunneld`. + +**Limitations / notes** + +- Not supported on physical iOS yet: keyboard/typing, pinch & rotate (multi-touch), + `open-url`, and `reinstall-app` — these return a clear "not supported" error. +- Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT` + (defaults to `49151`). + +--- + ## Installation #### Prerequisites diff --git a/packages/argent/scripts/bundle-tools.cjs b/packages/argent/scripts/bundle-tools.cjs index 31bbbc651..89aa9954c 100644 --- a/packages/argent/scripts/bundle-tools.cjs +++ b/packages/argent/scripts/bundle-tools.cjs @@ -73,6 +73,17 @@ const AX_TCP_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/tcp/ax-service"); const BIN_DIR = path.resolve(__dirname, "../bin"); const AX_BIN_DEST = path.resolve(BIN_DIR, "darwin/ax-service"); const AX_TCP_BIN_DEST = path.resolve(BIN_DIR, "darwin/tcp/ax-service"); +// argent-device-auth: macOS host helper for the branded physical-iOS tunnel +// auth prompt. Best-effort — only present once argent-private publishes the +// signed binary; until then physical iOS falls back to the osascript prompt. +const DEVICE_AUTH_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/argent-device-auth"); +const DEVICE_AUTH_BIN_DEST = path.resolve(BIN_DIR, "darwin/argent-device-auth"); +// Argent icon shown in that prompt (committed under native-devtools-ios/assets). +const DEVICE_ICON_SRC = path.resolve( + WORKSPACE_ROOT, + "packages/native-devtools-ios/assets/argent-icon.png" +); +const DEVICE_ICON_DEST = path.resolve(__dirname, "../assets/argent-icon.png"); // Host platform keys (see hostPlatformKey() in @argent/native-devtools-ios): // darwin is a universal binary; Linux ships one single-arch ELF per key. const SUPPORTED_HOST_PLATFORMS = ["darwin", "linux", "linux-arm64"]; @@ -169,6 +180,27 @@ const ASSETS = [ copiedLabel: "ax-service (tcp) binary", missLabel: "ax-service (tcp) binary", }, + // macOS host helper for the branded physical-iOS tunnel auth prompt. + // Best-effort: present once argent-private publishes the signed binary; + // until then physical iOS falls back to the (unbranded) osascript prompt. + { + kind: "file", + src: DEVICE_AUTH_BIN_SRC, + dest: DEVICE_AUTH_BIN_DEST, + mode: 0o755, + required: false, + copiedLabel: "argent-device-auth binary", + missLabel: "argent-device-auth binary", + }, + // Argent icon shown in the device-auth prompt (committed; best-effort copy). + { + kind: "file", + src: DEVICE_ICON_SRC, + dest: DEVICE_ICON_DEST, + required: false, + copiedLabel: "device-auth icon", + missLabel: "device-auth icon", + }, // Android host-side Perfetto trace processor: the third-party WASM engine // (trace_processor.wasm + emscripten glue + the EngineBase decoder + LICENSE). // wasm-trace-processor.ts resolves these via diff --git a/packages/configuration-core/src/flags.ts b/packages/configuration-core/src/flags.ts index 8da2e23ce..13689169a 100644 --- a/packages/configuration-core/src/flags.ts +++ b/packages/configuration-core/src/flags.ts @@ -52,6 +52,11 @@ export const FLAG_REGISTRY: readonly FlagDefinition[] = [ name: "artifacts-list-endpoint", description: "Expose GET /artifacts for remote artifact inventory consumers.", }, + { + name: "physical-ios-devices", + description: + "Discover and control physical iOS devices (iOS 27+) over Apple's CoreDevice tunnel via pymobiledevice3. Requires a running `sudo pymobiledevice3 remote tunneld`. Supports screenshot, tap, swipe, and hardware buttons.", + }, ]; // Look up a flag's definition — exported for consumers that want the diff --git a/packages/native-devtools-ios/assets/argent-icon.png b/packages/native-devtools-ios/assets/argent-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..dc54076a3c07229080e018c45cc8fb382aa593e0 GIT binary patch literal 4058 zcmd^C`8yO`^uJ@qQnGs!lA%Re>y6QCUTX`Ios^wo5QeN7%ODh$B1?u$cqL)%%V3OZ z%C2l&wV}*EzArB4oV#a00pTE+smrVTL1{z8eh`435U?9+R~32N`CzHEM@4yQI=|9 zz$@$ffp}{mKTvvb{*f4u!Hk=GUdJ=>TUp}qDjpq;R~PvHX$pcrrxPm&(_?S^9Ya3H zSZ2~EeLTZLi!ssbYit`CQPrCZ6M50~th@$HUtW8IKA?Am)8+C1tA~}06M7FN)JUc)(=3yn4eIiyBI&}e`q=o6TLBi3O>B5cLE-E7 z0rq9*x3|8tx?16pTB~a4{S!@1`$@NvY>mUJvn#q=C+;zqb;opB)7uQs_`TI1`D07? zIG}g;x$}AhBl4-!u2~OMSGQ(tax1zze=nDM#jg5nWAHZpJmKOpA8<>7&ggK5g+{+v zrtcxfjCijoC-X#GlKp&7bZ)O}RKhEDQEjO*5crS1_@vJG;2oUoR`Vi$W`b>MBhkv8 z%PvC%FF`3=tO<8-n^EWq0 zojtVQOouUoVuBtQ{O_d-Oj2oCL0h`n8cx}}Y|DGUf}MA`;M`vqQ-r}Yuj&<FTac|357}>mdW1ulF)({4Aq}~y!5;z)bJa_OU#${PMrAgQbz=~V`LH#h1 z3S*&<2AZ6O2y=j@C3o*Z58KzMMMA00>RK z66f5X4q9%7;(&zvq>sbkqSu{bSMtv|Lq6giT{G z7$rB*WHeVF-`TJAz(!H#C@K2_(LrWe?%IluL?QCjjBPua(C=wlc!%NX(2y#)|MG(Y z0xW0t{nw2~4YDy|j1%XOX4LX#3~Q{1*8OYYz=*_><=1Kxpea+2X{nMR)eoeExy**Y zeflRQ0!&)Fh?h=>{`xrCJ90S8X3Nr^Z?f!$dfj+Rsz-6}w1K!!0tyVe_ff`f;oT*I zG##Xuq3@@T)xWEXs_XRsGTp@f^g(kBWap|v=B7F`|3nyF))vFvi4>h!UfSKZQ`+C& zoyuvkn0R&)N;ci654w10m)-4aRr6!S^=uYiKv>tjjNcsK@-Mm7c>?OH%v?mfuXwJ# z>5g@&7n=Xr>(D;kU?-}-#x?8kFm3zmVRO;k5cTH%7!7uHI~qE}^Y!-b1@=Z1BYuWw zEn2Hm&9UX~_#;n7KPpw{s^9=2GEnOq>y$Rs;tbHL3a!y_!@U~+x2wF5r|c;M=Lj0# z46TzY`b5dTy<7~V7QxSyz0_*!0>4>8&PY<^%r~yt)ml)ZvGths!jGqDRCtJD>96B+ za3u=)vezmyjq}zjpcbZOQ@(G517{!hySsNRh;?vdt$1BUsJ+FXh1J(%^Cf<6!|p=x zBalX;S8^RXXZ(b9w8AIH$wE%o4sAqoy#d)hBP3kxPH7UHmWVGni0MdwGOm0oF+wNS z1~kei&+W`Stdc7+PH26in*7kArowgxsr4V1UKE@0;cknqY%|lK`Yi4mX~UkJ;lOod zc%$w7dLq0omT~{|>{V`Tv+Ix)_OL*mT*t_!Q1#VtB(x1}9(T}^oN)k9WUwBIL!7SC zvsgPVFgvmgrbP@c29lAW`_L=7lCpQCe8G41MCP>)_c@{?gc|qR4q_({mLJ!pbe7P8 z@*p)4U%gRpde3lCrByg_9rkVei@%Eq=oZ)hI#MD5@<xexFwmf}z@VFE?Dw9A)h^zP6_A@Uu!Um375*QT@s9+kQ^-$4i`0&SfWrE43b)A5Fw-0E@g=>&5E(V zDed(;Dy8CCDns|*b|2Z9eFtU^+Xuxcst980K15ybooU)*!Ah=vKSnaIzeWuBg&I4w z$oYDVW%M(_PxE|Agbrq11lqp+Z$VI53LTdFeOF?k{bx4h@0!VcC z3hKM6X_RS0a9bK|HW{thtGV_Gp>3Sgp-a5>YW4e4h&4^eb!p|$`Z2H~e_bnjf?7Xb zHCot>b8gArY#eH3w(w%z*B%piHHoqK>SO-%|HftCET_AU=(ub?ILpugl)@hB^=O`t{sVu;J_kbhU#r^(TkmK zt}GkWjT`U9W|4$W$)tI_R=OFhHncQX$1`gc88lya$#QC=>}}p@9inoKea+kyz%a@S zV!qC0EL+d)wK*AHAHqG-6(UIwIDl_rjvHI$#K$V>gGhxO!o>GMs_AP%3&d)9)E=De zv-I`kbSCg;6Lxo7i3@ejlJP$ayLXsHb5D1eyIg1pdA@=6HLQgeah+Wk#61-F;XGE= z%&zpE0|?k<&c#dbPwrWR$m@c<;?Buvl^tpC$xFF178o-?0Ex) zmLDn>AG!8KAi6mN1>NtitweE?NIlFJ82MW;n=nkdetSwlb;ztZQioAt(oJOg&TguF zYgL9!zSiYMmd%VB&~JA|N8=QQHq6KtspmFD!O=GDhLt`&9|ddG3rQpVpPBD1qSmTm zvs?6<$zW2KvL1cC=2LHy-zurSDF;g2w2Vo z3x>OrRR{{sO|IDqRluX2FmmBINPtMc?UbYu{aab2nqC$X`WFvru#pE(f>!pFZYbBK zXPIiInZEz5b!NiTt+lx!Uz>pUiZ^@2ur36yysk~QNT@>OjzrFkzBfVG%R~_KJkNH+ zi9iaWQ~q7eyJ7-}qh00p{ZF>n)rM#yZkKzJgbz0SumQ$G>eH#^1=ll5j)nbL6xS&%&3HBuw?b$;-z z*WF8_)q^1uS}3%UR6jTkGL;#si$-> zL~^%#_#mg9Qio;U03`6U@B015F~h9O1Dd~7on!w8;Ag!=UrQLL2zHy}*psW2(5a_B z!(bG0BM(Pw_SRk6_c`2e^MkcWylATg8Xvr!$Tr=WdPWtK!YX#kiXd{P+BB2tM7cbq z@yR>Yi&<}L3y7F7G)ke|EhYIDHB=7tbdx6J-L0D-GJ;Q`>i!T$qc e_rD0(eO?0lBJK>VR+l5af$?RtOBDvLasLB0Ax+=_ literal 0 HcmV?d00001 diff --git a/packages/native-devtools-ios/src/index.ts b/packages/native-devtools-ios/src/index.ts index 4a5eee6ed..d22d1dc0d 100644 --- a/packages/native-devtools-ios/src/index.ts +++ b/packages/native-devtools-ios/src/index.ts @@ -9,6 +9,9 @@ import * as fs from "node:fs"; const DYLIB_DIR = process.env.ARGENT_NATIVE_DEVTOOLS_DIR ?? path.join(__dirname, "..", "dylibs"); const BIN_DIR = process.env.ARGENT_SIMULATOR_SERVER_DIR ?? path.join(__dirname, "..", "bin"); const DYLIB_TCP_DIR = process.env.ARGENT_NATIVE_DEVTOOLS_TCP_DIR ?? path.join(DYLIB_DIR, "tcp"); +// Committed static assets (e.g. the Argent icon shown in the device-auth prompt). +const ASSETS_DIR = + process.env.ARGENT_NATIVE_DEVTOOLS_ASSETS_DIR ?? path.join(__dirname, "..", "assets"); // iOS Simulator only runs on macOS, so the dylibs that get injected into it // and the ax-service that gets `simctl spawn`d into it are only ever usable @@ -115,3 +118,24 @@ export function axServiceBinaryPathTcp(): string { requireDarwin("ax-service (tcp)"); return requireBinIn(platformTcpBinDir(), "ax-service"); } + +// argent-device-auth is the macOS host helper that pops the branded macOS +// authorization modal (password / Touch ID) and runs a command as root — +// Argent uses it to start the physical-iOS CoreDevice tunnel without a manual +// `sudo`. Unlike the resolvers above it returns null (rather than throwing) +// when absent, so callers can fall back to a less-branded escalation path. +// Override the binary with ARGENT_DEVICE_AUTH_HELPER (absolute path). +export function deviceAuthHelperPath(): string | null { + if (process.platform !== "darwin") return null; + const override = process.env.ARGENT_DEVICE_AUTH_HELPER; + if (override) return fs.existsSync(override) ? override : null; + const p = path.join(platformBinDir(), "argent-device-auth"); + return fs.existsSync(p) ? p : null; +} + +// Path to the Argent icon shown in the device-auth prompt, or null if missing. +// Override with ARGENT_DEVICE_ICON (absolute path to a PNG/icns). +export function argentIconPath(): string | null { + const p = process.env.ARGENT_DEVICE_ICON ?? path.join(ASSETS_DIR, "argent-icon.png"); + return fs.existsSync(p) ? p : null; +} diff --git a/packages/tool-server/src/blueprints/ax-service.ts b/packages/tool-server/src/blueprints/ax-service.ts index e0a377fec..03096d5cb 100644 --- a/packages/tool-server/src/blueprints/ax-service.ts +++ b/packages/tool-server/src/blueprints/ax-service.ts @@ -309,6 +309,14 @@ export const axServiceBlueprint: ServiceBlueprint = { `${AX_SERVICE_NAMESPACE} is iOS-only. The target '${device.id}' classifies as ${device.platform} — describe uses uiautomator on Android and the CDP DOM walker on Chromium, neither of which needs this service.` ); } + if (device.kind === "device") { + // ax-service uses `xcrun simctl spawn`, which only works on simulators. + // Physical iPhones are driven over CoreDevice and have no accessibility/ + // describe path yet. + throw new Error( + `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; describe/accessibility is not supported on physical iOS yet.` + ); + } // Reject before spawning. An undefined `device.id` slips through when an // inner tool is invoked via a wrapper that doesn't re-validate the inner // schema. Without this guard `getSocketPath(undefined).slice` would crash diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts new file mode 100644 index 000000000..ef9280ec0 --- /dev/null +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -0,0 +1,464 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { existsSync } from "node:fs"; +import { tmpdir, homedir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { + TypedEventEmitter, + type DeviceInfo, + type ServiceBlueprint, + type ServiceInstance, + type ServiceEvents, +} from "@argent/registry"; +import { isFlagEnabled } from "@argent/configuration-core"; +import { deviceAuthHelperPath, argentIconPath } from "@argent/native-devtools-ios"; + +const execFileAsync = promisify(execFile); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export const CORE_DEVICE_NAMESPACE = "CoreDevice"; + +// Opt-in flag (also gates discovery in list-devices). The privileged tunnel +// start must not be reachable unless the user enabled the experimental feature. +const PHYSICAL_IOS_FLAG = "physical-ios-devices"; + +// The registry's `ServiceRef.options` is typed as `Record`; +// the intersection adds the implicit string index signature an interface lacks. +type CoreDeviceFactoryOptions = Record & { device: DeviceInfo }; + +/** + * Backend for a physical iOS device, driven over Apple's CoreDevice "remote + * control" services via the `pymobiledevice3` CLI — no app installed on the + * device. This is a separate blueprint from the simulator-server because real + * iPhones speak an entirely different transport (the iOS-17+ RemoteXPC tunnel), + * mirroring how physical Android uses its own `android_device` controller. + * + * Requirements (all surfaced as actionable errors): iOS 27+ (Apple gates the + * touch/"remote control" services to 27.0+), `pymobiledevice3` installed, and a + * running CoreDevice tunnel (`sudo pymobiledevice3 remote tunneld`, which needs + * root to create the tunnel interface — every command here then runs unprivileged). + */ +export interface CoreDeviceApi { + /** Capture a PNG to a temp file and return its path. */ + screenshot(): Promise<{ path: string }>; + /** Tap at normalized (x, y) in 0..1. */ + tap(x: number, y: number): Promise; + /** Swipe/drag from (fromX, fromY) to (toX, toY), all normalized 0..1. */ + swipe(fromX: number, fromY: number, toX: number, toY: number, durationMs: number): Promise; + /** Press a hardware button by its pymobiledevice3 name (home/lock/volume-up/volume-down/...). */ + button(name: string): Promise; +} + +export function coreDeviceRef(device: DeviceInfo): { + urn: string; + options: CoreDeviceFactoryOptions; +} { + return { + urn: `${CORE_DEVICE_NAMESPACE}:${device.id}`, + options: { device }, + }; +} + +interface Rsd { + address: string; + port: number; +} + +function tunneldPort(): number { + const raw = process.env.ARGENT_PMD3_TUNNELD_PORT; + const n = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isInteger(n) && n > 0 && n <= 65535 ? n : 49151; +} + +/** Resolve the pymobiledevice3 executable: env override, common install dirs, then PATH. */ +function resolvePmd3(): string { + const override = process.env.ARGENT_PYMOBILEDEVICE3; + if (override) return override; + const candidates = [ + join(homedir(), ".local", "bin", "pymobiledevice3"), + "/opt/homebrew/bin/pymobiledevice3", + "/usr/local/bin/pymobiledevice3", + ]; + for (const c of candidates) if (existsSync(c)) return c; + return "pymobiledevice3"; +} + +/** + * Fail fast with an install hint when pymobiledevice3 is missing, rather than + * surfacing a raw `spawn ENOENT` from the first interaction. Runs `version` (a + * cheap subcommand); a missing binary throws ENOENT, anything else is tolerated. + */ +async function verifyPmd3Available(pmd3: string): Promise { + try { + await execFileAsync(pmd3, ["version"], { timeout: 10_000 }); + } catch (err) { + if ((err as { code?: string })?.code === "ENOENT") { + throw new Error( + `pymobiledevice3 was not found (tried "${pmd3}"). Physical iOS control needs it — ` + + `install it (e.g. \`pipx install pymobiledevice3\`) or set ARGENT_PYMOBILEDEVICE3 to its path.`, + { cause: err } + ); + } + // Non-ENOENT (odd `version` failure): don't block; a real command will report. + } +} + +/** Resolve pymobiledevice3 to an absolute path — the privileged root shell does + * not inherit the user's PATH (so `~/.local/bin` is invisible to it). */ +async function resolvePmd3Absolute(): Promise { + const p = resolvePmd3(); + if (p.startsWith("/")) return p; + try { + const { stdout } = await execFileAsync("/usr/bin/which", [p], { timeout: 5_000 }); + const abs = stdout.trim(); + if (abs.startsWith("/")) return abs; + } catch { + // fall through to the install hint + } + throw new Error( + `pymobiledevice3 was not found on PATH. Install it (e.g. \`pipx install pymobiledevice3\`) ` + + `or set ARGENT_PYMOBILEDEVICE3 to its absolute path.` + ); +} + +/** Single-quote a string for safe use as one /bin/sh word. */ +function shellSingleQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'`; +} + +/** Escape a /bin/sh command string for embedding inside an AppleScript double-quoted literal. */ +export function appleScriptQuote(cmd: string): string { + return cmd.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); +} + +/** + * The /bin/sh command (run as root) that starts a daemonized tunneld on `port`. + * HOME is pinned to root's home so pymobiledevice3 finds its RemoteXPC pairing + * records: the privileged-exec environment (Authorization Services) doesn't + * inherit a usable HOME, which otherwise leaves tunneld unable to form the + * device tunnel (it starts but never registers one). sudo set HOME=/var/root + * implicitly; here we set it explicitly. + * + * The log goes under /var/root (root's home, mode 0700) rather than a predictable + * path in world-writable /tmp: this command runs as root, and a root `>` redirect + * to a /tmp path a non-root user can pre-symlink is a classic privileged-tmp + * clobber (CWE-59). + */ +export function tunneldStartCommand(pmd3Abs: string, port: number): string { + return `HOME=/var/root ${shellSingleQuote(pmd3Abs)} remote tunneld --port ${port} -d > /var/root/argent-coredevice-tunneld.log 2>&1`; +} + +/** + * Start `pymobiledevice3 remote tunneld` as root via the standard macOS + * authorization dialog (password / Touch ID where the OS offers it) — so users + * never run sudo by hand. AppleScript's `with administrator privileges` routes + * through Authorization Services and shows the system modal in the active GUI + * session; `-d` daemonizes so the privileged shell returns immediately, leaving + * tunneld running as root for the rest of the session. Throws if the user + * cancels or no GUI session is available (headless) — callers fall back to the + * manual `sudo` instructions. + */ +async function startTunneldWithPrivilege(pmd3Abs: string, port: number): Promise { + const shellCmd = tunneldStartCommand(pmd3Abs, port); + // Generous timeout: the user has to see and approve the modal. + const timeout = 120_000; + + // Preferred: the signed host helper, which shows the modal branded as + // "Argent" with the Argent icon + a clear message via Authorization Services. + const helper = deviceAuthHelperPath(); + if (helper) { + const icon = argentIconPath() ?? ""; + const prompt = "Argent needs administrator access to connect to a physical iOS device."; + try { + await execFileAsync(helper, [icon, prompt, "/bin/sh", "-c", shellCmd], { timeout }); + return; + } catch (err) { + // Exit 3 = the user explicitly cancelled the prompt — respect that and + // don't pop a second (osascript) prompt. Any other failure (a broken, + // unsigned, quarantined, or 0-byte helper binary) degrades to the + // osascript admin prompt below rather than hard-failing. + if ((err as { code?: unknown }).code === 3) throw err; + } + } + + // Fallback when the helper isn't installed (e.g. a dev tree without the signed + // binary), is unusable, or there's no GUI: the generic osascript admin prompt. + // Functional but unbranded ("osascript wants to make changes"). + const appleScript = `do shell script "${appleScriptQuote(shellCmd)}" with administrator privileges`; + await execFileAsync("osascript", ["-e", appleScript], { timeout }); +} + +// Dedupe concurrent escalation prompts: parallel interactions share one modal. +// Cleared after settle so a later call can retry if the user cancelled. +let tunnelStartInFlight: Promise | null = null; + +function tunnelHelp(udid: string, reason: string): string { + const port = tunneldPort(); + return ( + `Physical iOS control needs a CoreDevice tunnel for ${udid}, but ${reason} ` + + `(checked tunneld at 127.0.0.1:${port}). Argent tries to start it automatically via the macOS ` + + `authorization prompt; if that was declined or no GUI session is available, start it manually ` + + `and leave it running:\n sudo pymobiledevice3 remote tunneld\n` + + `Also ensure the iPhone is on iOS 27+, unlocked, and trusted. ` + + `(Override the port with ARGENT_PMD3_TUNNELD_PORT.)` + ); +} + +/** + * Look up the device's RSD endpoint from a running `pymobiledevice3 remote + * tunneld` (its local REST API at 127.0.0.1:). Re-resolved per command so + * a tunneld restart mid-session is picked up without re-creating the service. + */ +async function resolveTunnel(udid: string): Promise { + const port = tunneldPort(); + let payload: Record>; + try { + const res = await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(4000) }); + // Something other than tunneld could be bound to the port; don't parse an + // error page as a tunnel list. + if (!res.ok) throw new Error(`tunneld responded HTTP ${res.status}`); + payload = (await res.json()) as typeof payload; + } catch (err) { + throw new Error(tunnelHelp(udid, "tunneld is not running"), { cause: err }); + } + const entry = payload?.[udid]; + const t = Array.isArray(entry) ? entry[0] : undefined; + if (!t?.["tunnel-address"] || t["tunnel-port"] == null) { + throw new Error(tunnelHelp(udid, "no active tunnel is registered for it")); + } + return { address: String(t["tunnel-address"]), port: Number(t["tunnel-port"]) }; +} + +/** + * Whether anything is serving on the tunneld port (any HTTP response counts). + * Distinguishes "tunneld not running → start it" from "tunneld running but this + * device's tunnel hasn't formed yet → just wait", so we don't pop a second, + * pointless root prompt (or fail trying to bind an already-used port). + */ +async function isTunneldReachable(port: number): Promise { + try { + await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(4000) }); + return true; + } catch { + return false; + } +} + +/** + * Return the device's RSD endpoint, auto-starting tunneld via the macOS auth + * modal when it isn't running — so the user never types sudo. Gated behind the + * physical-ios-devices flag (the privileged escalation is opt-in). If tunneld is + * already running but this device's tunnel hasn't registered (e.g. the iPhone is + * locked), it waits rather than popping a second, pointless prompt. + */ +export async function ensureCoreDeviceTunnel(udid: string): Promise { + if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { + throw new Error( + `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}` + ); + } + try { + return await resolveTunnel(udid); + } catch (notRunning) { + if (process.platform !== "darwin") throw notRunning; + const port = tunneldPort(); + + // Only escalate (root prompt) when tunneld isn't running at all. If it IS + // running, the device just needs to be unlocked/trusted for the handshake — + // don't re-prompt, just poll. + const reachable = await isTunneldReachable(port); + if (!reachable) { + if (!tunnelStartInFlight) { + tunnelStartInFlight = (async () => { + const pmd3Abs = await resolvePmd3Absolute(); + await startTunneldWithPrivilege(pmd3Abs, port); + })(); + } + try { + await tunnelStartInFlight; + } catch (escalationErr) { + tunnelStartInFlight = null; // allow a later retry + throw new Error(tunnelHelp(udid, "the authorization prompt was cancelled or unavailable"), { + cause: escalationErr, + }); + } + tunnelStartInFlight = null; + } + + // Poll for this device's tunnel to register (handshake needs it unlocked & trusted). + for (let i = 0; i < 15; i++) { + await sleep(2_000); + try { + return await resolveTunnel(udid); + } catch { + // keep polling + } + } + throw new Error( + tunnelHelp( + udid, + reachable + ? "tunneld is running but the device tunnel did not form — is the iPhone unlocked and trusted?" + : "the tunnel did not come up after starting tunneld" + ), + { cause: notRunning } + ); + } +} + +/** Extract a concise, human-readable line from a (possibly huge/binary) pmd3 failure. */ +function conciseError(label: string, err: unknown): Error { + const e = err as { stderr?: string; stdout?: string; message?: string }; + const blob = [e?.stderr, e?.stdout, e?.message] + .filter((v): v is string => typeof v === "string") + .join("\n"); + const line = + blob + .split("\n") + .map((s) => s.trim()) + .find((l) => /requires iOS|error|fail|not |unable|refused|timed out/i.test(l)) ?? + e?.message ?? + "unknown error"; + return new Error(`CoreDevice ${label} failed: ${line.slice(0, 240)}`, { cause: err }); +} + +const COREDEVICE = ["developer", "core-device"]; +const HID = [...COREDEVICE, "universal-hid-service"]; + +/** Normalized 0..1 → the device's 0..65535 HID coordinate space. */ +export function toHid(v: number): number { + return Math.max(0, Math.min(65535, Math.round(v * 65535))); +} + +/** + * Ensure the personalized DeveloperDiskImage is mounted (the CoreDevice services + * live in it). Idempotent: when already mounted, pymobiledevice3 exits non-zero + * with "already mounted", which we treat as success. CoreDevice usually mounts + * it automatically when the device connects, so this is a best-effort fallback — + * a genuine mount failure surfaces later as an actionable per-command error. + */ +async function ensureMounted(pmd3: string, rsd: Rsd): Promise { + try { + await execFileAsync(pmd3, ["mounter", "auto-mount", "--rsd", rsd.address, String(rsd.port)], { + timeout: 60_000, + }); + } catch { + // Best-effort: when the DDI is already mounted pymobiledevice3 exits non-zero + // with "already mounted"; a genuine mount failure surfaces later as an + // actionable per-command error. Either way, don't fail service creation here. + } +} + +export const coreDeviceBlueprint: ServiceBlueprint = { + namespace: CORE_DEVICE_NAMESPACE, + getURN(device: DeviceInfo) { + return `${CORE_DEVICE_NAMESPACE}:${device.id}`; + }, + async factory(_deps, _payload, options) { + const opts = options as unknown as CoreDeviceFactoryOptions | undefined; + const device = opts?.device; + if (!device?.id) { + throw new Error( + `${CORE_DEVICE_NAMESPACE}.factory requires a resolved DeviceInfo via options.device. ` + + `Use coreDeviceRef(device) when registering the service ref.` + ); + } + if (device.platform !== "ios" || device.kind !== "device") { + throw new Error( + `${CORE_DEVICE_NAMESPACE} only drives physical iOS devices; got ${device.platform}/${device.kind}.` + ); + } + + const pmd3 = resolvePmd3(); + const udid = device.id; + + // Surface setup problems up front, in the order a user fixes them: install + // pymobiledevice3, then ensure the tunnel (auto-started via the macOS auth + // modal if needed — no manual sudo), then mount the DDI. + await verifyPmd3Available(pmd3); + const rsd = await ensureCoreDeviceTunnel(udid); + await ensureMounted(pmd3, rsd); + + const run = async (label: string, args: string[], timeout: number): Promise => { + const tunnel = await resolveTunnel(udid); + try { + const { stdout } = await execFileAsync( + pmd3, + [...args, "--rsd", tunnel.address, String(tunnel.port)], + { timeout, maxBuffer: 16 * 1024 * 1024 } + ); + return stdout; + } catch (err) { + throw conciseError(label, err); + } + }; + + const events = new TypedEventEmitter(); + + const api: CoreDeviceApi = { + async screenshot() { + const path = join(tmpdir(), `argent-ios-shot-${randomUUID()}.png`); + await run("screenshot", [...COREDEVICE, "screen-capture", "screenshot", path], 20_000); + return { path }; + }, + async tap(x, y) { + const hx = toHid(x); + const hy = toHid(y); + // A zero-duration CONTACT+RELEASE (pmd3's `tap`) is dropped by iOS — a + // tap must dwell. Emit a short held drag with a tiny (~3px) move, away + // from the edge so it never clips, which registers as a real tap. + const hy2 = hy <= 65535 - 120 ? hy + 96 : hy - 96; + await run( + "tap", + [ + ...HID, + "drag", + String(hx), + String(hy), + String(hx), + String(hy2), + "--steps", + "3", + "--duration", + "0.15", + ], + 15_000 + ); + }, + async swipe(fromX, fromY, toX, toY, durationMs) { + const steps = Math.max(2, Math.min(60, Math.round(durationMs / 16))); + const seconds = (durationMs / 1000).toFixed(3); + await run( + "swipe", + [ + ...HID, + "drag", + String(toHid(fromX)), + String(toHid(fromY)), + String(toHid(toX)), + String(toHid(toY)), + "--steps", + String(steps), + "--duration", + seconds, + ], + 15_000 + ); + }, + async button(name) { + await run("button", [...COREDEVICE, "hid", "button", name, "press"], 10_000); + }, + }; + + const instance: ServiceInstance = { + api, + // Stateless: each interaction is a fresh pymobiledevice3 invocation, so + // there is no long-lived process to tear down. + dispose: async () => {}, + events, + }; + return instance; + }, +}; diff --git a/packages/tool-server/src/blueprints/native-devtools.ts b/packages/tool-server/src/blueprints/native-devtools.ts index 9f12c75ef..4ede657ef 100644 --- a/packages/tool-server/src/blueprints/native-devtools.ts +++ b/packages/tool-server/src/blueprints/native-devtools.ts @@ -367,6 +367,14 @@ export const nativeDevtoolsBlueprint: ServiceBlueprint {}); } else if (device.platform === "android") { diff --git a/packages/tool-server/src/tools/button/index.ts b/packages/tool-server/src/tools/button/index.ts index a04938573..659b92cff 100644 --- a/packages/tool-server/src/tools/button/index.ts +++ b/packages/tool-server/src/tools/button/index.ts @@ -1,10 +1,22 @@ import { z } from "zod"; -import type { Platform, ToolCapability, ToolDefinition } from "@argent/registry"; +import type { Platform, ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { UnsupportedOperationError } from "../../utils/capability"; import { sendCommand } from "../../utils/simulator-client"; +// Argent button name → pymobiledevice3 CoreDevice HID button name. CoreDevice +// exposes the physical buttons only; appSwitch (a SpringBoard gesture) and the +// iPhone 15 Pro action button have no HID equivalent, so they are omitted and +// rejected with a clear error on physical iOS. +const COREDEVICE_BUTTON: Partial> = { + home: "home", + power: "lock", + volumeUp: "volume-up", + volumeDown: "volume-down", +}; + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const zodSchema = z.object({ @@ -52,9 +64,13 @@ Returns { pressed: buttonName }. Fails if the simulator-server / emulator backend is not reachable for the given device.`, zodSchema, capability, - services: (params) => ({ - simulatorServer: simulatorServerRef(resolveDevice(params.udid)), - }), + services: (params): Record => { + const device = resolveDevice(params.udid); + if (isPhysicalIos(device)) { + return { coreDevice: coreDeviceRef(device) }; + } + return { simulatorServer: simulatorServerRef(device) }; + }, async execute(services, params) { const device = resolveDevice(params.udid); if (!BUTTONS_BY_PLATFORM[device.platform].has(params.button)) { @@ -64,6 +80,19 @@ Fails if the simulator-server / emulator backend is not reachable for the given `button '${params.button}' is not available on ${device.platform}` ); } + if (isPhysicalIos(device)) { + const name = COREDEVICE_BUTTON[params.button]; + if (!name) { + throw new UnsupportedOperationError( + "button", + device, + `button '${params.button}' is not available on physical iOS (CoreDevice exposes home, power, volumeUp, volumeDown)` + ); + } + const coreDevice = services.coreDevice as CoreDeviceApi; + await coreDevice.button(name); + return { pressed: params.button }; + } const api = services.simulatorServer as SimulatorServerApi; sendCommand(api, { cmd: "button", diff --git a/packages/tool-server/src/tools/devices/boot-device.ts b/packages/tool-server/src/tools/devices/boot-device.ts index b90944553..dd1249447 100644 --- a/packages/tool-server/src/tools/devices/boot-device.ts +++ b/packages/tool-server/src/tools/devices/boot-device.ts @@ -27,6 +27,8 @@ import { import { ensureDep } from "../../utils/check-deps"; import { linuxBootDiagnostics } from "../../utils/linux-preflight"; import { listIosSimulators } from "../../utils/ios-devices"; +import { isPhysicalIosUdid } from "../../utils/device-info"; +import { ensureCoreDeviceTunnel } from "../../blueprints/core-device"; import { bootElectronApp, type ElectronBootResult } from "./boot-electron"; const execFileAsync = promisify(execFile); @@ -410,6 +412,17 @@ async function bootIos( `Pass \`avdName\` (Android) instead of \`udid\` (iOS) to boot a device from this host.` ); } + + // A physical iPhone is already powered on — there is nothing to "boot". It is + // driven over CoreDevice (see core-device blueprint), not the simulator-server. + // Use this as the explicit "prepare" step: ensure the CoreDevice tunnel is up, + // auto-starting it via the macOS authorization prompt if needed (no manual + // sudo). Surfaces a clear error if the prompt is declined / unavailable. + if (isPhysicalIosUdid(udid)) { + await ensureCoreDeviceTunnel(udid); + return { platform: "ios", udid, booted: true }; + } + await ensureDep("xcrun"); const simState = await listIosSimulators() diff --git a/packages/tool-server/src/tools/devices/list-devices.ts b/packages/tool-server/src/tools/devices/list-devices.ts index cdbd04784..485aaa8f5 100644 --- a/packages/tool-server/src/tools/devices/list-devices.ts +++ b/packages/tool-server/src/tools/devices/list-devices.ts @@ -1,10 +1,25 @@ import { z } from "zod"; import type { ToolDefinition } from "@argent/registry"; +import { isFlagEnabled } from "@argent/configuration-core"; import { listAndroidDevices, listAvds } from "../../utils/adb"; -import { listIosSimulators, type IosSimulator } from "../../utils/ios-devices"; +import { listIosSimulators, listIosDevices } from "../../utils/ios-devices"; import { discoverChromiumDevices, type ChromiumDevice } from "../../utils/chromium-discovery"; -type IosDevice = IosSimulator & { platform: "ios" }; +const PHYSICAL_IOS_FLAG = "physical-ios-devices"; + +type IosDevice = { + platform: "ios"; + udid: string; + name: string; + state: string; + // "simulator" for an `xcrun simctl` simulator, "device" for a physical iPhone + // discovered via `xcrun devicectl` and driven over CoreDevice (pymobiledevice3). + kind: "simulator" | "device"; + // simulators only (the iOS runtime, e.g. "com.apple.CoreSimulator.SimRuntime.iOS-18-5") + runtime?: string; + // physical devices only (Apple product type, e.g. "iPhone15,4") + productType?: string | null; +}; type AndroidDevice = { platform: "android"; @@ -26,10 +41,13 @@ type ListDevicesResult = { avds: Array<{ name: string }>; }; +// A simulator is ready when "Booted"; a physical device is ready when "connected". +const iosReady = (d: IosDevice): boolean => d.state === "Booted" || d.state === "connected"; + function sortIos(a: IosDevice, b: IosDevice): number { - const aBooted = a.state === "Booted" ? 0 : 1; - const bBooted = b.state === "Booted" ? 0 : 1; - if (aBooted !== bBooted) return aBooted - bBooted; + const aReady = iosReady(a) ? 0 : 1; + const bReady = iosReady(b) ? 0 : 1; + if (aReady !== bReady) return aReady - bReady; const aIpad = a.name.includes("iPad") ? 1 : 0; const bIpad = b.name.includes("iPad") ? 1 : 0; return aIpad - bIpad; @@ -47,7 +65,7 @@ function sortAndroid(a: AndroidDevice, b: AndroidDevice): number { // Float booted/ready devices to the top of the merged list regardless of // platform — without this, all iOS entries are emitted before any Android. function readinessRank(d: IosDevice | AndroidDevice | ChromiumDevice): number { - if (d.platform === "ios") return d.state === "Booted" ? 0 : 1; + if (d.platform === "ios") return iosReady(d) ? 0 : 1; if (d.platform === "android") return d.state === "device" ? 0 : 1; return 0; // Chromium entries are only listed when their CDP is responsive } @@ -60,6 +78,7 @@ export const listDevicesTool: ToolDefinition, ListDevicesR Use at the start of a session to pick a target id ('udid' for iOS entries, 'serial' for Android, 'id' for Chromium) to pass to interaction tools, and to see which targets are already running. Returns { devices, avds } where each device carries a 'platform' discriminator ('ios', 'android', or 'chromium'), and 'avds' lists Android AVDs that can be booted via boot-device. Android entries also carry a 'kind' ('emulator' for a local AVD, 'device' for a physical phone connected over USB / wireless adb) — physical phones are detected from \`adb devices\` (any serial that is not an \`emulator-*\` one) and are driven through the same interaction tools as emulators; they do not need boot-device (just connect the phone with USB debugging authorised). +iOS entries likewise carry a 'kind' ('simulator', or 'device' for a connected physical iPhone). Physical iOS devices require the 'physical-ios-devices' flag (\`argent enable physical-ios-devices\`), iOS 27+, and a running CoreDevice tunnel (\`sudo pymobiledevice3 remote tunneld\`); they support screenshot, gesture-tap, gesture-swipe, and button. Chromium apps are discovered by probing CDP debugging ports (default 9222; extend via the ARGENT_CHROMIUM_PORTS= env var). They must already be running with --remote-debugging-port= — use boot-device with chromiumAppPath to launch one. Booted/ready devices are listed first. Platforms whose CLI is unavailable are silently omitted — an empty result usually means xcode-select or Android platform-tools is not installed.`, alwaysLoad: true, @@ -68,13 +87,36 @@ Booted/ready devices are listed first. Platforms whose CLI is unavailable are si zodSchema, services: () => ({}), async execute(_services, _params) { - const [ios, android, avds, chromium] = await Promise.all([ + const physicalIosEnabled = isFlagEnabled(PHYSICAL_IOS_FLAG); + const [ios, iosPhysical, android, avds, chromium] = await Promise.all([ listIosSimulators(), + physicalIosEnabled ? listIosDevices().catch(() => []) : Promise.resolve([]), listAndroidDevices().catch(() => []), listAvds(), discoverChromiumDevices().catch(() => []), ]); - const iosTagged: IosDevice[] = ios.map((s) => ({ platform: "ios", ...s })); + const iosTagged: IosDevice[] = [ + ...ios.map( + (s): IosDevice => ({ + platform: "ios", + kind: "simulator", + udid: s.udid, + name: s.name, + state: s.state, + runtime: s.runtime, + }) + ), + ...iosPhysical.map( + (d): IosDevice => ({ + platform: "ios", + kind: "device", + udid: d.udid, + name: d.name, + state: d.state, + productType: d.productType, + }) + ), + ]; iosTagged.sort(sortIos); const androidTagged: AndroidDevice[] = android.map((d) => ({ platform: "android", diff --git a/packages/tool-server/src/tools/gesture-swipe/index.ts b/packages/tool-server/src/tools/gesture-swipe/index.ts index 0388cbd99..bf80582eb 100644 --- a/packages/tool-server/src/tools/gesture-swipe/index.ts +++ b/packages/tool-server/src/tools/gesture-swipe/index.ts @@ -1,7 +1,8 @@ import { z } from "zod"; -import type { ToolCapability, ToolDefinition } from "@argent/registry"; +import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { sendCommand } from "../../utils/simulator-client"; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -44,12 +45,22 @@ Use when you need to scroll a list, dismiss a modal, drag an element, or navigat searchHint: "swipe scroll drag pan gesture device simulator emulator touch move", zodSchema, capability, - services: (params) => ({ - simulatorServer: simulatorServerRef(resolveDevice(params.udid)), - }), + services: (params): Record => { + const device = resolveDevice(params.udid); + if (isPhysicalIos(device)) { + return { coreDevice: coreDeviceRef(device) }; + } + return { simulatorServer: simulatorServerRef(device) }; + }, async execute(services, params) { const duration = params.durationMs ?? 300; const timestampMs = Date.now(); + const device = resolveDevice(params.udid); + if (isPhysicalIos(device)) { + const coreDevice = services.coreDevice as CoreDeviceApi; + await coreDevice.swipe(params.fromX, params.fromY, params.toX, params.toY, duration); + return { swiped: true, timestampMs }; + } const api = services.simulatorServer as SimulatorServerApi; const steps = Math.max(1, Math.round(duration / 16)); diff --git a/packages/tool-server/src/tools/gesture-tap/index.ts b/packages/tool-server/src/tools/gesture-tap/index.ts index b9f74d28c..c33e84864 100644 --- a/packages/tool-server/src/tools/gesture-tap/index.ts +++ b/packages/tool-server/src/tools/gesture-tap/index.ts @@ -2,7 +2,8 @@ import { z } from "zod"; import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; import { chromiumCdpRef, type ChromiumCdpApi } from "../../blueprints/chromium-cdp"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { sendCommand } from "../../utils/simulator-client"; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -54,6 +55,9 @@ Before tapping, determine the correct coordinates by using discovery tools — p if (device.platform === "chromium") { return { chromium: chromiumCdpRef(device) }; } + if (isPhysicalIos(device)) { + return { coreDevice: coreDeviceRef(device) }; + } return { simulatorServer: simulatorServerRef(device) }; }, async execute(services, params) { @@ -64,6 +68,11 @@ Before tapping, determine the correct coordinates by using discovery tools — p await tapChromium(chromium, params.x, params.y); return { tapped: true, timestampMs }; } + if (isPhysicalIos(device)) { + const coreDevice = services.coreDevice as CoreDeviceApi; + await coreDevice.tap(params.x, params.y); + return { tapped: true, timestampMs }; + } const api = services.simulatorServer as SimulatorServerApi; sendCommand(api, { cmd: "touch", diff --git a/packages/tool-server/src/tools/launch-app/index.ts b/packages/tool-server/src/tools/launch-app/index.ts index 90670b969..bc9b8a407 100644 --- a/packages/tool-server/src/tools/launch-app/index.ts +++ b/packages/tool-server/src/tools/launch-app/index.ts @@ -3,7 +3,7 @@ import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registr import { nativeDevtoolsRef } from "../../blueprints/native-devtools"; import { chromiumCdpRef } from "../../blueprints/chromium-cdp"; import { dispatchByPlatform } from "../../utils/cross-platform-tool"; -import { resolveDevice } from "../../utils/device-info"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import type { LaunchAppAndroidServices, LaunchAppIosServices, LaunchAppResult } from "./types"; import { iosImpl } from "./platforms/ios"; import { androidImpl } from "./platforms/android"; @@ -65,7 +65,11 @@ Common Android packages: com.android.settings, com.android.chrome, com.google.an // Only iOS needs the native-devtools service for launch-time injection. Chromium needs its CDP session. services: (params): Record => { const device = resolveDevice(params.udid); - if (device.platform === "ios") return { nativeDevtools: nativeDevtoolsRef(device) }; + // Physical iOS is driven over CoreDevice (devicectl launch in the handler) and + // has no native-devtools; resolving that service eagerly would throw its + // simulator-only guard before the handler ever runs. + if (device.platform === "ios" && !isPhysicalIos(device)) + return { nativeDevtools: nativeDevtoolsRef(device) }; if (device.platform === "chromium") return { chromium: chromiumCdpRef(device) }; return {}; }, diff --git a/packages/tool-server/src/tools/launch-app/platforms/ios.ts b/packages/tool-server/src/tools/launch-app/platforms/ios.ts index ddc5814ce..6c37e5862 100644 --- a/packages/tool-server/src/tools/launch-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/launch-app/platforms/ios.ts @@ -8,7 +8,22 @@ const execFileAsync = promisify(execFile); export const iosImpl: PlatformImpl = { requires: ["xcrun"], - handler: async (services, params) => { + handler: async (services, params, device) => { + // Physical iPhones are driven via CoreDevice — launch through devicectl + // (the app must already be installed/signed on the device). The + // native-devtools precheck is simulator-only, so it is skipped here. + if (device.kind === "device") { + await execFileAsync("xcrun", [ + "devicectl", + "device", + "process", + "launch", + "--device", + params.udid, + params.bundleId, + ]); + return { launched: true, bundleId: params.bundleId }; + } const blocked = await precheckNativeDevtools(services.nativeDevtools, params.udid); if (blocked) return blocked; await execFileAsync("xcrun", ["simctl", "launch", params.udid, params.bundleId]); diff --git a/packages/tool-server/src/tools/open-url/platforms/ios.ts b/packages/tool-server/src/tools/open-url/platforms/ios.ts index 00fc99ad1..810b2c375 100644 --- a/packages/tool-server/src/tools/open-url/platforms/ios.ts +++ b/packages/tool-server/src/tools/open-url/platforms/ios.ts @@ -7,7 +7,13 @@ const execFileAsync = promisify(execFile); export const iosImpl: PlatformImpl = { requires: ["xcrun"], - handler: async (_services, params) => { + handler: async (_services, params, device) => { + if (device.kind === "device") { + // CoreDevice/devicectl has no deep-link/open-url surface for physical + // iOS; only screenshot, gesture-tap, gesture-swipe, button, and launch-app + // are supported there today. + throw new Error("open-url is not supported on physical iOS devices."); + } await execFileAsync("xcrun", ["simctl", "openurl", params.udid, params.url]); return { opened: true, url: params.url }; }, diff --git a/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts b/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts index 9cd143eee..5e8c2fc2f 100644 --- a/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts @@ -8,7 +8,12 @@ const execFileAsync = promisify(execFile); export const iosImpl: PlatformImpl = { requires: ["xcrun"], - handler: async (_services, params) => { + handler: async (_services, params, device) => { + if (device.kind === "device") { + // Installing on a physical iPhone needs a device-signed .app and a + // provisioning profile; that is out of scope for the CoreDevice path. + throw new Error("reinstall-app is not supported on physical iOS devices."); + } const { udid, bundleId, appPath } = params; const absolute = resolvePath(appPath); try { diff --git a/packages/tool-server/src/tools/restart-app/index.ts b/packages/tool-server/src/tools/restart-app/index.ts index 767673934..0c6d6217c 100644 --- a/packages/tool-server/src/tools/restart-app/index.ts +++ b/packages/tool-server/src/tools/restart-app/index.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { nativeDevtoolsRef } from "../../blueprints/native-devtools"; import { dispatchByPlatform } from "../../utils/cross-platform-tool"; -import { resolveDevice } from "../../utils/device-info"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import type { RestartAppAndroidServices, RestartAppIosServices, RestartAppResult } from "./types"; import { iosImpl } from "./platforms/ios"; import { androidImpl } from "./platforms/android"; @@ -53,7 +53,11 @@ Returns { restarted, bundleId }. Fails if the app is not installed.`, // Only iOS needs the native-devtools service for relaunch injection. services: (params): Record => { const device = resolveDevice(params.udid); - return device.platform === "ios" ? { nativeDevtools: nativeDevtoolsRef(device) } : {}; + // Physical iOS has no native-devtools (it's rejected in the handler); + // resolving that service eagerly would throw its simulator-only guard first. + return device.platform === "ios" && !isPhysicalIos(device) + ? { nativeDevtools: nativeDevtoolsRef(device) } + : {}; }, execute: dispatchByPlatform< RestartAppIosServices, diff --git a/packages/tool-server/src/tools/restart-app/platforms/ios.ts b/packages/tool-server/src/tools/restart-app/platforms/ios.ts index a02e7050d..d56865ea3 100644 --- a/packages/tool-server/src/tools/restart-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/restart-app/platforms/ios.ts @@ -8,8 +8,11 @@ const execFileAsync = promisify(execFile); export const iosImpl: PlatformImpl = { requires: ["xcrun"], - handler: async (services, params) => { + handler: async (services, params, device) => { const { udid, bundleId } = params; + if (device.kind === "device") { + throw new Error("restart-app is not supported on physical iOS devices."); + } const blocked = await precheckNativeDevtools(services.nativeDevtools, udid); if (blocked) return blocked; try { diff --git a/packages/tool-server/src/tools/screenshot/index.ts b/packages/tool-server/src/tools/screenshot/index.ts index 80ff4ccca..e46334f9d 100644 --- a/packages/tool-server/src/tools/screenshot/index.ts +++ b/packages/tool-server/src/tools/screenshot/index.ts @@ -2,7 +2,8 @@ import { z } from "zod"; import type { ServiceRef, ToolCapability, ToolDefinition } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "../../blueprints/simulator-server"; import { chromiumCdpRef, type ChromiumCdpApi } from "../../blueprints/chromium-cdp"; -import { resolveDevice } from "../../utils/device-info"; +import { coreDeviceRef, type CoreDeviceApi } from "../../blueprints/core-device"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { httpScreenshot } from "../../utils/simulator-client"; import { requireArtifacts, type ArtifactHandle } from "../../artifacts"; @@ -73,6 +74,9 @@ Fails if the simulator-server / emulator backend / Chromium CDP is not reachable if (device.platform === "chromium") { return { chromium: chromiumCdpRef(device) }; } + if (isPhysicalIos(device)) { + return { coreDevice: coreDeviceRef(device) }; + } return { simulatorServer: simulatorServerRef(device) }; }, async execute(services, params, ctx) { @@ -87,6 +91,14 @@ Fails if the simulator-server / emulator backend / Chromium CDP is not reachable const image = await requireArtifacts(ctx).register(path, { mimeType: "image/png" }); return { image }; } + if (isPhysicalIos(device)) { + // CoreDevice returns a full-resolution PNG; rotation/scale/downscaler are + // simulator/Chromium-only knobs and don't apply to the device capture. + const coreDevice = services.coreDevice as CoreDeviceApi; + const { path } = await coreDevice.screenshot(); + const image = await requireArtifacts(ctx).register(path, { mimeType: "image/png" }); + return { image }; + } const api = services.simulatorServer as SimulatorServerApi; const signal = ctx?.signal ?? AbortSignal.timeout(16_000); const { path } = await httpScreenshot(api, params.rotation, signal, params.scale); diff --git a/packages/tool-server/src/utils/device-info.ts b/packages/tool-server/src/utils/device-info.ts index fe41bc367..ecccc0948 100644 --- a/packages/tool-server/src/utils/device-info.ts +++ b/packages/tool-server/src/utils/device-info.ts @@ -11,12 +11,29 @@ import type { DeviceInfo, DeviceKind, Platform } from "@argent/registry"; const IOS_UDID_SHAPE = /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$/; +/** + * Physical iPhone/iPad UDID shape on Apple silicon devices (A12+/iOS 17+): + * an 8-hex ECID prefix, a single dash, then 16 hex — e.g. + * `00008120-000E6D0C0ABBA01E`. This is distinct from the simulator UUID + * (four dashes) so a real device can be told apart from a simulator by shape + * alone, the same way Android emulators vs phones are distinguished. Older + * 40-hex device UDIDs belong to pre-A12 hardware that tops out well below the + * iOS 27 floor for the CoreDevice control path, so they are intentionally not matched. + */ +const IOS_PHYSICAL_UDID_SHAPE = /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}$/; + export const CHROMIUM_ID_PREFIX = "chromium-cdp-"; +/** Whether a udid is a physical iOS device (vs a simulator UUID), by shape. */ +export function isPhysicalIosUdid(udid: string): boolean { + return IOS_PHYSICAL_UDID_SHAPE.test(udid); +} + /** Returns the platform a `udid` belongs to based on its shape. */ export function classifyDevice(udid: string): Platform { if (udid.startsWith(CHROMIUM_ID_PREFIX)) return "chromium"; - return IOS_UDID_SHAPE.test(udid) ? "ios" : "android"; + if (IOS_UDID_SHAPE.test(udid) || IOS_PHYSICAL_UDID_SHAPE.test(udid)) return "ios"; + return "android"; } /** @@ -47,7 +64,9 @@ export function resolveDevice(udid: string): DeviceInfo { const platform = classifyDevice(udid); const kind: DeviceKind = platform === "ios" - ? "simulator" + ? isPhysicalIosUdid(udid) + ? "device" + : "simulator" : platform === "android" ? isAndroidEmulatorSerial(udid) ? "emulator" @@ -56,6 +75,11 @@ export function resolveDevice(udid: string): DeviceInfo { return { id: udid, platform, kind }; } +/** A physical iOS device (driven via CoreDevice/pymobiledevice3, not the simulator-server). */ +export function isPhysicalIos(device: DeviceInfo): boolean { + return device.platform === "ios" && device.kind === "device"; +} + /** Parses the CDP port out of a chromium device id. Returns null if the id is malformed. */ export function parseChromiumCdpPort(udid: string): number | null { if (!udid.startsWith(CHROMIUM_ID_PREFIX)) return null; diff --git a/packages/tool-server/src/utils/ios-devices.ts b/packages/tool-server/src/utils/ios-devices.ts index 8a9530b8a..68da26bb0 100644 --- a/packages/tool-server/src/utils/ios-devices.ts +++ b/packages/tool-server/src/utils/ios-devices.ts @@ -1,5 +1,9 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readFile, rm } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; const execFileAsync = promisify(execFile); @@ -10,6 +14,25 @@ export interface IosSimulator { runtime: string; } +export interface IosPhysicalDevice { + udid: string; + name: string; + /** Apple product type, e.g. "iPhone15,4". Null when devicectl omits it. */ + productType: string | null; + /** Always "connected" — only currently-reachable devices are returned. */ + state: string; +} + +interface DevicectlDevice { + hardwareProperties?: { udid?: string; platform?: string; productType?: string }; + deviceProperties?: { name?: string }; + connectionProperties?: { transportType?: string; tunnelState?: string }; +} + +interface DevicectlOutput { + result?: { devices?: DevicectlDevice[] }; +} + interface SimctlDevice { udid: string; name: string; @@ -46,3 +69,53 @@ export async function listIosSimulators(): Promise { return []; } } + +/** + * List connected physical iOS devices via `xcrun devicectl list devices`. + * + * devicectl only emits stable machine output to a file (`--json-output`), never + * stdout, so we write to a temp file and parse it. We keep only devices that are + * actually reachable right now: iOS platform with a `connectionProperties.transportType` + * (wired/network). Paired-but-offline devices carry a `tunnelState: "unavailable"` + * and no `transportType`, and are dropped — listing them would invite taps that + * can't land. Returns an empty array on any failure so the rest of `list-devices` + * stays usable on non-mac hosts or without Xcode. + */ +export function parsePhysicalIosDevices(data: DevicectlOutput): IosPhysicalDevice[] { + const out: IosPhysicalDevice[] = []; + for (const d of data.result?.devices ?? []) { + const udid = d.hardwareProperties?.udid; + const platform = d.hardwareProperties?.platform; + const transport = d.connectionProperties?.transportType; + // iOS only (skip watchOS/tvOS), and only currently-connected devices: a + // reachable device reports a `transportType` (wired/network); paired-but- + // offline ones carry `tunnelState: "unavailable"` and no transport. + if (!udid || platform !== "iOS" || !transport) continue; + if (d.connectionProperties?.tunnelState === "unavailable") continue; + out.push({ + udid, + name: d.deviceProperties?.name ?? "iPhone", + productType: d.hardwareProperties?.productType ?? null, + state: "connected", + }); + } + return out; +} + +export async function listIosDevices(): Promise { + if (process.platform !== "darwin") return []; + const outPath = join(tmpdir(), `argent-devicectl-${randomUUID()}.json`); + try { + await execFileAsync( + "xcrun", + ["devicectl", "list", "devices", "--quiet", "--json-output", outPath], + { timeout: 15_000 } + ); + const data: DevicectlOutput = JSON.parse(await readFile(outPath, "utf8")); + return parsePhysicalIosDevices(data); + } catch { + return []; + } finally { + await rm(outPath, { force: true }).catch(() => {}); + } +} diff --git a/packages/tool-server/src/utils/setup-registry.ts b/packages/tool-server/src/utils/setup-registry.ts index fd5944203..9dc435cc7 100644 --- a/packages/tool-server/src/utils/setup-registry.ts +++ b/packages/tool-server/src/utils/setup-registry.ts @@ -1,5 +1,6 @@ import { Registry } from "@argent/registry"; import { simulatorServerBlueprint } from "../blueprints/simulator-server"; +import { coreDeviceBlueprint } from "../blueprints/core-device"; import { nativeDevtoolsBlueprint } from "../blueprints/native-devtools"; import { androidDevtoolsBlueprint } from "../blueprints/android-devtools"; import { axServiceBlueprint } from "../blueprints/ax-service"; @@ -81,6 +82,7 @@ export function createRegistry(): Registry { const registry = new Registry(); registry.registerBlueprint(simulatorServerBlueprint); + registry.registerBlueprint(coreDeviceBlueprint); registry.registerBlueprint(jsRuntimeDebuggerBlueprint); registry.registerBlueprint(networkInspectorBlueprint); registry.registerBlueprint(reactProfilerSessionBlueprint); diff --git a/packages/tool-server/test/physical-ios.test.ts b/packages/tool-server/test/physical-ios.test.ts new file mode 100644 index 000000000..625cf07a4 --- /dev/null +++ b/packages/tool-server/test/physical-ios.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect } from "vitest"; +import { + classifyDevice, + resolveDevice, + isPhysicalIos, + isPhysicalIosUdid, +} from "../src/utils/device-info"; +import { parsePhysicalIosDevices } from "../src/utils/ios-devices"; +import { toHid, tunneldStartCommand, appleScriptQuote } from "../src/blueprints/core-device"; +import { launchAppTool } from "../src/tools/launch-app"; +import { restartAppTool } from "../src/tools/restart-app"; + +// A real iPhone's UDID: 8-hex ECID, one dash, 16 hex. +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; +const SIM_UDID = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; + +describe("physical iOS classification", () => { + it("classifies a physical iPhone UDID as ios", () => { + expect(classifyDevice(PHYSICAL_UDID)).toBe("ios"); + }); + + it("resolves a physical iPhone UDID to ios + device", () => { + const d = resolveDevice(PHYSICAL_UDID); + expect(d.platform).toBe("ios"); + expect(d.kind).toBe("device"); + expect(d.id).toBe(PHYSICAL_UDID); + }); + + it("still resolves a simulator UUID to ios + simulator", () => { + const d = resolveDevice(SIM_UDID); + expect(d.platform).toBe("ios"); + expect(d.kind).toBe("simulator"); + }); + + it("isPhysicalIosUdid distinguishes device from simulator shapes", () => { + expect(isPhysicalIosUdid(PHYSICAL_UDID)).toBe(true); + expect(isPhysicalIosUdid(SIM_UDID)).toBe(false); + }); + + it("isPhysicalIos is true only for ios+device", () => { + expect(isPhysicalIos(resolveDevice(PHYSICAL_UDID))).toBe(true); + expect(isPhysicalIos(resolveDevice(SIM_UDID))).toBe(false); + expect(isPhysicalIos(resolveDevice("emulator-5554"))).toBe(false); + }); + + it("does not reclassify Android serials as physical iOS", () => { + // No 8hex-16hex Android serials in the wild; guard against regressions. + expect(classifyDevice("HT82A0203045")).toBe("android"); + expect(classifyDevice("192.168.1.5:5555")).toBe("android"); + expect(resolveDevice("HT82A0203045").kind).toBe("device"); + }); +}); + +describe("parsePhysicalIosDevices (devicectl JSON)", () => { + const sample = { + result: { + devices: [ + // Connected iPhone — kept. + { + hardwareProperties: { udid: PHYSICAL_UDID, platform: "iOS", productType: "iPhone15,4" }, + deviceProperties: { name: "iPhone 15" }, + connectionProperties: { transportType: "wired", tunnelState: "disconnected" }, + }, + // Paired but offline (no transportType, tunnelState unavailable) — dropped. + { + hardwareProperties: { + udid: "00008030-00096526219B802E", + platform: "iOS", + productType: "iPhone12,8", + }, + deviceProperties: { name: "Old iPhone" }, + connectionProperties: { tunnelState: "unavailable" }, + }, + // A connected Apple Watch — dropped (not iOS). + { + hardwareProperties: { + udid: "11111111-2222222222222222", + platform: "watchOS", + productType: "Watch7,1", + }, + deviceProperties: { name: "Watch" }, + connectionProperties: { transportType: "wired" }, + }, + ], + }, + }; + + it("keeps only connected iOS devices with the right fields", () => { + const out = parsePhysicalIosDevices(sample); + expect(out).toEqual([ + { udid: PHYSICAL_UDID, name: "iPhone 15", productType: "iPhone15,4", state: "connected" }, + ]); + }); + + it("returns [] for empty/missing input", () => { + expect(parsePhysicalIosDevices({})).toEqual([]); + expect(parsePhysicalIosDevices({ result: { devices: [] } })).toEqual([]); + }); +}); + +describe("toHid (normalized 0..1 → 0..65535)", () => { + it("maps endpoints and midpoint", () => { + expect(toHid(0)).toBe(0); + expect(toHid(1)).toBe(65535); + expect(toHid(0.5)).toBe(32768); + }); + + it("clamps out-of-range input", () => { + expect(toHid(-0.2)).toBe(0); + expect(toHid(1.5)).toBe(65535); + }); +}); + +describe("privileged tunnel start command", () => { + it("single-quotes the binary path, pins HOME, passes the port + daemonize flag", () => { + const cmd = tunneldStartCommand("/Users/me/.local/bin/pymobiledevice3", 49151); + expect(cmd).toContain("HOME=/var/root"); + expect(cmd).toContain("'/Users/me/.local/bin/pymobiledevice3' remote tunneld --port 49151 -d"); + }); + + it("escapes a single quote in the path so it can't break out of the sh word", () => { + const cmd = tunneldStartCommand("/Users/o'brien/pmd3", 5000); + // ' -> '\'' so the whole path stays one shell word + expect(cmd).toContain(`'/Users/o'\\''brien/pmd3' remote tunneld`); + }); + + it("escapes backslashes and double-quotes for the AppleScript string literal", () => { + expect(appleScriptQuote('a "b" c')).toBe('a \\"b\\" c'); + expect(appleScriptQuote("a\\b")).toBe("a\\\\b"); + }); +}); + +describe("lifecycle tools don't resolve native-devtools for physical iOS", () => { + // Regression guard: the registry resolves a tool's services() BEFORE execute(), + // and the native-devtools service throws a simulator-only guard for physical + // devices. So resolving it for a physical iPhone would break launch-app (a + // supported tool) and mask restart-app's intended rejection message. Simulators + // must still get it. + const params = (udid: string) => ({ udid, bundleId: "com.apple.Preferences" }); + + it("launch-app: omit for physical device, keep for simulator", () => { + expect(launchAppTool.services(params(PHYSICAL_UDID)).nativeDevtools).toBeUndefined(); + expect(launchAppTool.services(params(SIM_UDID)).nativeDevtools).toBeDefined(); + }); + + it("restart-app: omit for physical device, keep for simulator", () => { + expect(restartAppTool.services(params(PHYSICAL_UDID)).nativeDevtools).toBeUndefined(); + expect(restartAppTool.services(params(SIM_UDID)).nativeDevtools).toBeDefined(); + }); +}); diff --git a/scripts/download-native-binaries.sh b/scripts/download-native-binaries.sh index 65f23450b..ab11d8c57 100755 --- a/scripts/download-native-binaries.sh +++ b/scripts/download-native-binaries.sh @@ -63,6 +63,23 @@ gh release download "${TAG}" \ --clobber chmod +x "${IOS_BIN_DIR}/ax-service" +# argent-device-auth is the macOS host helper that shows the branded admin +# prompt to start the physical-iOS CoreDevice tunnel as root. Like ax-service it +# is a darwin-only binary, resolved at bin/darwin/argent-device-auth. +# Optional: until the argent-private build publishes it, the release won't carry +# it — physical-iOS then falls back to the (unbranded) osascript admin prompt, +# so a missing asset must not fail the whole download. +echo " Downloading argent-device-auth..." +if gh release download "${TAG}" \ + --repo "${REPO}" \ + --pattern "argent-device-auth" \ + --dir "${IOS_BIN_DIR}" \ + --clobber 2>/dev/null; then + chmod +x "${IOS_BIN_DIR}/argent-device-auth" +else + echo " (not in this release yet — physical iOS will use the osascript prompt fallback)" +fi + echo " Downloading argent-android-devtools.apk..." # The release publishes the APK under a stable name (no versioning in the # filename) so this script doesn't have to know the version ahead of time; @@ -85,7 +102,8 @@ trap - EXIT echo "Downloaded native binaries to ${DYLIBS_DIR}/, ${IOS_BIN_DIR}/, and ${ANDROID_BIN_DIR}/" if command -v codesign &>/dev/null; then - for f in "${DYLIBS_DIR}"/*.dylib "${IOS_BIN_DIR}/ax-service"; do + for f in "${DYLIBS_DIR}"/*.dylib "${IOS_BIN_DIR}/ax-service" "${IOS_BIN_DIR}/argent-device-auth"; do + [ -f "$f" ] || continue codesign -dvv "$f" 2>&1 || echo "Warning: signature verification failed for $f" done fi From e7a105f0ec1ed8c8817e0ebfa556e3ccb6f1c881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Mon, 22 Jun 2026 15:35:55 +0200 Subject: [PATCH 02/16] fix(preview): don't surface physical iOS devices in the simulator-only target list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preview / Argent Lens window streams frames over simulator-server, which refuses physical iOS devices (kind === "device", driven over CoreDevice). With the physical-ios-devices flag on, a connected iPhone leaked into GET /preview/simulators because the iOS branch matched any platform === "ios" without checking kind — surfacing a target the UI can't drive (and emitting an entry with no runtime under a type that asserted runtime: string). - Exclude physical iOS (kind === "device") from the preview list, mirroring the existing Chromium exclusion. - Consume list-devices' own exported ListDevicesResult type instead of a hand-rolled copy that had already drifted from it (the soundness hole tsc couldn't catch because invokeTool takes a caller-supplied generic). - Extract the mapping into a pure, exported devicesToPreviewEntries() and add regression tests asserting physical iOS + Chromium are excluded while the simulator and Android emulator are kept. --- packages/tool-server/src/preview.ts | 120 +++++++++--------- .../src/tools/devices/list-devices.ts | 2 +- .../tool-server/test/physical-ios.test.ts | 62 +++++++++ 3 files changed, 122 insertions(+), 62 deletions(-) diff --git a/packages/tool-server/src/preview.ts b/packages/tool-server/src/preview.ts index 8080ed158..258e5575e 100644 --- a/packages/tool-server/src/preview.ts +++ b/packages/tool-server/src/preview.ts @@ -5,7 +5,7 @@ import express from "express"; import type { Registry } from "@argent/registry"; import { simulatorServerRef, type SimulatorServerApi } from "./blueprints/simulator-server"; import { resolveDevice } from "./utils/device-info"; -import { listDevicesTool } from "./tools/devices/list-devices"; +import { listDevicesTool, type ListDevicesResult } from "./tools/devices/list-devices"; function findUiHtml(): string | null { // Candidate paths (first match wins): @@ -29,71 +29,69 @@ function wsUrlFromHttp(httpUrl: string): string { return `${scheme}//${u.host}/ws`; } +type PreviewEntry = { + udid: string; + name: string; + state: string; + runtime: string; + isAvailable: boolean; + platform: "ios" | "android"; +}; + +/** + * Map the `list-devices` result to the targets the preview UI can actually + * drive. The UI keys off `udid` and `state === "Booted"` (iOS terminology), so + * Android serials are remapped to that shape — `simulator-server/:udid` already + * accepts Android serials via `resolveDevice(udid)`. + * + * Chromium and physical iOS devices are intentionally excluded: the preview UI + * streams frames through simulator-server's WebSocket, which only exists for + * simulators / Android. simulator-server outright refuses physical iOS + * (kind === "device", driven over CoreDevice instead), so surfacing either + * would let the UI offer a target it can't drive. Those consumers should use + * the MCP tools (screenshot, describe, gesture-*) directly. + */ +export function devicesToPreviewEntries(devices: ListDevicesResult["devices"]): PreviewEntry[] { + return devices.flatMap((d) => { + if (d.platform === "ios") { + if (d.kind === "device") return []; + return [ + { + udid: d.udid, + name: d.name, + state: d.state, + runtime: d.runtime ?? "", + isAvailable: true, + platform: "ios", + }, + ]; + } + if (d.platform === "android") { + return [ + { + udid: d.serial, + name: d.avdName ?? d.model ?? d.serial, + state: d.state === "device" ? "Booted" : d.state, + runtime: d.sdkLevel != null ? `Android API ${d.sdkLevel}` : "Android", + isAvailable: true, + platform: "android", + }, + ]; + } + return []; + }); +} + export function createPreviewRouter(registry: Registry): Router { const router = express.Router(); router.get("/simulators", async (_req: Request, res: Response) => { try { - const data = await registry.invokeTool<{ - devices: Array< - | { platform: "ios"; udid: string; name: string; state: string; runtime: string } - | { - platform: "android"; - serial: string; - state: string; - avdName?: string; - model?: string; - sdkLevel?: number | null; - } - | { platform: "chromium"; id: string; title: string; port: number } - >; - }>(listDevicesTool.id); - // The preview UI keys off `udid` and `state === "Booted"`, which are - // iOS terminology. Map Android serials to the same shape so the same - // dropdown can target both platforms — `simulator-server/:udid` already - // accepts Android serials via `resolveDevice(udid)`. - // - // Chromium is intentionally excluded: the preview UI streams frames - // through simulator-server's WebSocket, which only exists for iOS / - // Android. Surfacing chromium entries would let the UI offer a target - // it can't actually drive. Chromium consumers should use the MCP tools - // (screenshot, describe, gesture-*) directly. - type PreviewEntry = { - udid: string; - name: string; - state: string; - runtime: string; - isAvailable: boolean; - platform: "ios" | "android"; - }; - const simulators = data.devices.flatMap((d) => { - if (d.platform === "ios") { - return [ - { - udid: d.udid, - name: d.name, - state: d.state, - runtime: d.runtime, - isAvailable: true, - platform: "ios", - }, - ]; - } - if (d.platform === "android") { - return [ - { - udid: d.serial, - name: d.avdName ?? d.model ?? d.serial, - state: d.state === "device" ? "Booted" : d.state, - runtime: d.sdkLevel != null ? `Android API ${d.sdkLevel}` : "Android", - isAvailable: true, - platform: "android", - }, - ]; - } - return []; - }); - res.json({ simulators }); + // Reuse list-devices' own result type rather than a hand-rolled copy, so + // the preview's view of a device can't silently drift from what the tool + // actually returns (e.g. physical-iOS entries carrying `kind`/no runtime). + const data = await registry.invokeTool(listDevicesTool.id); + res.json({ simulators: devicesToPreviewEntries(data.devices) }); } catch (err) { res.status(500).json({ error: err instanceof Error ? err.message : String(err) }); } diff --git a/packages/tool-server/src/tools/devices/list-devices.ts b/packages/tool-server/src/tools/devices/list-devices.ts index 485aaa8f5..c48d5926f 100644 --- a/packages/tool-server/src/tools/devices/list-devices.ts +++ b/packages/tool-server/src/tools/devices/list-devices.ts @@ -36,7 +36,7 @@ type AndroidDevice = { sdkLevel: number | null; }; -type ListDevicesResult = { +export type ListDevicesResult = { devices: Array; avds: Array<{ name: string }>; }; diff --git a/packages/tool-server/test/physical-ios.test.ts b/packages/tool-server/test/physical-ios.test.ts index 625cf07a4..120255ca4 100644 --- a/packages/tool-server/test/physical-ios.test.ts +++ b/packages/tool-server/test/physical-ios.test.ts @@ -9,6 +9,8 @@ import { parsePhysicalIosDevices } from "../src/utils/ios-devices"; import { toHid, tunneldStartCommand, appleScriptQuote } from "../src/blueprints/core-device"; import { launchAppTool } from "../src/tools/launch-app"; import { restartAppTool } from "../src/tools/restart-app"; +import { devicesToPreviewEntries } from "../src/preview"; +import type { ListDevicesResult } from "../src/tools/devices/list-devices"; // A real iPhone's UDID: 8-hex ECID, one dash, 16 hex. const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; @@ -148,3 +150,63 @@ describe("lifecycle tools don't resolve native-devtools for physical iOS", () => expect(restartAppTool.services(params(SIM_UDID)).nativeDevtools).toBeDefined(); }); }); + +describe("preview target list excludes targets it can't stream", () => { + // Regression guard: the preview / Lens UI streams frames over simulator-server, + // which refuses physical iOS (kind === "device", driven over CoreDevice) and + // never serves Chromium. Those must not leak into the selectable target list. + const devices: ListDevicesResult["devices"] = [ + { + platform: "ios", + udid: SIM_UDID, + name: "iPhone 16 Pro", + state: "Booted", + kind: "simulator", + runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-5", + }, + { + platform: "ios", + udid: PHYSICAL_UDID, + name: "iPhone 15", + state: "connected", + kind: "device", + productType: "iPhone15,4", + }, + { + platform: "android", + serial: "emulator-5554", + state: "device", + isEmulator: true, + kind: "emulator", + model: "Pixel 7", + avdName: "Pixel_7_API_33", + sdkLevel: 34, + }, + { + platform: "chromium", + id: "chromium-1", + title: "App", + port: 9222, + url: "about:blank", + browser: "Chrome/120", + state: "Running", + }, + ]; + + it("includes the simulator and the Android emulator", () => { + const entries = devicesToPreviewEntries(devices); + expect(entries.map((e) => e.udid).sort()).toEqual([SIM_UDID, "emulator-5554"].sort()); + }); + + it("excludes the physical iPhone and the Chromium app", () => { + const entries = devicesToPreviewEntries(devices); + expect(entries.some((e) => e.udid === PHYSICAL_UDID)).toBe(false); + expect(entries.some((e) => e.platform !== "ios" && e.platform !== "android")).toBe(false); + }); + + it("keeps `runtime` a string for a simulator that reports one", () => { + const [sim] = devicesToPreviewEntries(devices); + expect(typeof sim!.runtime).toBe("string"); + expect(sim!.runtime.length).toBeGreaterThan(0); + }); +}); From 9cb76fb71ce42cee7fd958b5f50eea628d8b1643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Mon, 22 Jun 2026 17:52:34 +0200 Subject: [PATCH 03/16] docs(physical-ios): align capability strings with launch-app/restart-app behavior list-devices description and the simulator-server guard message listed screenshot/gesture-tap/gesture-swipe/button but omitted launch-app, which is implemented for physical iOS via devicectl. The README limitations list omitted restart-app, which (like open-url/reinstall-app) returns a clear "not supported" error on physical iOS. --- README.md | 2 +- packages/tool-server/src/blueprints/simulator-server.ts | 2 +- packages/tool-server/src/tools/devices/list-devices.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a13e6b6cf..6b9c391a6 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ and leave it running: `sudo pymobiledevice3 remote tunneld`. **Limitations / notes** - Not supported on physical iOS yet: keyboard/typing, pinch & rotate (multi-touch), - `open-url`, and `reinstall-app` — these return a clear "not supported" error. + `open-url`, `reinstall-app`, and `restart-app` — these return a clear "not supported" error. - Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT` (defaults to `49151`). diff --git a/packages/tool-server/src/blueprints/simulator-server.ts b/packages/tool-server/src/blueprints/simulator-server.ts index f2d576ab3..2871410eb 100644 --- a/packages/tool-server/src/blueprints/simulator-server.ts +++ b/packages/tool-server/src/blueprints/simulator-server.ts @@ -194,7 +194,7 @@ export const simulatorServerBlueprint: ServiceBlueprint, ListDevicesR Use at the start of a session to pick a target id ('udid' for iOS entries, 'serial' for Android, 'id' for Chromium) to pass to interaction tools, and to see which targets are already running. Returns { devices, avds } where each device carries a 'platform' discriminator ('ios', 'android', or 'chromium'), and 'avds' lists Android AVDs that can be booted via boot-device. Android entries also carry a 'kind' ('emulator' for a local AVD, 'device' for a physical phone connected over USB / wireless adb) — physical phones are detected from \`adb devices\` (any serial that is not an \`emulator-*\` one) and are driven through the same interaction tools as emulators; they do not need boot-device (just connect the phone with USB debugging authorised). -iOS entries likewise carry a 'kind' ('simulator', or 'device' for a connected physical iPhone). Physical iOS devices require the 'physical-ios-devices' flag (\`argent enable physical-ios-devices\`), iOS 27+, and a running CoreDevice tunnel (\`sudo pymobiledevice3 remote tunneld\`); they support screenshot, gesture-tap, gesture-swipe, and button. +iOS entries likewise carry a 'kind' ('simulator', or 'device' for a connected physical iPhone). Physical iOS devices require the 'physical-ios-devices' flag (\`argent enable physical-ios-devices\`), iOS 27+, and a running CoreDevice tunnel (\`sudo pymobiledevice3 remote tunneld\`); they support screenshot, gesture-tap, gesture-swipe, button, and launch-app. Chromium apps are discovered by probing CDP debugging ports (default 9222; extend via the ARGENT_CHROMIUM_PORTS= env var). They must already be running with --remote-debugging-port= — use boot-device with chromiumAppPath to launch one. Booted/ready devices are listed first. Platforms whose CLI is unavailable are silently omitted — an empty result usually means xcode-select or Android platform-tools is not installed.`, alwaysLoad: true, From a957bf4433f7c139373e517f24e4cda7b8a53495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Fri, 26 Jun 2026 14:09:11 +0200 Subject: [PATCH 04/16] fix(physical-ios): correct device discovery, tool routing, and capability matrix Review follow-ups on the physical-iOS CoreDevice feature: - discovery: parsePhysicalIosDevices surfaced every host iOS simulator as a phantom physical device (devicectl lists simulators with transportType "sameMachine"); gate on the physical ECID UDID shape, which also aligns discovery with classifyDevice's routing. Verified against real devicectl JSON and a connected iPhone (returns only the device, not the host's simulators). - run-sequence: was eagerly resolving simulator-server for a physical iPhone, failing the whole call before step 1; hold no eager service so per-step CoreDevice routing works. - describe: swallowed the simulator-only guards and returned an empty tree with a misleading "reboot the simulator" hint; reject with a clear error instead. - swipe: command timeout was a fixed 15s, killing long swipes mid-drag; scale the timeout with the requested duration and clamp degenerate durations (extracted swipeDragParams, unit-tested). - native-profiler: guard the session blueprint so a physical iPhone isn't driven via simulator-only simctl (which mislabeled it an unresponsive "simulator"). - capability matrix: tools whose iOS backend is simulator-only (keyboard, paste, rotate, gesture-pinch/rotate/custom, screenshot-diff, native-devtools-*, native-profiler-start) now declare apple.device:false, so physical iOS is rejected with a clean 400 at the capability gate rather than a 500 from a deep service guard. open-url/reinstall-app/restart-app throw UnsupportedOperationError. - launch-app: wrap the devicectl launch so "app not installed/signed" surfaces a clean message instead of a raw subprocess blob. - core-device: check the feature flag before the pymobiledevice3 setup probe; echo a custom tunneld port in the manual-fallback hint. - docs: flags description and README now match the supported tool set and the auto-start prompt reality. Verified end-to-end on a physical iPhone (iOS 27): discovery, tunnel auto-escalation, screenshot (1179x2556), and swipe. --- README.md | 8 +- packages/configuration-core/src/flags.ts | 2 +- packages/registry/src/errors.ts | 1 + packages/registry/src/failure-codes.ts | 1 + .../tool-server/src/blueprints/core-device.ts | 39 ++- .../src/blueprints/native-profiler-session.ts | 16 ++ .../src/tools/describe/platforms/ios/index.ts | 17 ++ .../src/tools/gesture-custom/index.ts | 2 +- .../src/tools/gesture-pinch/index.ts | 2 +- .../src/tools/gesture-rotate/index.ts | 2 +- .../tool-server/src/tools/keyboard/index.ts | 2 +- .../src/tools/launch-app/platforms/ios.ts | 36 ++- .../native-devtools/native-describe-screen.ts | 2 +- .../native-devtools/native-devtools-status.ts | 2 +- .../native-devtools/native-find-views.ts | 2 +- .../native-devtools/native-full-hierarchy.ts | 2 +- .../native-devtools/native-network-logs.ts | 2 +- .../native-user-interactable-view-at-point.ts | 2 +- .../native-devtools/native-view-at-point.ts | 2 +- .../src/tools/open-url/platforms/ios.ts | 10 +- packages/tool-server/src/tools/paste/index.ts | 2 +- .../native-profiler/native-profiler-start.ts | 2 +- .../src/tools/reinstall-app/platforms/ios.ts | 9 +- .../src/tools/restart-app/platforms/ios.ts | 10 +- .../tool-server/src/tools/rotate/index.ts | 2 +- .../src/tools/run-sequence/index.ts | 11 +- .../src/tools/screenshot-diff/index.ts | 2 +- packages/tool-server/src/utils/ios-devices.ts | 16 +- .../test/physical-ios-followups.test.ts | 265 ++++++++++++++++++ 29 files changed, 430 insertions(+), 41 deletions(-) create mode 100644 packages/tool-server/test/physical-ios-followups.test.ts diff --git a/README.md b/README.md index dcc33cef4..d652e4bd6 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,11 @@ up in `list-devices` with `kind: "device"`. `list-devices` then includes the iPhone, and the supported tools work against its UDID. The first interaction (or `boot-device`) starts the required CoreDevice tunnel -automatically: Argent shows a standard macOS authorization prompt — branded as Argent, -with Touch ID / password — to launch `pymobiledevice3 remote tunneld` as root (creating the -tunnel interface needs root once; every other command runs unprivileged). No manual `sudo`. +automatically: Argent shows a standard macOS authorization prompt (Touch ID / password) +to launch `pymobiledevice3 remote tunneld` as root (creating the tunnel interface needs +root once; every other command runs unprivileged). No manual `sudo`. When the signed +`argent-device-auth` helper is installed, the prompt is branded as Argent; otherwise it's +the system's default admin prompt. If the prompt is declined or there's no GUI session (headless), start the tunnel manually and leave it running: `sudo pymobiledevice3 remote tunneld`. diff --git a/packages/configuration-core/src/flags.ts b/packages/configuration-core/src/flags.ts index 191aa1847..cefc3a8eb 100644 --- a/packages/configuration-core/src/flags.ts +++ b/packages/configuration-core/src/flags.ts @@ -60,7 +60,7 @@ export const FLAG_REGISTRY: readonly FlagDefinition[] = [ { name: "physical-ios-devices", description: - "Discover and control physical iOS devices (iOS 27+) over Apple's CoreDevice tunnel via pymobiledevice3. Requires a running `sudo pymobiledevice3 remote tunneld`. Supports screenshot, tap, swipe, and hardware buttons.", + "Discover and control physical iOS devices (iOS 27+) over Apple's CoreDevice tunnel via pymobiledevice3. The required tunnel is auto-started via a macOS admin prompt (or run `sudo pymobiledevice3 remote tunneld` manually). Supports screenshot, tap, swipe, hardware buttons, and launch-app.", }, ]; diff --git a/packages/registry/src/errors.ts b/packages/registry/src/errors.ts index cb7ca26e3..b5d982a26 100644 --- a/packages/registry/src/errors.ts +++ b/packages/registry/src/errors.ts @@ -26,6 +26,7 @@ export const FAILURE_COMMANDS = [ "emulator", "vega", "xcrun_simctl", + "xcrun_devicectl", "xctrace", "native_devtools", "android_devtools", diff --git a/packages/registry/src/failure-codes.ts b/packages/registry/src/failure-codes.ts index 5e166c9e8..acb6c04b7 100644 --- a/packages/registry/src/failure-codes.ts +++ b/packages/registry/src/failure-codes.ts @@ -83,6 +83,7 @@ export const FAILURE_CODES = { ANDROID_REINSTALL_INSTALL_FAILED: "ANDROID_REINSTALL_INSTALL_FAILED", ANDROID_RESTART_FAILED: "ANDROID_RESTART_FAILED", IOS_LAUNCH_SIMCTL_FAILED: "IOS_LAUNCH_SIMCTL_FAILED", + IOS_LAUNCH_DEVICECTL_FAILED: "IOS_LAUNCH_DEVICECTL_FAILED", IOS_OPEN_URL_FAILED: "IOS_OPEN_URL_FAILED", IOS_REINSTALL_INSTALL_FAILED: "IOS_REINSTALL_INSTALL_FAILED", IOS_RESTART_LAUNCH_FAILED: "IOS_RESTART_LAUNCH_FAILED", diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts index ef9280ec0..1a58c9618 100644 --- a/packages/tool-server/src/blueprints/core-device.ts +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -196,11 +196,15 @@ let tunnelStartInFlight: Promise | null = null; function tunnelHelp(udid: string, reason: string): string { const port = tunneldPort(); + // Echo the custom port in the manual command — otherwise a user who set + // ARGENT_PMD3_TUNNELD_PORT would start tunneld on pmd3's default (49151) and + // discovery, which probes the custom port, would never find it. + const portFlag = port === 49151 ? "" : ` --port ${port}`; return ( `Physical iOS control needs a CoreDevice tunnel for ${udid}, but ${reason} ` + `(checked tunneld at 127.0.0.1:${port}). Argent tries to start it automatically via the macOS ` + `authorization prompt; if that was declined or no GUI session is available, start it manually ` + - `and leave it running:\n sudo pymobiledevice3 remote tunneld\n` + + `and leave it running:\n sudo pymobiledevice3 remote tunneld${portFlag}\n` + `Also ensure the iPhone is on iOS 27+, unlocked, and trusted. ` + `(Override the port with ARGENT_PMD3_TUNNELD_PORT.)` ); @@ -332,6 +336,25 @@ export function toHid(v: number): number { return Math.max(0, Math.min(65535, Math.round(v * 65535))); } +/** + * Derive the pmd3 `drag` parameters for a swipe of `durationMs`. A drag must + * dwell to register, so degenerate inputs are clamped: 0ms is dropped by iOS + * like a zero-dwell tap, and a negative value would reach pmd3 as a flag-like + * "-0.100" arg; both floor to 50ms. A pathological value is capped at 60s so it + * can't pin the device for minutes. The command timeout scales with the drag so + * a long swipe isn't SIGTERM-killed mid-gesture (pmd3 runs for ~`dur` ms; the + * +15s buffer covers the interpreter's startup). + */ +export function swipeDragParams(durationMs: number): { + steps: number; + seconds: string; + timeoutMs: number; +} { + const dur = Math.max(50, Math.min(60_000, Math.round(durationMs))); + const steps = Math.max(2, Math.min(60, Math.round(dur / 16))); + return { steps, seconds: (dur / 1000).toFixed(3), timeoutMs: dur + 15_000 }; +} + /** * Ensure the personalized DeveloperDiskImage is mounted (the CoreDevice services * live in it). Idempotent: when already mounted, pymobiledevice3 exits non-zero @@ -370,6 +393,15 @@ export const coreDeviceBlueprint: ServiceBlueprint = `${CORE_DEVICE_NAMESPACE} only drives physical iOS devices; got ${device.platform}/${device.kind}.` ); } + // Gate first — before any setup probe — so a flag-disabled user gets the + // "enable the flag" message rather than an incidental "install pymobiledevice3" + // (ensureCoreDeviceTunnel re-checks this; the duplicate keeps the message + // correct regardless of whether pmd3 happens to be installed). + if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { + throw new Error( + `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}` + ); + } const pmd3 = resolvePmd3(); const udid = device.id; @@ -428,8 +460,7 @@ export const coreDeviceBlueprint: ServiceBlueprint = ); }, async swipe(fromX, fromY, toX, toY, durationMs) { - const steps = Math.max(2, Math.min(60, Math.round(durationMs / 16))); - const seconds = (durationMs / 1000).toFixed(3); + const { steps, seconds, timeoutMs } = swipeDragParams(durationMs); await run( "swipe", [ @@ -444,7 +475,7 @@ export const coreDeviceBlueprint: ServiceBlueprint = "--duration", seconds, ], - 15_000 + timeoutMs ); }, async button(name) { diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index 09e61237a..cf22847eb 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -115,6 +115,22 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< } ); } + if (device.platform === "ios" && device.kind === "device") { + // The iOS profiler path is simulator-only — it enumerates processes via + // `xcrun simctl spawn`, which rejects a physical UDID and then mislabels + // the iPhone as an unresponsive "simulator". Physical iPhones are driven + // over CoreDevice and have no native-profiler path yet. (Android physical + // devices ARE supported, hence the iOS-specific guard.) + throw new FailureError( + `${NATIVE_PROFILER_SESSION_NAMESPACE} is iOS-simulator-only; the physical device '${device.id}' is driven over CoreDevice and native profiling is not supported on physical iOS yet.`, + { + error_code: FAILURE_CODES.NATIVE_PROFILER_WRONG_PLATFORM, + failure_stage: "native_profiler_session_factory_options", + failure_area: "tool_server", + error_kind: "unsupported", + } + ); + } const state: NativeProfilerSessionApi = { deviceId: device.id, platform: device.platform, diff --git a/packages/tool-server/src/tools/describe/platforms/ios/index.ts b/packages/tool-server/src/tools/describe/platforms/ios/index.ts index 3f12757f9..64b0185b0 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/index.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/index.ts @@ -1,6 +1,8 @@ import type { DeviceInfo, Registry, ToolDependency } from "@argent/registry"; import { axServiceRef, AXServiceApi } from "../../../../blueprints/ax-service"; import { nativeDevtoolsRef, NativeDevtoolsApi } from "../../../../blueprints/native-devtools"; +import { isPhysicalIos } from "../../../../utils/device-info"; +import { UnsupportedOperationError } from "../../../../utils/capability"; import { resolveNativeTargetApp } from "../../../../utils/native-target-app"; import { parseNativeDescribeScreenResult } from "../../../native-devtools/native-describe-contract"; import { DescribeTreeData, parseDescribeResult, type DescribeNode } from "../../contract"; @@ -35,6 +37,21 @@ export async function describeIos( device: DeviceInfo, params: DescribeIosParams ): Promise { + // Physical iPhones are driven over CoreDevice; both describe backends are + // simulator-only (ax-service shells `simctl spawn`; native-devtools injects a + // dylib via `simctl spawn`). Their blueprint guards throw for kind === "device", + // but the fallback chain below catches those throws and would otherwise return + // an empty tree plus the "reboot the simulator" degraded hint — a misleading + // result for hardware that has no simulator to reboot. Reject explicitly with a + // clear, 400-mapped error instead. + if (isPhysicalIos(device)) { + throw new UnsupportedOperationError( + "describe", + device, + "physical iOS is driven over CoreDevice and has no on-device accessibility/describe path; use screenshot to inspect the screen" + ); + } + let tree: DescribeNode; let hint: string | undefined; diff --git a/packages/tool-server/src/tools/gesture-custom/index.ts b/packages/tool-server/src/tools/gesture-custom/index.ts index 06d95aa4f..9379b6fab 100644 --- a/packages/tool-server/src/tools/gesture-custom/index.ts +++ b/packages/tool-server/src/tools/gesture-custom/index.ts @@ -49,7 +49,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/gesture-pinch/index.ts b/packages/tool-server/src/tools/gesture-pinch/index.ts index 9de8dc018..c7c014b87 100644 --- a/packages/tool-server/src/tools/gesture-pinch/index.ts +++ b/packages/tool-server/src/tools/gesture-pinch/index.ts @@ -49,7 +49,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/gesture-rotate/index.ts b/packages/tool-server/src/tools/gesture-rotate/index.ts index b92334634..b127d4be0 100644 --- a/packages/tool-server/src/tools/gesture-rotate/index.ts +++ b/packages/tool-server/src/tools/gesture-rotate/index.ts @@ -39,7 +39,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/keyboard/index.ts b/packages/tool-server/src/tools/keyboard/index.ts index 6ec5b3c8e..cfa595e20 100644 --- a/packages/tool-server/src/tools/keyboard/index.ts +++ b/packages/tool-server/src/tools/keyboard/index.ts @@ -42,7 +42,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, chromium: { app: true }, vega: { vvd: true }, diff --git a/packages/tool-server/src/tools/launch-app/platforms/ios.ts b/packages/tool-server/src/tools/launch-app/platforms/ios.ts index ed6794a86..1905cb181 100644 --- a/packages/tool-server/src/tools/launch-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/launch-app/platforms/ios.ts @@ -14,15 +14,33 @@ export const iosImpl: PlatformImpl = { id: "native-describe-screen", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Read the running app's native accessibility screen description via injected native devtools. Returns a flat list of accessibility leaf elements with: diff --git a/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts b/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts index bea9460a1..a904259af 100644 --- a/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts +++ b/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts @@ -27,7 +27,7 @@ type Result = export const nativeDevtoolsStatusTool: ToolDefinition = { id: "native-devtools-status", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Check whether native devtools are connected to a specific app and whether the next launch is prepared for injection. Use when you need to verify native devtools readiness before calling native-full-hierarchy, native-describe-screen, or native-network-logs. diff --git a/packages/tool-server/src/tools/native-devtools/native-find-views.ts b/packages/tool-server/src/tools/native-devtools/native-find-views.ts index 40a42b270..105d29f13 100644 --- a/packages/tool-server/src/tools/native-devtools/native-find-views.ts +++ b/packages/tool-server/src/tools/native-devtools/native-find-views.ts @@ -44,7 +44,7 @@ type Result = export const nativeFindViewsTool: ToolDefinition = { id: "native-find-views", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Search for specific UIViews in the running app by class name, accessibility identifier, label, tag, or React Native nativeID. Use when you need to locate a specific view by its properties without dumping the entire hierarchy. Returns { status: "ok", matches } with matching views including their frames, properties, optional ancestors, and optional children. Much more targeted than native-full-hierarchy. diff --git a/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts b/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts index b44ec8116..3f15be483 100644 --- a/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts +++ b/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts @@ -58,7 +58,7 @@ type Result = export const nativeFullHierarchyTool: ToolDefinition = { id: "native-full-hierarchy", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Get the complete UIKit view tree for the running app. WARNING: Output can be extremely large (100KB–500KB+) for complex apps, especially those built with SwiftUI. Prefer native-find-views for targeted queries. Use skipClasses / skipClassPrefixes to prune SwiftUI internal subtrees and reduce output size. Use the fields param to request only the properties you need. diff --git a/packages/tool-server/src/tools/native-devtools/native-network-logs.ts b/packages/tool-server/src/tools/native-devtools/native-network-logs.ts index adc7ccf66..928456612 100644 --- a/packages/tool-server/src/tools/native-devtools/native-network-logs.ts +++ b/packages/tool-server/src/tools/native-devtools/native-network-logs.ts @@ -29,7 +29,7 @@ type Result = export const nativeNetworkLogsTool: ToolDefinition = { id: "native-network-logs", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Retrieve network requests captured at the native NSURLProtocol level. Unlike the JS-level network inspector (view-network-logs), this captures ALL network traffic from the app including native modules, Swift/Objective-C networking, and background transfers that bypass JS fetch. Use when you need to inspect native-level HTTP traffic that is invisible to JS fetch interception. diff --git a/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts b/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts index e77919e09..fede7995c 100644 --- a/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts +++ b/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts @@ -63,7 +63,7 @@ type Result = export const nativeUserInteractableViewAtPointTool: ToolDefinition = { id: "native-user-interactable-view-at-point", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Inspect the deepest UIView at a raw native window point that would actually receive touch input. Unlike native-view-at-point, this respects userInteractionEnabled and is closer to diff --git a/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts b/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts index bd0125355..b0eb603d4 100644 --- a/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts +++ b/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts @@ -63,7 +63,7 @@ type Result = export const nativeViewAtPointTool: ToolDefinition = { id: "native-view-at-point", requires: ["xcrun"], - capability: { apple: { simulator: true, device: true } }, + capability: { apple: { simulator: true, device: false } }, description: `Inspect the deepest visible UIView at a raw native window point. Unlike native-user-interactable-view-at-point, this ignores userInteractionEnabled, diff --git a/packages/tool-server/src/tools/open-url/platforms/ios.ts b/packages/tool-server/src/tools/open-url/platforms/ios.ts index 9143c384f..720abfcff 100644 --- a/packages/tool-server/src/tools/open-url/platforms/ios.ts +++ b/packages/tool-server/src/tools/open-url/platforms/ios.ts @@ -2,6 +2,7 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { FAILURE_CODES, FailureError, subprocessFailureMetadata } from "@argent/registry"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; +import { UnsupportedOperationError } from "../../../utils/capability"; import type { OpenUrlParams, OpenUrlResult, OpenUrlServices } from "../types"; const execFileAsync = promisify(execFile); @@ -12,8 +13,13 @@ export const iosImpl: PlatformImpl = { diff --git a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts index bdde7fdfb..23f508e65 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-start.ts @@ -32,7 +32,7 @@ const zodSchema = z.object({ }); const capability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, } as const; diff --git a/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts b/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts index 983d673bf..29d1f064e 100644 --- a/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/reinstall-app/platforms/ios.ts @@ -3,6 +3,7 @@ import { promisify } from "node:util"; import { resolve as resolvePath } from "node:path"; import { FAILURE_CODES, FailureError, subprocessFailureMetadata } from "@argent/registry"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; +import { UnsupportedOperationError } from "../../../utils/capability"; import type { ReinstallAppParams, ReinstallAppResult, ReinstallAppServices } from "../types"; const execFileAsync = promisify(execFile); @@ -13,7 +14,13 @@ export const iosImpl: PlatformImpl { const { udid, bundleId } = params; if (device.kind === "device") { - throw new Error("restart-app is not supported on physical iOS devices."); + // Physical iOS is driven over CoreDevice and has no relaunch path (the + // native-devtools injection is simulator-only). UnsupportedOperationError + // maps to a clean 400 (a plain Error would surface as a generic 500). + throw new UnsupportedOperationError( + "restart-app", + device, + "physical iOS is driven over CoreDevice and has no relaunch path" + ); } const blocked = await precheckNativeDevtools(services.nativeDevtools, udid); if (blocked) return blocked; diff --git a/packages/tool-server/src/tools/rotate/index.ts b/packages/tool-server/src/tools/rotate/index.ts index 263bd3d66..e254c4775 100644 --- a/packages/tool-server/src/tools/rotate/index.ts +++ b/packages/tool-server/src/tools/rotate/index.ts @@ -18,7 +18,7 @@ interface Result { } const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/tools/run-sequence/index.ts b/packages/tool-server/src/tools/run-sequence/index.ts index 1b2a85ac1..d61ffad17 100644 --- a/packages/tool-server/src/tools/run-sequence/index.ts +++ b/packages/tool-server/src/tools/run-sequence/index.ts @@ -3,7 +3,7 @@ import type { Registry, ToolCapability, ToolDefinition } from "@argent/registry" import type { ServiceRef } from "@argent/registry"; import { simulatorServerRef } from "../../blueprints/simulator-server"; import { chromiumCdpRef } from "../../blueprints/chromium-cdp"; -import { resolveDevice } from "../../utils/device-info"; +import { resolveDevice, isPhysicalIos } from "../../utils/device-info"; import { assertSupported, UnsupportedOperationError } from "../../utils/capability"; import { sleep, DEFAULT_INTER_STEP_DELAY_MS } from "../../utils/timing"; @@ -125,6 +125,15 @@ Stops on the first error and returns partial results.`, if (device.platform === "chromium") { return { chromium: chromiumCdpRef(device) }; } + // Physical iOS is driven over CoreDevice, not simulator-server. Eagerly + // holding simulatorServerRef here would resolve the simulator-server + // blueprint, whose guard throws for kind === "device" — failing the whole + // sequence before step 1 even when every step is individually supported + // (gesture-tap/gesture-swipe/button). Hold nothing; each step resolves its + // own backend (CoreDevice) lazily via registry.invokeTool. + if (isPhysicalIos(device)) { + return {}; + } return { simulatorServer: simulatorServerRef(device) }; }, async execute(_services, params) { diff --git a/packages/tool-server/src/tools/screenshot-diff/index.ts b/packages/tool-server/src/tools/screenshot-diff/index.ts index 0440c57d2..cb5c3cece 100644 --- a/packages/tool-server/src/tools/screenshot-diff/index.ts +++ b/packages/tool-server/src/tools/screenshot-diff/index.ts @@ -74,7 +74,7 @@ export interface ScreenshotDiffResult { type CaptureScreenshot = typeof httpScreenshot; const capability: ToolCapability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, }; diff --git a/packages/tool-server/src/utils/ios-devices.ts b/packages/tool-server/src/utils/ios-devices.ts index 68da26bb0..e791c2f76 100644 --- a/packages/tool-server/src/utils/ios-devices.ts +++ b/packages/tool-server/src/utils/ios-devices.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { readFile, rm } from "node:fs/promises"; import { randomUUID } from "node:crypto"; +import { isPhysicalIosUdid } from "./device-info"; const execFileAsync = promisify(execFile); @@ -87,10 +88,17 @@ export function parsePhysicalIosDevices(data: DevicectlOutput): IosPhysicalDevic const udid = d.hardwareProperties?.udid; const platform = d.hardwareProperties?.platform; const transport = d.connectionProperties?.transportType; - // iOS only (skip watchOS/tvOS), and only currently-connected devices: a - // reachable device reports a `transportType` (wired/network); paired-but- - // offline ones carry `tunnelState: "unavailable"` and no transport. - if (!udid || platform !== "iOS" || !transport) continue; + // Keep only iOS (skip watchOS/tvOS), with a physical ECID UDID, that is + // currently reachable. The `isPhysicalIosUdid` (8hex-16hex) check is + // load-bearing: `devicectl list devices` also enumerates the host's iOS + // *simulators*, which report `platform: "iOS"` with + // `transportType: "sameMachine"` (verified against real devicectl JSON) — + // without the shape gate every simulator surfaces as a phantom physical + // device. It also keeps discovery consistent with `classifyDevice`, which + // routes only this UDID shape to the CoreDevice backend. A reachable device + // reports a `transportType` (wired/network); paired-but-offline ones carry + // `tunnelState: "unavailable"` and no transport, and are dropped. + if (!udid || platform !== "iOS" || !isPhysicalIosUdid(udid) || !transport) continue; if (d.connectionProperties?.tunnelState === "unavailable") continue; out.push({ udid, diff --git a/packages/tool-server/test/physical-ios-followups.test.ts b/packages/tool-server/test/physical-ios-followups.test.ts new file mode 100644 index 000000000..b438035ad --- /dev/null +++ b/packages/tool-server/test/physical-ios-followups.test.ts @@ -0,0 +1,265 @@ +/** + * Follow-up coverage for the physical-iOS (CoreDevice) feature. These tests pin + * behaviors that the original physical-ios.test.ts left uncovered and that a + * regression could silently break: + * + * - discovery must NOT surface the host's iOS simulators as phantom physical + * devices (devicectl enumerates them with transportType "sameMachine"); + * - the `button` CoreDevice HID mapping + rejection of buttons with no HID; + * - the privileged-tunnel flag gate (root escalation must be opt-in); + * - tools that are unsupported on physical iOS reject with a 400-mapped + * UnsupportedOperationError, not a generic 500 (open-url/reinstall/restart, + * describe, native-profiler), while staying supported on simulators/Android; + * - run-sequence must not eagerly hold simulator-server for a physical iPhone; + * - swipe duration clamping + timeout scaling. + */ +import { describe, it, expect, vi } from "vitest"; + +// core-device is the only module under test that reads the feature flag; mock it +// so the flag gate can be exercised deterministically regardless of the host's +// ~/.argent/flags.json. (See variant-flag-gate.test.ts for the same pattern.) +vi.mock("@argent/configuration-core", () => ({ isFlagEnabled: vi.fn() })); +import { isFlagEnabled } from "@argent/configuration-core"; + +import { resolveDevice, isPhysicalIosUdid } from "../src/utils/device-info"; +import { parsePhysicalIosDevices } from "../src/utils/ios-devices"; +import { UnsupportedOperationError, assertSupported } from "../src/utils/capability"; +import { swipeDragParams, ensureCoreDeviceTunnel } from "../src/blueprints/core-device"; +import { nativeProfilerSessionBlueprint } from "../src/blueprints/native-profiler-session"; +import { buttonTool } from "../src/tools/button"; +import { createRunSequenceTool } from "../src/tools/run-sequence"; +import { describeIos } from "../src/tools/describe/platforms/ios"; +import { iosImpl as openUrlIos } from "../src/tools/open-url/platforms/ios"; +import { iosImpl as reinstallIos } from "../src/tools/reinstall-app/platforms/ios"; +import { iosImpl as restartIos } from "../src/tools/restart-app/platforms/ios"; +import { gestureTapTool } from "../src/tools/gesture-tap"; +import { keyboardTool } from "../src/tools/keyboard"; +import { gesturePinchTool } from "../src/tools/gesture-pinch"; +import { screenshotDiffTool } from "../src/tools/screenshot-diff"; +import { nativeDescribeScreenTool } from "../src/tools/native-devtools/native-describe-screen"; +import { nativeProfilerStartTool } from "../src/tools/profiler/native-profiler/native-profiler-start"; + +const mockFlag = vi.mocked(isFlagEnabled); + +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; +const SIM_UDID = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; + +describe("discovery does not surface simulators as physical devices", () => { + // Real `xcrun devicectl list devices` JSON also lists every host iOS + // simulator (platform "iOS", transportType "sameMachine"); without the + // UDID-shape gate those leak in as phantom physical devices. + const data = { + result: { + devices: [ + // The real, connected iPhone — KEPT. + { + hardwareProperties: { udid: PHYSICAL_UDID, platform: "iOS", productType: "iPhone15,4" }, + deviceProperties: { name: "Real iPhone" }, + connectionProperties: { transportType: "wired", tunnelState: "disconnected" }, + }, + // A booted iOS simulator (UUID shape, sameMachine transport) — DROPPED. + { + hardwareProperties: { udid: SIM_UDID, platform: "iOS", productType: "iPhone17,2" }, + deviceProperties: { name: "iPhone 16 Pro Max" }, + connectionProperties: { transportType: "sameMachine", tunnelState: "connected" }, + }, + // A shut-down simulator (UUID shape) — DROPPED (would otherwise be + // reported as a "connected" device). + { + hardwareProperties: { + udid: "39646432-58B6-4A21-923A-00F0EDE4FF81", + platform: "iOS", + productType: "iPhone17,3", + }, + deviceProperties: { name: "iPhone 16" }, + connectionProperties: { transportType: "sameMachine", tunnelState: "disconnected" }, + }, + // A paired-but-offline real device (physical shape, no transport) — DROPPED. + { + hardwareProperties: { + udid: "00008030-00096526219B802E", + platform: "iOS", + productType: "iPhone12,8", + }, + deviceProperties: { name: "Old iPhone" }, + connectionProperties: { tunnelState: "unavailable" }, + }, + ], + }, + }; + + it("returns only the real physical iPhone", () => { + const out = parsePhysicalIosDevices(data); + expect(out).toEqual([ + { udid: PHYSICAL_UDID, name: "Real iPhone", productType: "iPhone15,4", state: "connected" }, + ]); + }); + + it("every returned device has a physical-shape UDID", () => { + for (const d of parsePhysicalIosDevices(data)) { + expect(isPhysicalIosUdid(d.udid)).toBe(true); + } + }); +}); + +describe("button — CoreDevice HID mapping on physical iOS", () => { + const press = async (button: string) => { + const coreDevice = { button: vi.fn().mockResolvedValue(undefined) }; + const res = await buttonTool.execute({ coreDevice } as never, { + udid: PHYSICAL_UDID, + button, + } as never); + return { coreDevice, res }; + }; + + it("maps the four supported buttons to their pymobiledevice3 HID names", async () => { + for (const [argent, hid] of [ + ["home", "home"], + ["power", "lock"], + ["volumeUp", "volume-up"], + ["volumeDown", "volume-down"], + ]) { + const { coreDevice, res } = await press(argent); + expect(coreDevice.button).toHaveBeenCalledWith(hid); + expect(res).toEqual({ pressed: argent }); + } + }); + + it("rejects buttons with no CoreDevice HID equivalent", async () => { + // appSwitch / actionButton exist on iOS but have no HID button. + await expect(press("appSwitch")).rejects.toBeInstanceOf(UnsupportedOperationError); + await expect(press("actionButton")).rejects.toBeInstanceOf(UnsupportedOperationError); + }); +}); + +describe("privileged tunnel start is gated behind the feature flag", () => { + it("rejects with the enable hint when physical-ios-devices is off", async () => { + mockFlag.mockReturnValue(false); + await expect(ensureCoreDeviceTunnel(PHYSICAL_UDID)).rejects.toThrow( + /Physical iOS support is disabled.*argent enable physical-ios-devices/s + ); + expect(mockFlag).toHaveBeenCalledWith("physical-ios-devices"); + }); +}); + +describe("tools unsupported on physical iOS reject with UnsupportedOperationError (400)", () => { + const device = resolveDevice(PHYSICAL_UDID); + + it("open-url", async () => { + await expect( + openUrlIos.handler({} as never, { udid: PHYSICAL_UDID, url: "https://x" } as never, device) + ).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + it("reinstall-app", async () => { + await expect( + reinstallIos.handler( + {} as never, + { udid: PHYSICAL_UDID, bundleId: "com.x", appPath: "/tmp/x.app" } as never, + device + ) + ).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + it("restart-app", async () => { + await expect( + restartIos.handler({} as never, { udid: PHYSICAL_UDID, bundleId: "com.x" } as never, device) + ).rejects.toBeInstanceOf(UnsupportedOperationError); + }); + + it("describe — rejects instead of returning a misleading empty tree", async () => { + await expect(describeIos({} as never, device, {})).rejects.toBeInstanceOf( + UnsupportedOperationError + ); + }); +}); + +describe("native-profiler-session blueprint", () => { + it("rejects a physical iPhone (iOS profiler path is simulator-only)", async () => { + await expect( + nativeProfilerSessionBlueprint.factory({} as never, undefined as never, { + device: resolveDevice(PHYSICAL_UDID), + }) + ).rejects.toThrow(/iOS-simulator-only|physical/); + }); + + it("still accepts an Android device (its physical-device path is supported)", async () => { + const instance = await nativeProfilerSessionBlueprint.factory({} as never, undefined as never, { + device: resolveDevice("emulator-5554"), + }); + expect(instance.api.platform).toBe("android"); + }); +}); + +describe("run-sequence does not eagerly hold simulator-server for physical iOS", () => { + const tool = createRunSequenceTool({} as never); + const params = (udid: string) => ({ udid, steps: [{ tool: "gesture-tap", args: {} }] }); + + it("holds no simulator-server service for a physical iPhone", () => { + const services = tool.services(params(PHYSICAL_UDID)); + expect(services.simulatorServer).toBeUndefined(); + expect(Object.keys(services)).toHaveLength(0); + }); + + it("still holds simulator-server for a simulator", () => { + expect(tool.services(params(SIM_UDID)).simulatorServer).toBeDefined(); + }); +}); + +describe("capability matrix is honest about physical-iOS support (clean 400 at the gate)", () => { + const physical = resolveDevice(PHYSICAL_UDID); + const sim = resolveDevice(SIM_UDID); + const androidEmu = resolveDevice("emulator-5554"); + + it("supported tools accept a physical iPhone", () => { + expect(() => assertSupported("gesture-tap", gestureTapTool.capability, physical)).not.toThrow(); + expect(() => assertSupported("button", buttonTool.capability, physical)).not.toThrow(); + }); + + it("simulator-only tools reject a physical iPhone via the capability gate", () => { + for (const [id, cap] of [ + ["keyboard", keyboardTool.capability], + ["gesture-pinch", gesturePinchTool.capability], + ["screenshot-diff", screenshotDiffTool.capability], + ["native-describe-screen", nativeDescribeScreenTool.capability], + ["native-profiler-start", nativeProfilerStartTool.capability], + ] as const) { + expect(() => assertSupported(id, cap, physical)).toThrow(UnsupportedOperationError); + // ...but still work on a simulator (no regression to simulator support). + expect(() => assertSupported(id, cap, sim)).not.toThrow(); + } + }); + + it("native-profiler-start still accepts a physical Android device", () => { + expect(() => assertSupported("native-profiler-start", nativeProfilerStartTool.capability, androidEmu)).not.toThrow(); + }); +}); + +describe("swipeDragParams — clamping and timeout scaling", () => { + it("clamps a typical swipe and scales the timeout past the drag duration", () => { + const p = swipeDragParams(300); + expect(p.seconds).toBe("0.300"); + expect(p.steps).toBe(19); + expect(p.timeoutMs).toBe(15_300); + }); + + it("a long swipe gets a timeout that outlasts the drag (the bug this fixes)", () => { + const p = swipeDragParams(20_000); + expect(p.seconds).toBe("20.000"); + expect(p.steps).toBe(60); // step count is capped + expect(p.timeoutMs).toBe(35_000); + expect(p.timeoutMs).toBeGreaterThan(20_000); + }); + + it("floors degenerate (zero/negative) durations to a real dwell", () => { + expect(swipeDragParams(0).seconds).toBe("0.050"); + expect(swipeDragParams(-100).seconds).toBe("0.050"); + expect(swipeDragParams(0).steps).toBeGreaterThanOrEqual(2); + }); + + it("caps a pathological duration", () => { + const p = swipeDragParams(10_000_000); + expect(p.seconds).toBe("60.000"); + expect(p.timeoutMs).toBe(75_000); + }); +}); From 54429bb25090269a4b0aa92b4323f8ae01e15bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Fri, 26 Jun 2026 14:28:04 +0200 Subject: [PATCH 05/16] fix(physical-ios): scope the native-profiler guard to the live-capture entry The previous commit guarded the shared native-profiler-session blueprint against physical iOS, but that blueprint also backs the device-agnostic analysis tools (profiler-stack-query / profiler-combined-report / -analyze), which profiler-query-android-capability.test.ts intentionally keeps supported on Apple device. The guard made those analysis tools 500 on a physical-iOS device_id. Scope the fix to where the actual bug is: only native-profiler-start enumerates running processes via simulator-only `simctl` (mislabeling a real iPhone as an unresponsive "simulator"), and it already declares apple.device:false, so the capability gate rejects it cleanly (400) before execute() runs. Drop the redundant session-blueprint guard; the analysis tools keep their cross-device capability. Also formats physical-ios-followups.test.ts. --- .../src/blueprints/native-profiler-session.ts | 16 -------- .../test/physical-ios-followups.test.ts | 37 +++++++------------ 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/packages/tool-server/src/blueprints/native-profiler-session.ts b/packages/tool-server/src/blueprints/native-profiler-session.ts index cf22847eb..09e61237a 100644 --- a/packages/tool-server/src/blueprints/native-profiler-session.ts +++ b/packages/tool-server/src/blueprints/native-profiler-session.ts @@ -115,22 +115,6 @@ export const nativeProfilerSessionBlueprint: ServiceBlueprint< } ); } - if (device.platform === "ios" && device.kind === "device") { - // The iOS profiler path is simulator-only — it enumerates processes via - // `xcrun simctl spawn`, which rejects a physical UDID and then mislabels - // the iPhone as an unresponsive "simulator". Physical iPhones are driven - // over CoreDevice and have no native-profiler path yet. (Android physical - // devices ARE supported, hence the iOS-specific guard.) - throw new FailureError( - `${NATIVE_PROFILER_SESSION_NAMESPACE} is iOS-simulator-only; the physical device '${device.id}' is driven over CoreDevice and native profiling is not supported on physical iOS yet.`, - { - error_code: FAILURE_CODES.NATIVE_PROFILER_WRONG_PLATFORM, - failure_stage: "native_profiler_session_factory_options", - failure_area: "tool_server", - error_kind: "unsupported", - } - ); - } const state: NativeProfilerSessionApi = { deviceId: device.id, platform: device.platform, diff --git a/packages/tool-server/test/physical-ios-followups.test.ts b/packages/tool-server/test/physical-ios-followups.test.ts index b438035ad..69596f6c3 100644 --- a/packages/tool-server/test/physical-ios-followups.test.ts +++ b/packages/tool-server/test/physical-ios-followups.test.ts @@ -25,7 +25,6 @@ import { resolveDevice, isPhysicalIosUdid } from "../src/utils/device-info"; import { parsePhysicalIosDevices } from "../src/utils/ios-devices"; import { UnsupportedOperationError, assertSupported } from "../src/utils/capability"; import { swipeDragParams, ensureCoreDeviceTunnel } from "../src/blueprints/core-device"; -import { nativeProfilerSessionBlueprint } from "../src/blueprints/native-profiler-session"; import { buttonTool } from "../src/tools/button"; import { createRunSequenceTool } from "../src/tools/run-sequence"; import { describeIos } from "../src/tools/describe/platforms/ios"; @@ -105,10 +104,13 @@ describe("discovery does not surface simulators as physical devices", () => { describe("button — CoreDevice HID mapping on physical iOS", () => { const press = async (button: string) => { const coreDevice = { button: vi.fn().mockResolvedValue(undefined) }; - const res = await buttonTool.execute({ coreDevice } as never, { - udid: PHYSICAL_UDID, - button, - } as never); + const res = await buttonTool.execute( + { coreDevice } as never, + { + udid: PHYSICAL_UDID, + button, + } as never + ); return { coreDevice, res }; }; @@ -174,23 +176,6 @@ describe("tools unsupported on physical iOS reject with UnsupportedOperationErro }); }); -describe("native-profiler-session blueprint", () => { - it("rejects a physical iPhone (iOS profiler path is simulator-only)", async () => { - await expect( - nativeProfilerSessionBlueprint.factory({} as never, undefined as never, { - device: resolveDevice(PHYSICAL_UDID), - }) - ).rejects.toThrow(/iOS-simulator-only|physical/); - }); - - it("still accepts an Android device (its physical-device path is supported)", async () => { - const instance = await nativeProfilerSessionBlueprint.factory({} as never, undefined as never, { - device: resolveDevice("emulator-5554"), - }); - expect(instance.api.platform).toBe("android"); - }); -}); - describe("run-sequence does not eagerly hold simulator-server for physical iOS", () => { const tool = createRunSequenceTool({} as never); const params = (udid: string) => ({ udid, steps: [{ tool: "gesture-tap", args: {} }] }); @@ -222,6 +207,10 @@ describe("capability matrix is honest about physical-iOS support (clean 400 at t ["gesture-pinch", gesturePinchTool.capability], ["screenshot-diff", screenshotDiffTool.capability], ["native-describe-screen", nativeDescribeScreenTool.capability], + // native-profiler-start does LIVE capture via simulator-only simctl (the + // process enumeration mislabels a real iPhone as a "simulator"), so it + // rejects physical iOS at the gate. (Its post-capture sibling tools stay + // device-agnostic — see profiler-query-android-capability.test.ts.) ["native-profiler-start", nativeProfilerStartTool.capability], ] as const) { expect(() => assertSupported(id, cap, physical)).toThrow(UnsupportedOperationError); @@ -231,7 +220,9 @@ describe("capability matrix is honest about physical-iOS support (clean 400 at t }); it("native-profiler-start still accepts a physical Android device", () => { - expect(() => assertSupported("native-profiler-start", nativeProfilerStartTool.capability, androidEmu)).not.toThrow(); + expect(() => + assertSupported("native-profiler-start", nativeProfilerStartTool.capability, androidEmu) + ).not.toThrow(); }); }); From d155208b2f56277d891b5b072dbea57aac58b441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Fri, 26 Jun 2026 14:42:15 +0200 Subject: [PATCH 06/16] fix(physical-ios): gate launch-app behind the feature flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit launch-app drives a physical iPhone via `xcrun devicectl` directly rather than the CoreDevice service, so its handler never hit the flag check that gates every other physical-iOS operation — with `physical-ios-devices` off, launch-app on a physical UDID still launched the app. Extract the gate into a shared assertPhysicalIosEnabled() (reused by the CoreDevice factory and tunnel start) and call it from the launch-app physical-iOS branch before shelling devicectl. Add coverage: the launch-app flag gate, assertPhysicalIosEnabled, and the gesture-swipe physical-iOS routing (coords + duration forwarding). --- .../tool-server/src/blueprints/core-device.ts | 27 +++++--- .../src/tools/launch-app/platforms/ios.ts | 6 ++ .../test/physical-ios-followups.test.ts | 66 ++++++++++++++++++- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts index 1a58c9618..b712fc249 100644 --- a/packages/tool-server/src/blueprints/core-device.ts +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -24,6 +24,21 @@ export const CORE_DEVICE_NAMESPACE = "CoreDevice"; // start must not be reachable unless the user enabled the experimental feature. const PHYSICAL_IOS_FLAG = "physical-ios-devices"; +/** + * Throw the standard "enable the flag" error unless physical-iOS support is on. + * The single gate for every physical-iOS operation — the CoreDevice factory and + * tunnel start funnel through it, and `launch-app` (which drives a real device + * via `devicectl` rather than the CoreDevice backend) calls it directly so it + * can't bypass the opt-in. + */ +export function assertPhysicalIosEnabled(): void { + if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { + throw new Error( + `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}` + ); + } +} + // The registry's `ServiceRef.options` is typed as `Record`; // the intersection adds the implicit string index signature an interface lacks. type CoreDeviceFactoryOptions = Record & { device: DeviceInfo }; @@ -258,11 +273,7 @@ async function isTunneldReachable(port: number): Promise { * locked), it waits rather than popping a second, pointless prompt. */ export async function ensureCoreDeviceTunnel(udid: string): Promise { - if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { - throw new Error( - `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}` - ); - } + assertPhysicalIosEnabled(); try { return await resolveTunnel(udid); } catch (notRunning) { @@ -397,11 +408,7 @@ export const coreDeviceBlueprint: ServiceBlueprint = // "enable the flag" message rather than an incidental "install pymobiledevice3" // (ensureCoreDeviceTunnel re-checks this; the duplicate keeps the message // correct regardless of whether pmd3 happens to be installed). - if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { - throw new Error( - `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}` - ); - } + assertPhysicalIosEnabled(); const pmd3 = resolvePmd3(); const udid = device.id; diff --git a/packages/tool-server/src/tools/launch-app/platforms/ios.ts b/packages/tool-server/src/tools/launch-app/platforms/ios.ts index 1905cb181..bd6ad5429 100644 --- a/packages/tool-server/src/tools/launch-app/platforms/ios.ts +++ b/packages/tool-server/src/tools/launch-app/platforms/ios.ts @@ -2,6 +2,7 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { FAILURE_CODES, FailureError, subprocessFailureMetadata } from "@argent/registry"; import { precheckNativeDevtools } from "../../../blueprints/native-devtools"; +import { assertPhysicalIosEnabled } from "../../../blueprints/core-device"; import type { PlatformImpl } from "../../../utils/cross-platform-tool"; import type { LaunchAppIosServices, LaunchAppParams, LaunchAppResult } from "../types"; @@ -14,6 +15,11 @@ export const iosImpl: PlatformImpl { }); }); +describe("launch-app enforces the physical-iOS flag (no bypass)", () => { + // launch-app drives a real device via `devicectl` directly (not the + // CoreDevice service), so unlike screenshot/tap/swipe it must enforce the + // opt-in itself — otherwise it would be the one physical-iOS operation + // reachable while the feature is disabled. + it("assertPhysicalIosEnabled throws when the flag is off, not when on", () => { + mockFlag.mockReturnValue(false); + expect(() => assertPhysicalIosEnabled()).toThrow(/Physical iOS support is disabled/); + mockFlag.mockReturnValue(true); + expect(() => assertPhysicalIosEnabled()).not.toThrow(); + }); + + it("launch-app rejects a physical iPhone when the flag is off (before shelling devicectl)", async () => { + mockFlag.mockReturnValue(false); + await expect( + launchAppIos.handler( + {} as never, + { udid: PHYSICAL_UDID, bundleId: "com.apple.Preferences" } as never, + resolveDevice(PHYSICAL_UDID) + ) + ).rejects.toThrow(/Physical iOS support is disabled.*argent enable physical-ios-devices/s); + }); +}); + +describe("gesture-swipe routes physical iOS to the CoreDevice backend", () => { + it("forwards normalized coords and duration to coreDevice.swipe", async () => { + const coreDevice = { swipe: vi.fn().mockResolvedValue(undefined) }; + const res = await gestureSwipeTool.execute( + { coreDevice } as never, + { + udid: PHYSICAL_UDID, + fromX: 0.5, + fromY: 0.7, + toX: 0.5, + toY: 0.3, + durationMs: 250, + } as never + ); + expect(coreDevice.swipe).toHaveBeenCalledWith(0.5, 0.7, 0.5, 0.3, 250); + expect(res).toMatchObject({ swiped: true }); + }); + + it("defaults the duration to 300ms when omitted", async () => { + const coreDevice = { swipe: vi.fn().mockResolvedValue(undefined) }; + await gestureSwipeTool.execute( + { coreDevice } as never, + { + udid: PHYSICAL_UDID, + fromX: 0.2, + fromY: 0.2, + toX: 0.8, + toY: 0.8, + } as never + ); + expect(coreDevice.swipe).toHaveBeenCalledWith(0.2, 0.2, 0.8, 0.8, 300); + }); +}); + describe("tools unsupported on physical iOS reject with UnsupportedOperationError (400)", () => { const device = resolveDevice(PHYSICAL_UDID); From 13de304f6bc62e5444b7f34b82f00de01eab35a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Fri, 26 Jun 2026 15:01:31 +0200 Subject: [PATCH 07/16] docs(physical-ios): list describe + native inspection/profiling tools as unsupported The README limitations section listed only the interaction/lifecycle tools that reject on physical iOS; align it with the actual behavior by also noting that describe (accessibility), the native-* inspection tools, native-profiler-*, and screenshot-diff are simulator-only. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d652e4bd6..3b0dc5508 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,10 @@ and leave it running: `sudo pymobiledevice3 remote tunneld`. **Limitations / notes** -- Not supported on physical iOS yet: keyboard/typing, pinch & rotate (multi-touch), - `open-url`, `reinstall-app`, and `restart-app` — these return a clear "not supported" error. +- Not supported on physical iOS yet: `describe` / accessibility inspection (use `screenshot` + instead), keyboard/typing, pinch & rotate (multi-touch), `open-url`, `reinstall-app`, + `restart-app`, and the native inspection / profiling tools (`native-*`, `native-profiler-*`, + `screenshot-diff`) — all return a clear "not supported" error. - Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT` (defaults to `49151`). From b73616b2c0c89b7c693bd029306c823b046f8a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Tue, 30 Jun 2026 15:40:19 +0200 Subject: [PATCH 08/16] fix(physical-ios): translate CoreDeviceError 9021 into an iOS-27 hint; cover pmd3 argv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On iOS 18-26 Apple gates host-driven input (the CoreDevice "remote control" services behind tap/swipe/button) to iOS 27+, so the device is discovered and advertised as tap-capable, then fails mid-session with a raw `CoreDeviceError 9021`. Detect that specific failure in `conciseError` (matched against pymobiledevice3's stderr/stdout only, never the argv-echoing execFile message, so a HID coordinate that contains "9021" can't false-positive) and surface an actionable message that points at iOS 27 and notes screenshots still work. `screen-capture` is not gated, so the screenshot path is untouched. Also note the physical-iOS/CoreDevice routing in the gesture-tap and gesture-swipe descriptions, and add a focused test that drives the CoreDevice factory through a mocked execFile boundary to pin the exact pmd3 argv for tap, button, screenshot, and swipe (previously untested — every other physical-iOS test mocks the CoreDeviceApi wholesale) plus the 9021 translation. --- .../tool-server/src/blueprints/core-device.ts | 27 +++ .../src/tools/gesture-swipe/index.ts | 2 +- .../src/tools/gesture-tap/index.ts | 2 +- .../test/physical-ios-coredevice-argv.test.ts | 216 ++++++++++++++++++ 4 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 packages/tool-server/test/physical-ios-coredevice-argv.test.ts diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts index b712fc249..f0ba9c94a 100644 --- a/packages/tool-server/src/blueprints/core-device.ts +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -323,8 +323,35 @@ export async function ensureCoreDeviceTunnel(udid: string): Promise { } } +/** + * Apple gates host-driven input (the CoreDevice "remote control" services that + * back tap/swipe/button) to iOS 27+. On iOS 18-26 those services exist but the + * device rejects the command with `CoreDeviceError 9021`, which `conciseError` + * would otherwise surface as a raw, unactionable line. `screen-capture` is NOT + * gated, so screenshots keep working and never hit this path. + * + * Detected against pymobiledevice3's own output (stderr/stdout) only — never the + * execFile `message`, which echoes the argv where a HID coordinate (0..65535) + * could itself be `9021` and false-positive a bare numeric match. + */ +function isRemoteControlGatedError(err: unknown): boolean { + const e = err as { stderr?: string; stdout?: string }; + const output = [e?.stderr, e?.stdout] + .filter((v): v is string => typeof v === "string") + .join("\n"); + return /core\s*device\s*error\W*9021/i.test(output) || /\b9021\b/.test(output); +} + /** Extract a concise, human-readable line from a (possibly huge/binary) pmd3 failure. */ function conciseError(label: string, err: unknown): Error { + if (isRemoteControlGatedError(err)) { + return new Error( + `CoreDevice ${label} failed: this iPhone is on an iOS below 27; host-driven input ` + + `(tap/swipe/button) requires iOS 27+. Only screenshot is supported on earlier iOS ` + + `versions (Apple CoreDeviceError 9021).`, + { cause: err } + ); + } const e = err as { stderr?: string; stdout?: string; message?: string }; const blob = [e?.stderr, e?.stdout, e?.message] .filter((v): v is string => typeof v === "string") diff --git a/packages/tool-server/src/tools/gesture-swipe/index.ts b/packages/tool-server/src/tools/gesture-swipe/index.ts index bf80582eb..5e5dce6aa 100644 --- a/packages/tool-server/src/tools/gesture-swipe/index.ts +++ b/packages/tool-server/src/tools/gesture-swipe/index.ts @@ -40,7 +40,7 @@ export const gestureSwipeTool: ToolDefinition = { description: `Execute a smooth swipe / drag touch gesture between two points on the device (iOS simulator or Android emulator). All from/to positions are normalized 0.0–1.0 (fractions of screen width/height, not pixels), same as gesture-tap. Generates interpolated Move events for a natural feel (~60fps). Swipe up (fromY > toY) to scroll content down. -Use when you need to scroll a list, dismiss a modal, drag an element, or navigate between pages. Not supported on Chromium — use gesture-scroll there instead. Returns { swiped: true, timestampMs }. Fails if the simulator-server / emulator backend is not reachable for the given device.`, +Use when you need to scroll a list, dismiss a modal, drag an element, or navigate between pages. Not supported on Chromium — use gesture-scroll there instead. Returns { swiped: true, timestampMs }. On a physical iPhone, swipes route over CoreDevice. Fails if the simulator-server / emulator backend is not reachable for the given device.`, alwaysLoad: true, searchHint: "swipe scroll drag pan gesture device simulator emulator touch move", zodSchema, diff --git a/packages/tool-server/src/tools/gesture-tap/index.ts b/packages/tool-server/src/tools/gesture-tap/index.ts index c33e84864..506820694 100644 --- a/packages/tool-server/src/tools/gesture-tap/index.ts +++ b/packages/tool-server/src/tools/gesture-tap/index.ts @@ -44,7 +44,7 @@ export const gestureTapTool: ToolDefinition = { description: `Press the device screen (iOS simulator, Android emulator, or Chromium app) at normalized coordinates: x and y are fractions of screen width and height in 0.0–1.0 (not pixels). Sends a Down event followed by an Up event at the same point. For Chromium, this dispatches a CDP mouse-press/release on the renderer. Use when you need to tap a button, link, or any tappable element on the screen. -Returns { tapped: true, timestampMs }. Fails if the simulator-server / emulator backend / Chromium CDP is not reachable for the given device. +Returns { tapped: true, timestampMs }. On a physical iPhone, taps route over CoreDevice. Fails if the simulator-server / emulator backend / Chromium CDP is not reachable for the given device. Before tapping, determine the correct coordinates by using discovery tools — pick by platform: iOS / Android use \`describe\`, \`native-describe-screen\`, or \`debugger-component-tree\`; Chromium uses \`describe\` (the DOM walker), since the native and RN-specific discovery tools don't apply. More information in \`argent-device-interact\` skill`, alwaysLoad: true, searchHint: "tap press button element device simulator emulator chromium touch down up click", diff --git a/packages/tool-server/test/physical-ios-coredevice-argv.test.ts b/packages/tool-server/test/physical-ios-coredevice-argv.test.ts new file mode 100644 index 000000000..0986e01c5 --- /dev/null +++ b/packages/tool-server/test/physical-ios-coredevice-argv.test.ts @@ -0,0 +1,216 @@ +/** + * Coverage for the actual pymobiledevice3 command line the CoreDevice backend + * shells out, plus the iOS-27 host-input gate translation. The rest of the + * physical-iOS suite mocks the CoreDeviceApi wholesale, so the real argv strings + * (`developer core-device universal-hid-service drag …`, `… hid button … press`, + * `… screen-capture screenshot …`) and the `conciseError` 9021 mapping were + * untested — a pmd3 CLI rename or a typo in those arrays would ship silently. + * + * These tests drive `coreDeviceBlueprint.factory` end to end, mocking only the + * `node:child_process` execFile boundary (so the genuine argv is asserted) and + * the tunneld HTTP lookup (so no real device/tunnel is needed). + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { DeviceInfo } from "@argent/registry"; + +// Hoisted so the vi.mock factory below can close over the same mock instance. +const { execFileMock } = vi.hoisted(() => ({ execFileMock: vi.fn() })); + +// The flag gate is exercised elsewhere; here it's always on so the factory runs. +vi.mock("@argent/configuration-core", () => ({ isFlagEnabled: () => true })); + +// Override only execFile; keep the rest of child_process real so unrelated +// imports in the graph are unaffected. `promisify(execFile)` (run at module +// load) then wraps this mock with the default callback convention. +vi.mock("node:child_process", async (importActual) => { + const actual = await importActual(); + return { ...actual, execFile: execFileMock }; +}); + +import { coreDeviceBlueprint } from "../src/blueprints/core-device"; + +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; +const TUNNEL = { address: "fd11:2233::1", port: 54321 }; +const device: DeviceInfo = { id: PHYSICAL_UDID, platform: "ios", kind: "device" }; + +type CommandResult = { stdout?: string; stderr?: string } | { error: unknown }; + +/** + * Build the CoreDevice API with the factory, recording every real interaction + * command's argv. The setup probes (`version`, `mounter auto-mount`) are + * answered with success and not recorded; the per-interaction command (anything + * starting with `developer`) is recorded and answered by `onCommand`. + */ +async function makeApi(onCommand: (file: string, args: string[]) => CommandResult) { + const calls: Array<{ file: string; args: string[] }> = []; + execFileMock.mockImplementation((...all: unknown[]) => { + const cb = all[all.length - 1] as (err: unknown, res?: unknown) => void; + const file = all[0] as string; + const args = all[1] as string[]; + const sub = args[0]; + if (sub === "version" || sub === "mounter") { + cb(null, { stdout: "", stderr: "" }); + return; + } + calls.push({ file, args }); + const r = onCommand(file, args); + if ("error" in r) cb(r.error); + else cb(null, { stdout: r.stdout ?? "", stderr: r.stderr ?? "" }); + }); + const instance = await coreDeviceBlueprint.factory({}, "ignored" as never, { device }); + return { api: instance.api, calls }; +} + +beforeEach(() => { + execFileMock.mockReset(); + // tunneld REST lookup: report this device's RSD endpoint so resolveTunnel + // succeeds without a real tunnel. + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + json: async () => ({ + [PHYSICAL_UDID]: [{ "tunnel-address": TUNNEL.address, "tunnel-port": TUNNEL.port }], + }), + })) + ); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("CoreDevice pmd3 argv (the exact command line, untested elsewhere)", () => { + const rsd = ["--rsd", TUNNEL.address, String(TUNNEL.port)]; + + it("tap → `universal-hid-service drag` with a short held dwell move + RSD endpoint", async () => { + const { api, calls } = await makeApi(() => ({ stdout: "" })); + await api.tap(0.5, 0.5); + expect(calls).toHaveLength(1); + // 0.5 → 32768; the dwell move nudges y by +96 (away from the edge) so a + // zero-dwell contact isn't dropped by iOS. + expect(calls[0]!.args).toEqual([ + "developer", + "core-device", + "universal-hid-service", + "drag", + "32768", + "32768", + "32768", + "32864", + "--steps", + "3", + "--duration", + "0.15", + ...rsd, + ]); + }); + + it("button → `hid button press` with RSD endpoint", async () => { + const { api, calls } = await makeApi(() => ({ stdout: "" })); + await api.button("home"); + expect(calls[0]!.args).toEqual([ + "developer", + "core-device", + "hid", + "button", + "home", + "press", + ...rsd, + ]); + }); + + it("screenshot → `screen-capture screenshot ` with RSD endpoint", async () => { + const { api, calls } = await makeApi(() => ({ stdout: "" })); + const { path } = await api.screenshot(); + const args = calls[0]!.args; + expect(args.slice(0, 4)).toEqual(["developer", "core-device", "screen-capture", "screenshot"]); + expect(args[4]).toMatch(/argent-ios-shot-.*\.png$/); + expect(args.slice(5)).toEqual(rsd); + // The capture path returned to the caller is the one passed to pmd3. + expect(path).toBe(args[4]); + }); + + it("swipe → `universal-hid-service drag` from→to with clamped step count + RSD endpoint", async () => { + const { api, calls } = await makeApi(() => ({ stdout: "" })); + await api.swipe(0.2, 0.8, 0.6, 0.4, 300); + expect(calls[0]!.args).toEqual([ + "developer", + "core-device", + "universal-hid-service", + "drag", + "13107", // toHid(0.2) + "52428", // toHid(0.8) + "39321", // toHid(0.6) + "26214", // toHid(0.4) + "--steps", + "19", // round(300/16), capped at 60 + "--duration", + "0.300", + ...rsd, + ]); + }); +}); + +describe("iOS-27 host-input gate (CoreDeviceError 9021) is translated, screenshot unaffected", () => { + const gated = Object.assign(new Error("Command failed: pymobiledevice3 … 9021"), { + stderr: "CoreDeviceError 9021: The operation could not be completed.", + stdout: "", + code: 1, + }); + + it("tap surfaces an actionable iOS-27 message instead of the raw 9021 line", async () => { + const { api } = await makeApi(() => ({ error: gated })); + const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; + expect(e.message).toMatch(/requires iOS 27\+/); + expect(e.message).toMatch(/Only screenshot is supported/i); + expect(e.message).toContain("CoreDeviceError 9021"); + // Names the attempted interaction so the user knows what was gated. + expect(e.message).toContain("tap"); + // The raw pmd3 error is preserved as the cause for diagnostics. + expect((e as Error & { cause?: unknown }).cause).toBe(gated); + }); + + it("button is gated identically", async () => { + const { api } = await makeApi(() => ({ error: gated })); + const e = (await api.button("home").catch((x: unknown) => x)) as Error; + expect(e.message).toMatch(/requires iOS 27\+/); + }); + + it("screenshot still works on the same pre-iOS-27 device (screen-capture is not gated)", async () => { + // Same device: host input (hid) is rejected with 9021, but screen-capture + // succeeds — exactly the documented pre-27 behavior. + const { api } = await makeApi((_file, args) => + args[2] === "screen-capture" ? { stdout: "" } : { error: gated } + ); + await expect(api.tap(0.5, 0.5)).rejects.toThrow(/requires iOS 27\+/); + const { path } = await api.screenshot(); + expect(path).toMatch(/argent-ios-shot-.*\.png$/); + }); + + it("does NOT mistranslate a non-9021 failure (the mapping is 9021-specific)", async () => { + const refused = Object.assign(new Error("boom"), { + stderr: "Connection refused by the device", + stdout: "", + code: 1, + }); + const { api } = await makeApi(() => ({ error: refused })); + const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; + expect(e.message).toContain("CoreDevice tap failed:"); + expect(e.message).toContain("Connection refused"); + expect(e.message).not.toMatch(/iOS 27/); + }); + + it("does NOT mistranslate a HID coordinate that merely contains 9021 in the echoed argv", async () => { + // execFile's error `message` echoes the argv; a coordinate like 29021 must + // not trip the gate detection (which only inspects stderr/stdout). + const coordErr = Object.assign( + new Error("Command failed: pymobiledevice3 developer core-device … 29021"), + { stderr: "the device handshake failed", stdout: "", code: 1 } + ); + const { api } = await makeApi(() => ({ error: coordErr })); + const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; + expect(e.message).not.toMatch(/iOS 27/); + expect(e.message).toContain("handshake failed"); + }); +}); From 3975f703ce681035ee81ee68a72538538d21da82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Tue, 30 Jun 2026 15:57:24 +0200 Subject: [PATCH 09/16] test(physical-ios): make the 9021-in-argv guard bite; note CoreDevice on button --- packages/tool-server/src/tools/button/index.ts | 1 + .../test/physical-ios-coredevice-argv.test.ts | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/tool-server/src/tools/button/index.ts b/packages/tool-server/src/tools/button/index.ts index ee9fa5374..864c41d7b 100644 --- a/packages/tool-server/src/tools/button/index.ts +++ b/packages/tool-server/src/tools/button/index.ts @@ -65,6 +65,7 @@ export const buttonTool: ToolDefinition = { Supported buttons depend on the platform: home, back, power, volumeUp, volumeDown, appSwitch, actionButton — buttons not present on the target platform (e.g. 'back' on iOS, 'actionButton' on Android) are rejected with a clear error. Use when you need to trigger hardware button events. Returns { pressed: buttonName }. +On a physical iPhone, button presses route over CoreDevice (home, power, volumeUp, volumeDown). Fails if the simulator-server / emulator backend is not reachable for the given device.`, zodSchema, capability, diff --git a/packages/tool-server/test/physical-ios-coredevice-argv.test.ts b/packages/tool-server/test/physical-ios-coredevice-argv.test.ts index 0986e01c5..e7522f844 100644 --- a/packages/tool-server/test/physical-ios-coredevice-argv.test.ts +++ b/packages/tool-server/test/physical-ios-coredevice-argv.test.ts @@ -201,11 +201,18 @@ describe("iOS-27 host-input gate (CoreDeviceError 9021) is translated, screensho expect(e.message).not.toMatch(/iOS 27/); }); - it("does NOT mistranslate a HID coordinate that merely contains 9021 in the echoed argv", async () => { - // execFile's error `message` echoes the argv; a coordinate like 29021 must - // not trip the gate detection (which only inspects stderr/stdout). + it("does NOT mistranslate a standalone 9021 HID coordinate echoed in the argv message", async () => { + // execFile's error `message` echoes the full argv, and a HID coordinate maps + // [0,1]→[0,65535], so a real tap can produce the *standalone* token 9021 + // (toHid(0.13765) === 9021). The gate detector must inspect pmd3's stderr/stdout + // only — never `message` — or this non-gated handshake failure would be + // mislabeled "upgrade to iOS 27". The token here is a bare `9021` (not 29021), + // so this test genuinely fails if the detector ever starts scanning `message`. const coordErr = Object.assign( - new Error("Command failed: pymobiledevice3 developer core-device … 29021"), + new Error( + "Command failed: pymobiledevice3 developer core-device universal-hid-service " + + "drag --rsd fd00::1 50000 --duration 0.05 9021 32768 9021 40000" + ), { stderr: "the device handshake failed", stdout: "", code: 1 } ); const { api } = await makeApi(() => ({ error: coordErr })); From 14329d5d0b873e8bed26d3a1c286e90ae29f7d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Wed, 1 Jul 2026 18:49:15 +0200 Subject: [PATCH 10/16] fix(ios): don't eagerly resolve CoreDevice for a button with no HID equivalent button's services() resolved coreDeviceRef() unconditionally for any physical-iOS button press, before execute() validates whether the button is even supported. Since the registry resolves services() before execute(), pressing appSwitch/actionButton on a physical iPhone triggered CoreDevice's tunnel-setup side effects (possibly a macOS admin prompt) just to reject the button afterward. Skip resolving the service when the button has no CoreDevice HID mapping. --- packages/tool-server/src/tools/button/index.ts | 5 +++++ .../test/physical-ios-followups.test.ts | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/tool-server/src/tools/button/index.ts b/packages/tool-server/src/tools/button/index.ts index 864c41d7b..dba77cf61 100644 --- a/packages/tool-server/src/tools/button/index.ts +++ b/packages/tool-server/src/tools/button/index.ts @@ -72,6 +72,11 @@ Fails if the simulator-server / emulator backend is not reachable for the given services: (params): Record => { const device = resolveDevice(params.udid); if (isPhysicalIos(device)) { + // A button with no CoreDevice HID equivalent (appSwitch/actionButton) is + // always rejected by execute() below, before it ever touches + // services.coreDevice — don't pay for resolving the CoreDevice service + // (tunnel setup, possibly a macOS admin prompt) just to reject anyway. + if (!COREDEVICE_BUTTON[params.button]) return {}; return { coreDevice: coreDeviceRef(device) }; } return { simulatorServer: simulatorServerRef(device) }; diff --git a/packages/tool-server/test/physical-ios-followups.test.ts b/packages/tool-server/test/physical-ios-followups.test.ts index 2a5a58ef5..3e6aca9df 100644 --- a/packages/tool-server/test/physical-ios-followups.test.ts +++ b/packages/tool-server/test/physical-ios-followups.test.ts @@ -138,6 +138,21 @@ describe("button — CoreDevice HID mapping on physical iOS", () => { await expect(press("appSwitch")).rejects.toBeInstanceOf(UnsupportedOperationError); await expect(press("actionButton")).rejects.toBeInstanceOf(UnsupportedOperationError); }); + + it("does not resolve the CoreDevice service for a button with no HID equivalent", () => { + // services() runs before execute() (the registry resolves services first), + // so eagerly resolving coreDevice here would pay for tunnel setup — + // possibly a macOS admin prompt — just to reject the button afterward. + for (const button of ["appSwitch", "actionButton"]) { + const services = buttonTool.services!({ udid: PHYSICAL_UDID, button } as never); + expect(services.coreDevice).toBeUndefined(); + } + }); + + it("still resolves the CoreDevice service for a supported button", () => { + const services = buttonTool.services!({ udid: PHYSICAL_UDID, button: "home" } as never); + expect(services.coreDevice).toBeDefined(); + }); }); describe("privileged tunnel start is gated behind the feature flag", () => { From 381f7514117173a66a065117c822cb80faa52394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Wed, 1 Jul 2026 18:49:27 +0200 Subject: [PATCH 11/16] fix(ios): use FailureError/FAILURE_CODES for CoreDevice errors, fix message accuracy core-device.ts threw plain Error at all 12 sites, and ax-service.ts / simulator-server.ts's own new physical-iOS rejection branches did too - unlike every sibling blueprint, which uses FailureError with a dedicated error_code. Every physical-iOS failure (auth declined, no tunnel, missing pymobiledevice3, iOS<27 gate, flag disabled) was landing in telemetry as generic REGISTRY_TOOL_FAILURE_UNCLASSIFIED, indistinguishable from any other crash. Add CORE_DEVICE_* / AX_PHYSICAL_DEVICE_UNSUPPORTED / SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED failure codes and convert every throw site. Also, three smaller correctness fixes found in the same pass: - resolveTunnel's `tunnel-port == null` check let a port of 0 through as "present", inconsistent with the sibling tunnel-address check which treats any falsy value as missing. - The iOS<27 gate message claimed "tap/swipe/button" all require iOS 27+, but this is hardware-verified false: screenshot and hardware buttons work below iOS 27, only touch (tap/swipe) is actually gated. - conciseError's line-matching heuristic could pick a misleading rich-boxed Python source-context line (e.g. "if rsd is not None:") instead of the real exception (e.g. "OSError: No route to host") from a stale-tunnel traceback, and searched execFile's synthesized "Command failed: " message with equal priority to the real stderr/stdout output. --- packages/registry/src/errors.ts | 1 + packages/registry/src/failure-codes.ts | 14 ++ .../tool-server/src/blueprints/ax-service.ts | 10 +- .../tool-server/src/blueprints/core-device.ts | 168 ++++++++++++++---- .../src/blueprints/simulator-server.ts | 10 +- .../test/physical-ios-coredevice-argv.test.ts | 30 +++- 6 files changed, 198 insertions(+), 35 deletions(-) diff --git a/packages/registry/src/errors.ts b/packages/registry/src/errors.ts index b5d982a26..cc628f76e 100644 --- a/packages/registry/src/errors.ts +++ b/packages/registry/src/errors.ts @@ -32,6 +32,7 @@ export const FAILURE_COMMANDS = [ "android_devtools", "ax_service", "simulator_server", + "pymobiledevice3", "cdp", "npm", "npx", diff --git a/packages/registry/src/failure-codes.ts b/packages/registry/src/failure-codes.ts index acb6c04b7..daae0b515 100644 --- a/packages/registry/src/failure-codes.ts +++ b/packages/registry/src/failure-codes.ts @@ -66,6 +66,7 @@ export const FAILURE_CODES = { SIMULATOR_SERVER_READY_TIMEOUT: "SIMULATOR_SERVER_READY_TIMEOUT", SIMULATOR_SERVER_PROCESS_ERROR: "SIMULATOR_SERVER_PROCESS_ERROR", SIMULATOR_SERVER_TERMINATED: "SIMULATOR_SERVER_TERMINATED", + SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED: "SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED", AX_QUERY_TIMEOUT: "AX_QUERY_TIMEOUT", AX_DAEMON_READY_TIMEOUT: "AX_DAEMON_READY_TIMEOUT", @@ -73,6 +74,7 @@ export const FAILURE_CODES = { AX_DAEMON_PROCESS_ERROR: "AX_DAEMON_PROCESS_ERROR", AX_FACTORY_OPTIONS_MISSING: "AX_FACTORY_OPTIONS_MISSING", AX_WRONG_PLATFORM: "AX_WRONG_PLATFORM", + AX_PHYSICAL_DEVICE_UNSUPPORTED: "AX_PHYSICAL_DEVICE_UNSUPPORTED", AX_DEVICE_ID_INVALID: "AX_DEVICE_ID_INVALID", AX_DESCRIBE_ERROR: "AX_DESCRIBE_ERROR", AX_QUERY_FAILED: "AX_QUERY_FAILED", @@ -87,6 +89,18 @@ export const FAILURE_CODES = { IOS_OPEN_URL_FAILED: "IOS_OPEN_URL_FAILED", IOS_REINSTALL_INSTALL_FAILED: "IOS_REINSTALL_INSTALL_FAILED", IOS_RESTART_LAUNCH_FAILED: "IOS_RESTART_LAUNCH_FAILED", + + CORE_DEVICE_FLAG_DISABLED: "CORE_DEVICE_FLAG_DISABLED", + CORE_DEVICE_FACTORY_OPTIONS_MISSING: "CORE_DEVICE_FACTORY_OPTIONS_MISSING", + CORE_DEVICE_WRONG_DEVICE: "CORE_DEVICE_WRONG_DEVICE", + CORE_DEVICE_PMD3_NOT_FOUND: "CORE_DEVICE_PMD3_NOT_FOUND", + CORE_DEVICE_TUNNEL_UNREACHABLE: "CORE_DEVICE_TUNNEL_UNREACHABLE", + CORE_DEVICE_TUNNEL_NOT_REGISTERED: "CORE_DEVICE_TUNNEL_NOT_REGISTERED", + CORE_DEVICE_TUNNEL_AUTH_DECLINED: "CORE_DEVICE_TUNNEL_AUTH_DECLINED", + CORE_DEVICE_TUNNEL_TIMEOUT: "CORE_DEVICE_TUNNEL_TIMEOUT", + CORE_DEVICE_IOS_VERSION_TOO_OLD: "CORE_DEVICE_IOS_VERSION_TOO_OLD", + CORE_DEVICE_COMMAND_FAILED: "CORE_DEVICE_COMMAND_FAILED", + NATIVE_DEVTOOLS_DESCRIBE_ERROR: "NATIVE_DEVTOOLS_DESCRIBE_ERROR", NATIVE_DEVTOOLS_VIEW_AT_POINT_ERROR: "NATIVE_DEVTOOLS_VIEW_AT_POINT_ERROR", NATIVE_DEVTOOLS_USER_INTERACTABLE_VIEW_AT_POINT_ERROR: diff --git a/packages/tool-server/src/blueprints/ax-service.ts b/packages/tool-server/src/blueprints/ax-service.ts index 759588722..9b989e2c7 100644 --- a/packages/tool-server/src/blueprints/ax-service.ts +++ b/packages/tool-server/src/blueprints/ax-service.ts @@ -354,8 +354,14 @@ export const axServiceBlueprint: ServiceBlueprint = { // ax-service uses `xcrun simctl spawn`, which only works on simulators. // Physical iPhones are driven over CoreDevice and have no accessibility/ // describe path yet. - throw new Error( - `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; describe/accessibility is not supported on physical iOS yet.` + throw new FailureError( + `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; describe/accessibility is not supported on physical iOS yet.`, + { + error_code: FAILURE_CODES.AX_PHYSICAL_DEVICE_UNSUPPORTED, + failure_stage: "ax_service_factory_platform", + failure_area: "tool_server", + error_kind: "unsupported", + } ); } // Reject before spawning. An undefined `device.id` slips through when an diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts index f0ba9c94a..80c1fd6b9 100644 --- a/packages/tool-server/src/blueprints/core-device.ts +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -6,6 +6,8 @@ import { join } from "node:path"; import { randomUUID } from "node:crypto"; import { TypedEventEmitter, + FAILURE_CODES, + FailureError, type DeviceInfo, type ServiceBlueprint, type ServiceInstance, @@ -33,8 +35,14 @@ const PHYSICAL_IOS_FLAG = "physical-ios-devices"; */ export function assertPhysicalIosEnabled(): void { if (!isFlagEnabled(PHYSICAL_IOS_FLAG)) { - throw new Error( - `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}` + throw new FailureError( + `Physical iOS support is disabled. Enable it with: argent enable ${PHYSICAL_IOS_FLAG}`, + { + error_code: FAILURE_CODES.CORE_DEVICE_FLAG_DISABLED, + failure_stage: "core_device_flag_gate", + failure_area: "tool_server", + error_kind: "unsupported", + } ); } } @@ -110,10 +118,17 @@ async function verifyPmd3Available(pmd3: string): Promise { await execFileAsync(pmd3, ["version"], { timeout: 10_000 }); } catch (err) { if ((err as { code?: string })?.code === "ENOENT") { - throw new Error( + throw new FailureError( `pymobiledevice3 was not found (tried "${pmd3}"). Physical iOS control needs it — ` + `install it (e.g. \`pipx install pymobiledevice3\`) or set ARGENT_PYMOBILEDEVICE3 to its path.`, - { cause: err } + { + error_code: FAILURE_CODES.CORE_DEVICE_PMD3_NOT_FOUND, + failure_stage: "core_device_verify_pmd3", + failure_area: "tool_server", + error_kind: "dependency_missing", + failure_command: "pymobiledevice3", + }, + { cause: err instanceof Error ? err : new Error(String(err)) } ); } // Non-ENOENT (odd `version` failure): don't block; a real command will report. @@ -132,9 +147,16 @@ async function resolvePmd3Absolute(): Promise { } catch { // fall through to the install hint } - throw new Error( + throw new FailureError( `pymobiledevice3 was not found on PATH. Install it (e.g. \`pipx install pymobiledevice3\`) ` + - `or set ARGENT_PYMOBILEDEVICE3 to its absolute path.` + `or set ARGENT_PYMOBILEDEVICE3 to its absolute path.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_PMD3_NOT_FOUND, + failure_stage: "core_device_resolve_pmd3_absolute", + failure_area: "tool_server", + error_kind: "dependency_missing", + failure_command: "pymobiledevice3", + } ); } @@ -240,12 +262,31 @@ async function resolveTunnel(udid: string): Promise { if (!res.ok) throw new Error(`tunneld responded HTTP ${res.status}`); payload = (await res.json()) as typeof payload; } catch (err) { - throw new Error(tunnelHelp(udid, "tunneld is not running"), { cause: err }); + throw new FailureError( + tunnelHelp(udid, "tunneld is not running"), + { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_UNREACHABLE, + failure_stage: "core_device_resolve_tunnel", + failure_area: "tool_server", + error_kind: "network", + failure_command: "pymobiledevice3", + }, + { cause: err instanceof Error ? err : new Error(String(err)) } + ); } const entry = payload?.[udid]; const t = Array.isArray(entry) ? entry[0] : undefined; - if (!t?.["tunnel-address"] || t["tunnel-port"] == null) { - throw new Error(tunnelHelp(udid, "no active tunnel is registered for it")); + // `!t["tunnel-port"]` (not `== null`) deliberately also rejects 0 — tunneld + // never assigns port 0 to a real tunnel, so treat it the same as missing, + // consistent with the tunnel-address check on the left. + if (!t?.["tunnel-address"] || !t["tunnel-port"]) { + throw new FailureError(tunnelHelp(udid, "no active tunnel is registered for it"), { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_NOT_REGISTERED, + failure_stage: "core_device_resolve_tunnel", + failure_area: "tool_server", + error_kind: "not_found", + failure_command: "pymobiledevice3", + }); } return { address: String(t["tunnel-address"]), port: Number(t["tunnel-port"]) }; } @@ -295,9 +336,20 @@ export async function ensureCoreDeviceTunnel(udid: string): Promise { await tunnelStartInFlight; } catch (escalationErr) { tunnelStartInFlight = null; // allow a later retry - throw new Error(tunnelHelp(udid, "the authorization prompt was cancelled or unavailable"), { - cause: escalationErr, - }); + throw new FailureError( + tunnelHelp(udid, "the authorization prompt was cancelled or unavailable"), + { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_AUTH_DECLINED, + failure_stage: "core_device_ensure_tunnel_escalation", + failure_area: "tool_server", + error_kind: "subprocess", + failure_command: "pymobiledevice3", + }, + { + cause: + escalationErr instanceof Error ? escalationErr : new Error(String(escalationErr)), + } + ); } tunnelStartInFlight = null; } @@ -311,14 +363,21 @@ export async function ensureCoreDeviceTunnel(udid: string): Promise { // keep polling } } - throw new Error( + throw new FailureError( tunnelHelp( udid, reachable ? "tunneld is running but the device tunnel did not form — is the iPhone unlocked and trusted?" : "the tunnel did not come up after starting tunneld" ), - { cause: notRunning } + { + error_code: FAILURE_CODES.CORE_DEVICE_TUNNEL_TIMEOUT, + failure_stage: "core_device_ensure_tunnel_poll", + failure_area: "tool_server", + error_kind: "timeout", + failure_command: "pymobiledevice3", + }, + { cause: notRunning instanceof Error ? notRunning : new Error(String(notRunning)) } ); } } @@ -343,27 +402,64 @@ function isRemoteControlGatedError(err: unknown): boolean { } /** Extract a concise, human-readable line from a (possibly huge/binary) pmd3 failure. */ -function conciseError(label: string, err: unknown): Error { +function conciseError(label: string, err: unknown): FailureError { + const cause = err instanceof Error ? err : new Error(String(err)); if (isRemoteControlGatedError(err)) { - return new Error( - `CoreDevice ${label} failed: this iPhone is on an iOS below 27; host-driven input ` + - `(tap/swipe/button) requires iOS 27+. Only screenshot is supported on earlier iOS ` + + // Hardware-verified (iPhone 15, iOS 18.7.8 vs 27.0): screenshot and hardware + // buttons (`hid button`) work below iOS 27 — only touch (tap/swipe/drag) is + // actually gated. Don't overclaim "button" is affected too. + return new FailureError( + `CoreDevice ${label} failed: this iPhone is on an iOS below 27; host-driven touch input ` + + `(tap/swipe) requires iOS 27+. Screenshot and hardware buttons work on earlier iOS ` + `versions (Apple CoreDeviceError 9021).`, - { cause: err } + { + error_code: FAILURE_CODES.CORE_DEVICE_IOS_VERSION_TOO_OLD, + failure_stage: "core_device_command", + failure_area: "tool_server", + error_kind: "unsupported", + failure_command: "pymobiledevice3", + }, + { cause } ); } const e = err as { stderr?: string; stdout?: string; message?: string }; - const blob = [e?.stderr, e?.stdout, e?.message] + // stderr/stdout are pymobiledevice3's own output — the trustworthy source for + // the real cause. `message` (execFile's synthesized "Command failed: ") + // is deliberately searched LAST, as a fallback only: it's just the invoked + // command line, not a real diagnostic, and searching it with equal priority + // would let its ever-present "Command failed" text outrank a genuine + // stderr/stdout line lower in that output. + const outputLines = [e?.stderr, e?.stdout] .filter((v): v is string => typeof v === "string") - .join("\n"); + .join("\n") + .split("\n") + .map((s) => s.trim()) + // pymobiledevice3 renders failures via Python's `rich`, which frames source + // context in a box (each line starting with a `│`/`┃` rule) — never the + // actual exception. A keyword like "not "/"fail" can spuriously match a + // highlighted source line (e.g. "if rsd is not None:") before the real + // cause appears, so those are never eligible picks. + .filter((l) => l.length > 0 && !/^[│┃╭╮╰╯─━]/.test(l)); const line = - blob - .split("\n") - .map((s) => s.trim()) - .find((l) => /requires iOS|error|fail|not |unable|refused|timed out/i.test(l)) ?? + // Prefer a genuine `SomeError: message` / `SomeException: message` line — + // the actual raised exception, not just an incidentally keyword-matching + // one — and prefer the LAST such line, since that's where Python puts the + // real exception in a traceback (rich or plain). + outputLines.findLast((l) => /^[\w.]*(?:Error|Exception)\b.*:/.test(l)) ?? + outputLines.findLast((l) => /requires iOS|error|fail|not |unable|refused|timed out/i.test(l)) ?? e?.message ?? "unknown error"; - return new Error(`CoreDevice ${label} failed: ${line.slice(0, 240)}`, { cause: err }); + return new FailureError( + `CoreDevice ${label} failed: ${line.slice(0, 240)}`, + { + error_code: FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED, + failure_stage: "core_device_command", + failure_area: "tool_server", + error_kind: "subprocess", + failure_command: "pymobiledevice3", + }, + { cause } + ); } const COREDEVICE = ["developer", "core-device"]; @@ -421,14 +517,26 @@ export const coreDeviceBlueprint: ServiceBlueprint = const opts = options as unknown as CoreDeviceFactoryOptions | undefined; const device = opts?.device; if (!device?.id) { - throw new Error( + throw new FailureError( `${CORE_DEVICE_NAMESPACE}.factory requires a resolved DeviceInfo via options.device. ` + - `Use coreDeviceRef(device) when registering the service ref.` + `Use coreDeviceRef(device) when registering the service ref.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_FACTORY_OPTIONS_MISSING, + failure_stage: "core_device_factory_options", + failure_area: "tool_server", + error_kind: "validation", + } ); } if (device.platform !== "ios" || device.kind !== "device") { - throw new Error( - `${CORE_DEVICE_NAMESPACE} only drives physical iOS devices; got ${device.platform}/${device.kind}.` + throw new FailureError( + `${CORE_DEVICE_NAMESPACE} only drives physical iOS devices; got ${device.platform}/${device.kind}.`, + { + error_code: FAILURE_CODES.CORE_DEVICE_WRONG_DEVICE, + failure_stage: "core_device_factory_platform", + failure_area: "tool_server", + error_kind: "validation", + } ); } // Gate first — before any setup probe — so a flag-disabled user gets the diff --git a/packages/tool-server/src/blueprints/simulator-server.ts b/packages/tool-server/src/blueprints/simulator-server.ts index 5fcfd61ee..1500353ec 100644 --- a/packages/tool-server/src/blueprints/simulator-server.ts +++ b/packages/tool-server/src/blueprints/simulator-server.ts @@ -250,9 +250,15 @@ export const simulatorServerBlueprint: ServiceBlueprint ({ error: gated })); const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; expect(e.message).toMatch(/requires iOS 27\+/); - expect(e.message).toMatch(/Only screenshot is supported/i); + // Hardware-verified: screenshot AND buttons work below iOS 27 — only touch + // (tap/swipe) is actually gated. The message must not overclaim button too. + expect(e.message).toMatch(/Screenshot and hardware buttons work/i); expect(e.message).toContain("CoreDeviceError 9021"); // Names the attempted interaction so the user knows what was gated. expect(e.message).toContain("tap"); @@ -201,6 +203,32 @@ describe("iOS-27 host-input gate (CoreDeviceError 9021) is translated, screensho expect(e.message).not.toMatch(/iOS 27/); }); + it("picks the real exception, not a rich-boxed source line, from a stale-tunnel traceback", async () => { + // Real captured shape (a genuinely unreachable/stale RSD endpoint): pymobiledevice3 + // renders its traceback via Python's `rich`, framing SOURCE CODE context lines in a + // box — several of which coincidentally match the "fail"/"not " keyword heuristic + // (e.g. "if rsd is not None:") well before the actual OSError line at the bottom. + const staleTunnel = Object.assign(new Error("Command failed"), { + stderr: [ + "Traceback (most recent call last):", + "╭──────────────── Traceback (most recent call last) ────────────────╮", + "│ /pmd3/remote/core_device_tunnel_service.py:218 in connect │", + "│ │", + "│ 218 │ │ if rsd is not None: │", + "│ 219 │ │ │ raise ConnectionError('rsd is not available') │", + "╰────────────────────────────────────────────────────────────────────╯", + "OSError: [Errno 65] No route to host", + ].join("\n"), + stdout: "", + code: 1, + }); + const { api } = await makeApi(() => ({ error: staleTunnel })); + const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; + expect(e.message).toContain("CoreDevice tap failed:"); + expect(e.message).toContain("OSError: [Errno 65] No route to host"); + expect(e.message).not.toContain("if rsd is not None"); + }); + it("does NOT mistranslate a standalone 9021 HID coordinate echoed in the argv message", async () => { // execFile's error `message` echoes the full argv, and a HID coordinate maps // [0,1]→[0,65535], so a real tap can produce the *standalone* token 9021 From d404d5226bef4361faf3a3db68e1a7bf03398673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Wed, 1 Jul 2026 18:49:33 +0200 Subject: [PATCH 12/16] fix(ios): reject physical iOS cleanly in native-profiler-stop/analyze too native-profiler-start already rejects physical iOS (apple.device:false) since live capture uses simctl process enumeration that mislabels a real iPhone as a simulator. stop/analyze were left at device:true, so - since a session can never exist for physical iOS - they always fell through to a generic "no active session" error instead of the same clean, actionable rejection. Also fixes the README's "native-profiler-*" claim, which the stop/analyze gap made inaccurate, and two other accuracy issues found in the same pass: the iOS<27 limitation overstated to include hardware buttons, and a note that launch-app (devicectl) doesn't need the CoreDevice tunnel other tools require. --- README.md | 9 ++++++--- .../profiler/native-profiler/native-profiler-analyze.ts | 6 +++++- .../profiler/native-profiler/native-profiler-stop.ts | 6 +++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c2fbd471b..8ea46a524 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ up in `list-devices` with `kind: "device"`. **Requirements** -- **iOS 27 or later** — Apple gates host-driven input ("remote control") to iOS 27+; on - earlier versions the device reports `CoreDeviceError 9021` and only screenshots work. +- **iOS 27 or later for tap/swipe** — Apple gates host-driven touch input to iOS 27+; on + earlier versions those commands report `CoreDeviceError 9021`. Screenshot and hardware + buttons work on earlier iOS versions too. - macOS with Xcode, and `pymobiledevice3` installed (e.g. `pipx install pymobiledevice3`). - The iPhone connected, unlocked, trusted, with **Developer Mode** on. @@ -71,7 +72,9 @@ and leave it running: `sudo pymobiledevice3 remote tunneld`. - Not supported on physical iOS yet: `describe` / accessibility inspection (use `screenshot` instead), keyboard/typing, pinch & rotate (multi-touch), `open-url`, `reinstall-app`, `restart-app`, and the native inspection / profiling tools (`native-*`, `native-profiler-*`, - `screenshot-diff`) — all return a clear "not supported" error. + `screenshot-diff`) — all return a clear "not supported" error. `launch-app` (via `devicectl`) + works independently of the CoreDevice tunnel — it can succeed even before the tunnel setup + above has run. - Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT` (defaults to `49151`). diff --git a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts index 182fe1420..36b099993 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-analyze.ts @@ -18,8 +18,12 @@ const zodSchema = z.object({ .describe("Target device id from `list-devices` (iOS UDID or Android serial)."), }); +// A session can never exist for physical iOS (native-profiler-start rejects it, +// same apple.device:false reasoning) — reject here too for a clean, consistent +// error rather than the confusing "no active session" a physical UDID would +// otherwise always hit. const capability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, } as const; diff --git a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts index 6d7d87cdd..e3a76e948 100644 --- a/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts +++ b/packages/tool-server/src/tools/profiler/native-profiler/native-profiler-stop.ts @@ -50,8 +50,12 @@ interface AndroidStopArtifacts { type StopResult = IosStopArtifacts | AndroidStopArtifacts; +// A session can never exist for physical iOS (native-profiler-start rejects it, +// same apple.device:false reasoning) — reject here too for a clean, consistent +// error rather than the confusing "no active session" a physical UDID would +// otherwise always hit. const capability = { - apple: { simulator: true, device: true }, + apple: { simulator: true, device: false }, android: { emulator: true, device: true, unknown: true }, } as const; From 49db8444c13730c777ef7dd05a3e207c3afdf0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Thu, 2 Jul 2026 00:22:55 +0200 Subject: [PATCH 13/16] fix(ios): gate native-devtools xcrun per device kind so remote sims aren't blocked --- .../native-devtools/native-describe-screen.ts | 4 +- .../native-devtools/native-devtools-status.ts | 4 +- .../native-devtools/native-find-views.ts | 4 +- .../native-devtools/native-full-hierarchy.ts | 4 +- .../native-devtools/native-network-logs.ts | 4 +- .../native-user-interactable-view-at-point.ts | 4 +- .../native-devtools/native-view-at-point.ts | 4 +- ...ative-devtools-remote-sim-dep-gate.test.ts | 154 ++++++++++++++++++ 8 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts diff --git a/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts b/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts index a66c1555e..77c67b7c1 100644 --- a/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts +++ b/packages/tool-server/src/tools/native-devtools/native-describe-screen.ts @@ -41,7 +41,6 @@ type Result = export const nativeDescribeScreenTool: ToolDefinition = { id: "native-describe-screen", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Read the running app's native accessibility screen description via injected native devtools. @@ -63,6 +62,9 @@ If status is restart_required: call restart-app then retry.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts b/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts index c1a0780c0..b810d7113 100644 --- a/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts +++ b/packages/tool-server/src/tools/native-devtools/native-devtools-status.ts @@ -27,7 +27,6 @@ type Result = export const nativeDevtoolsStatusTool: ToolDefinition = { id: "native-devtools-status", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Check whether native devtools are connected to a specific app and whether the next launch is prepared for injection. Use when you need to verify native devtools readiness before calling native-full-hierarchy, native-describe-screen, or native-network-logs. @@ -49,6 +48,9 @@ Fails if the simulator server is not running for the given UDID or the bundleId }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-find-views.ts b/packages/tool-server/src/tools/native-devtools/native-find-views.ts index 41ea6542d..41ce7e8c4 100644 --- a/packages/tool-server/src/tools/native-devtools/native-find-views.ts +++ b/packages/tool-server/src/tools/native-devtools/native-find-views.ts @@ -44,7 +44,6 @@ type Result = export const nativeFindViewsTool: ToolDefinition = { id: "native-find-views", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Search for specific UIViews in the running app by class name, accessibility identifier, label, tag, or React Native nativeID. Use when you need to locate a specific view by its properties without dumping the entire hierarchy. @@ -57,6 +56,9 @@ Fails if native devtools are not connected, the app is not running, or status is }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts b/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts index 165c1b8df..c13550129 100644 --- a/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts +++ b/packages/tool-server/src/tools/native-devtools/native-full-hierarchy.ts @@ -58,7 +58,6 @@ type Result = export const nativeFullHierarchyTool: ToolDefinition = { id: "native-full-hierarchy", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Get the complete UIKit view tree for the running app. WARNING: Output can be extremely large (100KB–500KB+) for complex apps, especially those built with SwiftUI. Prefer native-find-views for targeted queries. @@ -72,6 +71,9 @@ Fails if native devtools are not connected or the app is not running.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-network-logs.ts b/packages/tool-server/src/tools/native-devtools/native-network-logs.ts index 99b81fd7c..420b214c2 100644 --- a/packages/tool-server/src/tools/native-devtools/native-network-logs.ts +++ b/packages/tool-server/src/tools/native-devtools/native-network-logs.ts @@ -29,7 +29,6 @@ type Result = export const nativeNetworkLogsTool: ToolDefinition = { id: "native-network-logs", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Retrieve network requests captured at the native NSURLProtocol level. Unlike the JS-level network inspector (view-network-logs), this captures ALL network traffic from the app including native modules, Swift/Objective-C networking, and background transfers that bypass JS fetch. @@ -42,6 +41,9 @@ Fails if native devtools are not connected or the app is not running.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts b/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts index 92ddb84eb..b886925e2 100644 --- a/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts +++ b/packages/tool-server/src/tools/native-devtools/native-user-interactable-view-at-point.ts @@ -63,7 +63,6 @@ type Result = export const nativeUserInteractableViewAtPointTool: ToolDefinition = { id: "native-user-interactable-view-at-point", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Inspect the deepest UIView at a raw native window point that would actually receive touch input. @@ -80,6 +79,9 @@ If status is restart_required: call restart-app then retry.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts b/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts index 6147bb314..2eb607494 100644 --- a/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts +++ b/packages/tool-server/src/tools/native-devtools/native-view-at-point.ts @@ -63,7 +63,6 @@ type Result = export const nativeViewAtPointTool: ToolDefinition = { id: "native-view-at-point", - requires: ["xcrun"], capability: { apple: { simulator: true, device: false }, appleRemote: { simulator: true } }, description: `Inspect the deepest visible UIView at a raw native window point. @@ -80,6 +79,9 @@ If status is restart_required: call restart-app then retry.`, }), async execute(services, params) { const device = resolveDevice(params.udid); + // Gate host deps per device kind: local sims need xcrun, remote sims route + // via sim-remote — a global `requires:["xcrun"]` would wrongly 424 a remote + // sim on an xcrun-less host, so the dep check lives here, not on the def. await ensureDeps(device.platform === "ios-remote" ? ["sim-remote"] : ["xcrun"]); const api = services.nativeDevtools as NativeDevtoolsApi; diff --git a/packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts b/packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts new file mode 100644 index 000000000..c240b683f --- /dev/null +++ b/packages/tool-server/test/native-devtools-remote-sim-dep-gate.test.ts @@ -0,0 +1,154 @@ +/** + * The native-devtools tools (native-describe-screen, native-find-views, …) + * support both local iOS simulators (driven via `xcrun simctl spawn`) and + * REMOTE iOS simulators (routed through `sim-remote`, no local xcrun needed) — + * their capability declares `appleRemote: { simulator: true }`. + * + * A regression once put a *global* `requires: ["xcrun"]` on these tools. The + * HTTP dispatcher probes `ToolDefinition.requires` unconditionally (before + * execution, regardless of the resolved device), so a remote sim on an + * xcrun-less host got a bogus 424 Failed Dependency even though it never needs + * xcrun. The correct gating is per device kind, done inside `execute` + * (`sim-remote` for remote sims, `xcrun` for local sims) — mirroring how + * `describe` declares its deps per branch. + * + * This pins that contract: remote sims are NOT xcrun-gated, local sims still + * are, and remote sims are gated on the RIGHT binary (sim-remote). + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import request from "supertest"; +import { Registry, TypedEventEmitter } from "@argent/registry"; + +const execFileMock = vi.fn(); +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { ...actual, execFile: (...args: unknown[]) => execFileMock(...args) }; +}); + +const resolveAndroidBinaryMock = vi.fn(); +vi.mock("../src/utils/android-binary", () => ({ + resolveAndroidBinary: (name: "adb" | "emulator") => resolveAndroidBinaryMock(name), + __resetAndroidBinaryCacheForTesting: () => {}, +})); + +import { createHttpApp } from "../src/http"; +import { __resetDepCacheForTests } from "../src/utils/check-deps"; +import { nativeDescribeScreenTool } from "../src/tools/native-devtools/native-describe-screen"; +import { nativeDevtoolsBlueprint, type NativeDevtoolsApi } from "../src/blueprints/native-devtools"; + +// Controls which host binaries the real dep-check treats as available: any dep +// in `missing` fails its `command -v` probe, everything else resolves. +function stubProbe(missing: readonly string[]): void { + execFileMock.mockImplementation( + ( + _cmd: string, + args: string[], + _opts: unknown, + cb: (err: Error | null, stdout?: string, stderr?: string) => void + ) => { + const script = args[1] ?? ""; + const dep = script.replace("command -v ", "").trim(); + if (missing.includes(dep)) cb(new Error(`not found: ${dep}`)); + else cb(null, `/usr/bin/${dep}\n`, ""); + } + ); + resolveAndroidBinaryMock.mockImplementation(async (name: string) => + missing.includes(name) ? null : `/usr/bin/${name}` + ); +} + +// Minimal fake NativeDevtools api so `execute` completes without any real +// simctl / sim-remote / socket I/O. `requiresAppRestart: true` makes the +// precheck return `restart_required`, a clean terminal result — the point is +// only that the request reaches execution instead of being 424'd at the gate. +function makeStubApi(): NativeDevtoolsApi { + return { + isEnvSetup: () => true, + socketPath: "/tmp/stub.sock", + ensureEnvReady: async () => {}, + reverifyEnv: async () => {}, + getInitFailure: () => null, + isConnected: () => false, + isAppRunning: async () => false, + listConnectedBundleIds: () => [], + requiresAppRestart: async () => true, + activateNetworkInspection: () => {}, + getNetworkLog: () => [], + clearNetworkLog: () => {}, + getAppState: async () => { + throw new Error("unused in this test"); + }, + detectFrontmostBundleId: async () => null, + queryViewHierarchy: async () => ({}), + }; +} + +function makeRegistry(): Registry { + const registry = new Registry(); + // Reuse the real blueprint's namespace/getURN but swap in a no-I/O factory so + // service resolution can't hang or shell out — the dep gating we're testing + // is entirely separate from the service factory. + registry.registerBlueprint({ + ...nativeDevtoolsBlueprint, + factory: async () => ({ + api: makeStubApi(), + dispose: async () => {}, + events: new TypedEventEmitter(), + }), + }); + registry.registerTool(nativeDescribeScreenTool); + return registry; +} + +const REMOTE_UDID = "remote:12345678-1234-1234-1234-123456789012"; +const LOCAL_UDID = "12345678-1234-1234-1234-123456789012"; +// Physical iPhone UDID shape (8-hex ECID, single dash, 16 hex). +const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; + +describe("native-devtools tools — per-device-kind dependency gating", () => { + beforeEach(() => { + __resetDepCacheForTests(); + execFileMock.mockReset(); + resolveAndroidBinaryMock.mockReset(); + }); + + it("does NOT block a REMOTE sim on missing xcrun (routes via sim-remote)", async () => { + stubProbe(["xcrun"]); // xcrun absent, sim-remote present + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: REMOTE_UDID, bundleId: "com.example.app" }); + // Reaches execution instead of the xcrun preflight 424. + expect(res.status).not.toBe(424); + expect(res.status).toBe(200); + }); + + it("still gates a LOCAL sim on xcrun (via the in-execute dep check)", async () => { + stubProbe(["xcrun"]); + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: LOCAL_UDID, bundleId: "com.example.app" }); + expect(res.status).toBe(424); + expect(res.body.missing).toEqual(["xcrun"]); + }); + + it("gates a REMOTE sim on the RIGHT binary — sim-remote, not xcrun", async () => { + stubProbe(["xcrun", "sim-remote"]); // both absent + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: REMOTE_UDID, bundleId: "com.example.app" }); + expect(res.status).toBe(424); + expect(res.body.missing).toEqual(["sim-remote"]); + }); + + it("rejects a PHYSICAL iOS device at the capability gate (path unaffected)", async () => { + stubProbe([]); // all deps present — must still be rejected before deps + const { app } = createHttpApp(makeRegistry()); + const res = await request(app) + .post("/tools/native-describe-screen") + .send({ udid: PHYSICAL_UDID, bundleId: "com.example.app" }); + expect(res.status).toBe(400); + }); +}); From 3ee0bc1980ec5c8818103a540e9345276f80ab51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Mon, 6 Jul 2026 20:06:11 +0200 Subject: [PATCH 14/16] docs(physical-ios): describe is Apple-gated, not "not yet" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app-free CoreDevice accessibility path was investigated on a physical iPhone (iOS 27). The on-device accessibility tree is served by CoreDevice's axAuditDaemon, but Apple gates it: the DTX service `com.apple.accessibility.axAuditDaemon.remoteserver.shim.remote` requires the `com.apple.mobile.lockdown.remote.trusted` entitlement — over the developer (untrusted) CoreDevice tunnel pymobiledevice3 forms, the daemon accepts the socket but terminates it on the first request (every audit selector, and even the standard DTX capability handshake). The RemoteXPC replacement `…axAuditDaemon.remoteAXService` requires `AppleInternal`, unreachable by any third party. DTX transport itself works over the same tunnel (`developer dvt proclist` succeeds), so this is Apple's auth wall, not a transport gap. Screenshot (`canViewDeviceDisplay`) remains the only app-free screen capture. Replace the imprecise describe rejection ("no on-device accessibility/describe path") and the ax-service backstop's "not supported yet" with accurate, evidence-based messages, and split describe out of the README "not yet" list. --- README.md | 18 ++++++---- .../tool-server/src/blueprints/ax-service.ts | 10 ++++-- .../src/tools/describe/platforms/ios/index.ts | 33 ++++++++++++++----- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 441cf7b01..1f26a6767 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,18 @@ and leave it running: `sudo pymobiledevice3 remote tunneld`. **Limitations / notes** -- Not supported on physical iOS yet: `describe` / accessibility inspection (use `screenshot` - instead), keyboard/typing, pinch & rotate (multi-touch), `open-url`, `reinstall-app`, - `restart-app`, and the native inspection / profiling tools (`native-*`, `native-profiler-*`, - `screenshot-diff`) — all return a clear "not supported" error. `launch-app` (via `devicectl`) - works independently of the CoreDevice tunnel — it can succeed even before the tunnel setup - above has run. +- `describe` / accessibility inspection is **not available** on a physical device — use + `screenshot` instead. This is an Apple restriction, not a missing feature: the on-device + accessibility tree is served by CoreDevice's `axAuditDaemon`, which Apple gates to trusted / + `AppleInternal` callers. Over the developer (untrusted) CoreDevice tunnel `pymobiledevice3` + forms, the daemon accepts the connection but drops it on the first request, and its modern + RemoteXPC replacement requires an `AppleInternal` entitlement — so there is no app-free + accessibility tree to read (hardware-verified on iOS 27). +- Not supported yet (return a clear "not supported" error): keyboard/typing, pinch & rotate + (multi-touch), `open-url`, `reinstall-app`, `restart-app`, and the native inspection / + profiling tools (`native-*`, `native-profiler-*`, `screenshot-diff`). `launch-app` (via + `devicectl`) works independently of the CoreDevice tunnel — it can succeed even before the + tunnel setup above has run. - Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT` (defaults to `49151`). diff --git a/packages/tool-server/src/blueprints/ax-service.ts b/packages/tool-server/src/blueprints/ax-service.ts index f2f79b816..cff93128f 100644 --- a/packages/tool-server/src/blueprints/ax-service.ts +++ b/packages/tool-server/src/blueprints/ax-service.ts @@ -237,10 +237,14 @@ export const axServiceBlueprint: ServiceBlueprint = { } if (device.kind === "device") { // ax-service uses `xcrun simctl spawn`, which only works on simulators. - // Physical iPhones are driven over CoreDevice and have no accessibility/ - // describe path yet. + // Physical iPhones are driven over CoreDevice, which has no app-free + // accessibility path: Apple gates the on-device axAuditDaemon to + // trusted/AppleInternal callers (see describe/platforms/ios/index.ts for + // the hardware-verified detail). `describe` rejects physical iOS before it + // reaches this guard — this is the backstop for any direct ax-service + // resolution. throw new FailureError( - `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; describe/accessibility is not supported on physical iOS yet.`, + `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice, whose accessibility service Apple restricts to trusted/AppleInternal callers; describe is not available on physical iOS.`, { error_code: FAILURE_CODES.AX_PHYSICAL_DEVICE_UNSUPPORTED, failure_stage: "ax_service_factory_platform", diff --git a/packages/tool-server/src/tools/describe/platforms/ios/index.ts b/packages/tool-server/src/tools/describe/platforms/ios/index.ts index 6a2d97128..7e51a4af1 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/index.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/index.ts @@ -58,18 +58,35 @@ export async function describeIos( params: DescribeIosParams, options: DescribeIosOptions = {} ): Promise { - // Physical iPhones are driven over CoreDevice; both describe backends are - // simulator-only (ax-service shells `simctl spawn`; native-devtools injects a - // dylib via `simctl spawn`). Their blueprint guards throw for kind === "device", - // but the fallback chain below catches those throws and would otherwise return - // an empty tree plus the "reboot the simulator" degraded hint — a misleading - // result for hardware that has no simulator to reboot. Reject explicitly with a - // clear, 400-mapped error instead. + // Physical iPhones are driven over CoreDevice, and there is no app-free + // describe (accessibility tree) path on a real device today: + // - The two simulator backends can't run against hardware: ax-service + // shells `simctl spawn` and native-devtools injects a dylib via + // `simctl spawn`, both simulator-only. Their blueprint guards throw for + // kind === "device"; without this early return the fallback chain below + // would swallow those throws and return an empty tree + the "reboot the + // simulator" degraded hint — meaningless for hardware with no simulator. + // - The on-device accessibility tree is served by CoreDevice's + // axAuditDaemon, but Apple gates it to trusted/AppleInternal callers + // (hardware-verified on iOS 27, 2026-07): the DTX service + // `com.apple.accessibility.axAuditDaemon.remoteserver.shim.remote` + // requires the `com.apple.mobile.lockdown.remote.trusted` entitlement — + // over the developer (untrusted) CoreDevice tunnel pymobiledevice3 forms, + // the daemon accepts the socket but terminates it on the first request + // (every audit selector, and even the standard DTX capability handshake). + // The RemoteXPC replacement `…axAuditDaemon.remoteAXService` requires the + // `AppleInternal` entitlement, unreachable by any third party. DTX + // transport itself works over the same tunnel (`developer dvt proclist` + // succeeds), so this is Apple's auth wall, not a transport gap. Screenshot + // (CoreDevice `canViewDeviceDisplay`) is the only app-free screen capture. + // Reject explicitly with a clear, 400-mapped error and point at screenshot. if (isPhysicalIos(device)) { throw new UnsupportedOperationError( "describe", device, - "physical iOS is driven over CoreDevice and has no on-device accessibility/describe path; use screenshot to inspect the screen" + "Apple restricts the physical device's CoreDevice accessibility service to " + + "trusted/AppleInternal callers, so there is no app-free accessibility tree; " + + "use screenshot to inspect the screen" ); } From dc6587d4bf7e4edbe34fd9cb98a5867511dd2c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Mon, 6 Jul 2026 21:43:21 +0200 Subject: [PATCH 15/16] perf(physical-ios): persistent CoreDevice sidecar + home-screen describe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive a physical iPhone through one long-lived pymobiledevice3 process per device instead of spawning the CLI per interaction. Each `pymobiledevice3` invocation cost ~0.8s (~0.5s just `import pymobiledevice3`); the sidecar connects the RSD tunnel and opens the HID/screenshot services once, so a warm tap/button/swipe is now ~150-320ms (5x) — verified live on an iPhone 15 (iOS 27) through the tool-server. The agent is a small Python program, base64-embedded in coredevice-agent.ts (survives esbuild, no build-pipeline wiring), materialized to a temp file and run with pmd3's own venv interpreter; it speaks newline-delimited JSON over stdio. core-device.ts now maps its responses (incl. the iOS-27 9021 gate) to FailureErrors and tears the process down on dispose. Also make `describe` work on a physical iPhone: it returns the SpringBoard home-screen layout (app icons + dock) via CoreDevice's springboardservices — the only app-free structured screen data on a real device. Icon frames are derived from the home-screen grid (row-major first-fit so multi-cell widgets don't shift the icons after them) and are approximate; the hint says to confirm with screenshot and that in-app content isn't reachable. Verified the describe→tap loop live (tapped Settings via describe's frame; it opened). In-app accessibility stays Apple-gated: the axAuditDaemon DTX service drops the connection on the first message over the developer tunnel, and its RemoteXPC replacement needs the AppleInternal entitlement. Replaces the per-call argv test with sidecar/adapter/agent-protocol coverage. --- README.md | 23 +- .../tool-server/src/blueprints/ax-service.ts | 14 +- .../tool-server/src/blueprints/core-device.ts | 178 ++++++------- .../src/blueprints/coredevice-agent.py | 189 +++++++++++++ .../src/blueprints/coredevice-agent.ts | 252 ++++++++++++++++++ .../src/tools/describe/contract.ts | 3 +- .../src/tools/describe/platforms/ios/index.ts | 69 ++--- .../platforms/ios/ios-springboard-adapter.ts | 242 +++++++++++++++++ .../test/physical-ios-coredevice-argv.test.ts | 251 ----------------- .../physical-ios-coredevice-sidecar.test.ts | 173 ++++++++++++ .../test/physical-ios-followups.test.ts | 26 +- 11 files changed, 1015 insertions(+), 405 deletions(-) create mode 100644 packages/tool-server/src/blueprints/coredevice-agent.py create mode 100644 packages/tool-server/src/blueprints/coredevice-agent.ts create mode 100644 packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts delete mode 100644 packages/tool-server/test/physical-ios-coredevice-argv.test.ts create mode 100644 packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts diff --git a/README.md b/README.md index 1f26a6767..085303dc2 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,11 @@ Argent drives a growing set of targets through a single toolkit, each with the r Argent can drive a **physical iPhone** — no app installed on the device — over Apple's CoreDevice "remote control" services (the same path Xcode's device window uses), via [`pymobiledevice3`](https://github.com/doronz88/pymobiledevice3). Supported interactions: -`screenshot`, `gesture-tap`, `gesture-swipe`, `button`, and `launch-app`. The device shows -up in `list-devices` with `kind: "device"`. +`screenshot`, `gesture-tap`, `gesture-swipe`, `button`, `launch-app`, and `describe` (the +home-screen app grid — see the note below). The device shows up in `list-devices` with +`kind: "device"`. Interactions run through one persistent `pymobiledevice3` helper per +device (connected once), so a tap/screenshot costs a socket write rather than a fresh +Python cold-start. **Requirements** @@ -83,13 +86,15 @@ and leave it running: `sudo pymobiledevice3 remote tunneld`. **Limitations / notes** -- `describe` / accessibility inspection is **not available** on a physical device — use - `screenshot` instead. This is an Apple restriction, not a missing feature: the on-device - accessibility tree is served by CoreDevice's `axAuditDaemon`, which Apple gates to trusted / - `AppleInternal` callers. Over the developer (untrusted) CoreDevice tunnel `pymobiledevice3` - forms, the daemon accepts the connection but drops it on the first request, and its modern - RemoteXPC replacement requires an `AppleInternal` entitlement — so there is no app-free - accessibility tree to read (hardware-verified on iOS 27). +- `describe` returns the **SpringBoard home-screen layout** (app icons + dock, via CoreDevice's + `springboardservices`) — the only app-free _structured_ screen data on a physical device. + Icon frames are derived from the home-screen grid geometry, so they're approximate: good + enough to tap the right icon, but confirm with `screenshot` before a precise tap. It always + reflects the home screen, so if an app is open its content is **not** there — use `screenshot`. + There is no app-free _in-app_ accessibility tree: that's served by CoreDevice's `axAuditDaemon`, + which Apple gates to trusted / `AppleInternal` callers (hardware-verified on iOS 27 — the DTX + service drops the connection on the first request over the developer tunnel, and its modern + RemoteXPC replacement needs the `AppleInternal` entitlement). - Not supported yet (return a clear "not supported" error): keyboard/typing, pinch & rotate (multi-touch), `open-url`, `reinstall-app`, `restart-app`, and the native inspection / profiling tools (`native-*`, `native-profiler-*`, `screenshot-diff`). `launch-app` (via diff --git a/packages/tool-server/src/blueprints/ax-service.ts b/packages/tool-server/src/blueprints/ax-service.ts index cff93128f..0b986ef36 100644 --- a/packages/tool-server/src/blueprints/ax-service.ts +++ b/packages/tool-server/src/blueprints/ax-service.ts @@ -237,14 +237,14 @@ export const axServiceBlueprint: ServiceBlueprint = { } if (device.kind === "device") { // ax-service uses `xcrun simctl spawn`, which only works on simulators. - // Physical iPhones are driven over CoreDevice, which has no app-free - // accessibility path: Apple gates the on-device axAuditDaemon to - // trusted/AppleInternal callers (see describe/platforms/ios/index.ts for - // the hardware-verified detail). `describe` rejects physical iOS before it - // reaches this guard — this is the backstop for any direct ax-service - // resolution. + // Physical iPhones are driven over CoreDevice; their in-app accessibility + // tree is Apple-gated (the axAuditDaemon rejects trusted/AppleInternal + // callers — see describe/platforms/ios/index.ts). `describe` handles a + // physical device itself (via the SpringBoard home-screen layout) and + // never resolves ax-service for it — this is a backstop for any other + // direct ax-service resolution. throw new FailureError( - `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice, whose accessibility service Apple restricts to trusted/AppleInternal callers; describe is not available on physical iOS.`, + `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; its in-app accessibility tree is Apple-gated, so use describe (SpringBoard home screen) or screenshot instead.`, { error_code: FAILURE_CODES.AX_PHYSICAL_DEVICE_UNSUPPORTED, failure_stage: "ax_service_factory_platform", diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts index 80c1fd6b9..b6a81de64 100644 --- a/packages/tool-server/src/blueprints/core-device.ts +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -1,6 +1,7 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { existsSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; import { tmpdir, homedir } from "node:os"; import { join } from "node:path"; import { randomUUID } from "node:crypto"; @@ -15,6 +16,12 @@ import { } from "@argent/registry"; import { isFlagEnabled } from "@argent/configuration-core"; import { deviceAuthHelperPath, argentIconPath } from "@argent/native-devtools-ios"; +import { + CoreDeviceAgent, + CoreDeviceAgentError, + materializeAgentScript, + resolvePmd3Python, +} from "./coredevice-agent"; const execFileAsync = promisify(execFile); @@ -63,6 +70,13 @@ type CoreDeviceFactoryOptions = Record & { device: DeviceInfo } * running CoreDevice tunnel (`sudo pymobiledevice3 remote tunneld`, which needs * root to create the tunnel interface — every command here then runs unprivileged). */ +export interface CoreDeviceHomescreen { + /** SpringBoard `getIconState` — the home-screen app/folder/widget layout. */ + iconState: unknown; + /** SpringBoard icon-grid geometry (columns/rows/icon size/screen points). */ + metrics: Record; +} + export interface CoreDeviceApi { /** Capture a PNG to a temp file and return its path. */ screenshot(): Promise<{ path: string }>; @@ -72,6 +86,12 @@ export interface CoreDeviceApi { swipe(fromX: number, fromY: number, toX: number, toY: number, durationMs: number): Promise; /** Press a hardware button by its pymobiledevice3 name (home/lock/volume-up/volume-down/...). */ button(name: string): Promise; + /** + * The SpringBoard home-screen layout — the only app-free *structured* screen + * data reachable on a physical iPhone (in-app accessibility is Apple-gated; + * see describe/platforms/ios). Backs `describe` for the home screen. + */ + homescreen(): Promise; } export function coreDeviceRef(device: DeviceInfo): { @@ -383,31 +403,16 @@ export async function ensureCoreDeviceTunnel(udid: string): Promise { } /** - * Apple gates host-driven input (the CoreDevice "remote control" services that - * back tap/swipe/button) to iOS 27+. On iOS 18-26 those services exist but the - * device rejects the command with `CoreDeviceError 9021`, which `conciseError` - * would otherwise surface as a raw, unactionable line. `screen-capture` is NOT - * gated, so screenshots keep working and never hit this path. - * - * Detected against pymobiledevice3's own output (stderr/stdout) only — never the - * execFile `message`, which echoes the argv where a HID coordinate (0..65535) - * could itself be `9021` and false-positive a bare numeric match. + * Map a CoreDevice agent failure to a FailureError. The agent flags Apple's + * host-input gate (CoreDeviceError 9021) explicitly via `gated9021`: on iOS + * 18-26 the "remote control" services exist but reject touch input, while + * `screen-capture` and hardware buttons (`hid button`) keep working — so this + * message is scoped to tap/swipe, hardware-verified on an iPhone 15 + * (iOS 18.7.8 vs 27.0). */ -function isRemoteControlGatedError(err: unknown): boolean { - const e = err as { stderr?: string; stdout?: string }; - const output = [e?.stderr, e?.stdout] - .filter((v): v is string => typeof v === "string") - .join("\n"); - return /core\s*device\s*error\W*9021/i.test(output) || /\b9021\b/.test(output); -} - -/** Extract a concise, human-readable line from a (possibly huge/binary) pmd3 failure. */ -function conciseError(label: string, err: unknown): FailureError { +export function agentError(label: string, err: unknown): FailureError { const cause = err instanceof Error ? err : new Error(String(err)); - if (isRemoteControlGatedError(err)) { - // Hardware-verified (iPhone 15, iOS 18.7.8 vs 27.0): screenshot and hardware - // buttons (`hid button`) work below iOS 27 — only touch (tap/swipe/drag) is - // actually gated. Don't overclaim "button" is affected too. + if (err instanceof CoreDeviceAgentError && err.gated9021) { return new FailureError( `CoreDevice ${label} failed: this iPhone is on an iOS below 27; host-driven touch input ` + `(tap/swipe) requires iOS 27+. Screenshot and hardware buttons work on earlier iOS ` + @@ -422,35 +427,8 @@ function conciseError(label: string, err: unknown): FailureError { { cause } ); } - const e = err as { stderr?: string; stdout?: string; message?: string }; - // stderr/stdout are pymobiledevice3's own output — the trustworthy source for - // the real cause. `message` (execFile's synthesized "Command failed: ") - // is deliberately searched LAST, as a fallback only: it's just the invoked - // command line, not a real diagnostic, and searching it with equal priority - // would let its ever-present "Command failed" text outrank a genuine - // stderr/stdout line lower in that output. - const outputLines = [e?.stderr, e?.stdout] - .filter((v): v is string => typeof v === "string") - .join("\n") - .split("\n") - .map((s) => s.trim()) - // pymobiledevice3 renders failures via Python's `rich`, which frames source - // context in a box (each line starting with a `│`/`┃` rule) — never the - // actual exception. A keyword like "not "/"fail" can spuriously match a - // highlighted source line (e.g. "if rsd is not None:") before the real - // cause appears, so those are never eligible picks. - .filter((l) => l.length > 0 && !/^[│┃╭╮╰╯─━]/.test(l)); - const line = - // Prefer a genuine `SomeError: message` / `SomeException: message` line — - // the actual raised exception, not just an incidentally keyword-matching - // one — and prefer the LAST such line, since that's where Python puts the - // real exception in a traceback (rich or plain). - outputLines.findLast((l) => /^[\w.]*(?:Error|Exception)\b.*:/.test(l)) ?? - outputLines.findLast((l) => /requires iOS|error|fail|not |unable|refused|timed out/i.test(l)) ?? - e?.message ?? - "unknown error"; return new FailureError( - `CoreDevice ${label} failed: ${line.slice(0, 240)}`, + `CoreDevice ${label} failed: ${cause.message.slice(0, 240)}`, { error_code: FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED, failure_stage: "core_device_command", @@ -462,9 +440,6 @@ function conciseError(label: string, err: unknown): FailureError { ); } -const COREDEVICE = ["developer", "core-device"]; -const HID = [...COREDEVICE, "universal-hid-service"]; - /** Normalized 0..1 → the device's 0..65535 HID coordinate space. */ export function toHid(v: number): number { return Math.max(0, Math.min(65535, Math.round(v * 65535))); @@ -555,17 +530,26 @@ export const coreDeviceBlueprint: ServiceBlueprint = const rsd = await ensureCoreDeviceTunnel(udid); await ensureMounted(pmd3, rsd); - const run = async (label: string, args: string[], timeout: number): Promise => { - const tunnel = await resolveTunnel(udid); + // Resolve the interpreter that has pymobiledevice3 importable (the CLI's own + // venv python) and materialize the agent program, then start ONE long-lived + // process for this device: it connects the RSD tunnel and opens the HID / + // screenshot services once, so each interaction is a socket write instead of + // a fresh ~0.8s Python cold-start (~0.5s of which is just the pmd3 import). + const python = await resolvePmd3Python(await resolvePmd3Absolute()); + const scriptPath = await materializeAgentScript(); + const agent = new CoreDeviceAgent(python, scriptPath, udid, tunneldPort()); + await agent.start(); + + const call = async ( + label: string, + op: string, + args: Record = {}, + timeoutMs = 30_000 + ): Promise> => { try { - const { stdout } = await execFileAsync( - pmd3, - [...args, "--rsd", tunnel.address, String(tunnel.port)], - { timeout, maxBuffer: 16 * 1024 * 1024 } - ); - return stdout; + return (await agent.request(op, args, timeoutMs)) as Record; } catch (err) { - throw conciseError(label, err); + throw agentError(label, err); } }; @@ -573,63 +557,51 @@ export const coreDeviceBlueprint: ServiceBlueprint = const api: CoreDeviceApi = { async screenshot() { + const res = await call("screenshot", "screenshot", {}, 30_000); + const b64 = typeof res.image_b64 === "string" ? res.image_b64 : ""; const path = join(tmpdir(), `argent-ios-shot-${randomUUID()}.png`); - await run("screenshot", [...COREDEVICE, "screen-capture", "screenshot", path], 20_000); + await writeFile(path, Buffer.from(b64, "base64")); return { path }; }, async tap(x, y) { - const hx = toHid(x); - const hy = toHid(y); - // A zero-duration CONTACT+RELEASE (pmd3's `tap`) is dropped by iOS — a - // tap must dwell. Emit a short held drag with a tiny (~3px) move, away - // from the edge so it never clips, which registers as a real tap. - const hy2 = hy <= 65535 - 120 ? hy + 96 : hy - 96; - await run( - "tap", - [ - ...HID, - "drag", - String(hx), - String(hy), - String(hx), - String(hy2), - "--steps", - "3", - "--duration", - "0.15", - ], - 15_000 - ); + // A zero-duration tap is dropped by iOS; the agent emits a short held + // dwell-drag. We hand it the 0..65535 HID coordinate. + await call("tap", "tap", { x: toHid(x), y: toHid(y) }, 15_000); }, async swipe(fromX, fromY, toX, toY, durationMs) { const { steps, seconds, timeoutMs } = swipeDragParams(durationMs); - await run( + await call( "swipe", - [ - ...HID, - "drag", - String(toHid(fromX)), - String(toHid(fromY)), - String(toHid(toX)), - String(toHid(toY)), - "--steps", - String(steps), - "--duration", - seconds, - ], + "swipe", + { + x1: toHid(fromX), + y1: toHid(fromY), + x2: toHid(toX), + y2: toHid(toY), + steps, + duration: Number(seconds), + }, timeoutMs ); }, async button(name) { - await run("button", [...COREDEVICE, "hid", "button", name, "press"], 10_000); + await call("button", "button", { name }, 15_000); + }, + async homescreen() { + const res = await call("homescreen", "homescreen", {}, 20_000); + return { + iconState: res.icon_state, + metrics: (res.metrics as Record) ?? {}, + }; }, }; const instance: ServiceInstance = { api, - // Stateless: each interaction is a fresh pymobiledevice3 invocation, so - // there is no long-lived process to tear down. - dispose: async () => {}, + // One persistent pymobiledevice3 process per device — tear it down. + dispose: async () => { + agent.dispose(); + }, events, }; return instance; diff --git a/packages/tool-server/src/blueprints/coredevice-agent.py b/packages/tool-server/src/blueprints/coredevice-agent.py new file mode 100644 index 000000000..7e1bacfca --- /dev/null +++ b/packages/tool-server/src/blueprints/coredevice-agent.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Persistent CoreDevice agent — one long-lived process per physical iPhone. + +Replaces per-call `pymobiledevice3` CLI spawns (each ~0.8s, ~0.5s just the +Python import) with a single process that connects the RSD tunnel once, holds +the touchscreen media-stream session + screenshot service open, and executes +newline-delimited JSON commands on stdin, replying with one JSON line each. + +Reuses pymobiledevice3's own CLI helpers so behaviour is identical to the +`developer core-device …` commands (dwell-drag tap, mainTouchscreen reports, +Indigo hardware buttons, screen-capture PNG). + +Protocol (one JSON object per line): + <- {"udid": "...", "port": 49151} (argv, not stdin) + -> {"ready": true} (or {"ready": false, "error": "..."}) + <- {"id": 1, "op": "screenshot"} + -> {"id": 1, "ok": true, "image_b64": "..."} + <- {"id": 2, "op": "tap", "x": 32768, "y": 20000} (x/y already 0..65535) + <- {"id": 3, "op": "swipe", "x1":.., "y1":.., "x2":.., "y2":.., "steps":19, "duration":0.3} + <- {"id": 4, "op": "button", "name": "home"} + <- {"id": 5, "op": "homescreen"} (springboard icon grid) + -> {"id": N, "ok": true, ...} | {"id": N, "error": "...", "gated_9021": bool} +""" +import asyncio +import base64 +import contextlib +import json +import sys + +from pymobiledevice3.remote.remote_service_discovery import RemoteServiceDiscoveryService +from pymobiledevice3.remote.core_device.hid_service import ( + touch_session, + IndigoHIDService, + DIGITIZER_SURFACE_MAIN_TOUCHSCREEN, +) +from pymobiledevice3.remote.core_device.screen_capture_service import ScreenCaptureService +from pymobiledevice3.services.springboard import SpringBoardServicesService +from pymobiledevice3.cli.developer.core_device import _do_drag, _send_button_press, _NAMED_BUTTONS + +import urllib.request + + +def _resolve_rsd(udid: str, port: int): + payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=4)) + entry = payload.get(udid) or [] + t = entry[0] if entry else {} + addr, tport = t.get("tunnel-address"), t.get("tunnel-port") + if not addr or not tport: + raise RuntimeError(f"no active tunnel registered for {udid} on tunneld :{port}") + return addr, int(tport) + + +async def _maybe_await(v): + return await v if asyncio.iscoroutine(v) else v + + +class Agent: + def __init__(self, udid: str, port: int): + self.udid = udid + self.port = port + self.rsd = None + self.stack = contextlib.AsyncExitStack() + self.touch = None # UniversalHIDServiceService, held open (media stream stays warm) + + async def connect(self): + addr, tport = _resolve_rsd(self.udid, self.port) + self.rsd = RemoteServiceDiscoveryService((addr, tport)) + await self.rsd.connect() + + async def _ensure_touch(self): + # Lazily open (and keep) the touchscreen media-stream session — the auth + # gate backboardd needs for injected touches. Kept warm so taps don't pay + # the media-stream startup each time. + if self.touch is None: + self.touch = await self.stack.enter_async_context(touch_session(self.rsd)) + return self.touch + + async def op_screenshot(self, _): + # ScreenCaptureService delivers one PNG per open (the stream ends after + # the frame), so open a fresh one each call — cheap now the interpreter + # and tunnel are already warm. + async with ScreenCaptureService(self.rsd) as screen: + resp = await screen.capture_screenshot() + return {"image_b64": base64.b64encode(resp["image"]).decode("ascii")} + + async def op_tap(self, msg): + svc = await self._ensure_touch() + x, y = int(msg["x"]), int(msg["y"]) + # Zero-dwell taps are dropped by iOS; emit a short held drag with a tiny + # move away from the edge (mirrors core-device.ts / the CLI drag path). + y2 = y + 96 if y <= 65535 - 120 else y - 96 + await _do_drag(svc, x, y, x, y2, 3, 0.15, tsid=DIGITIZER_SURFACE_MAIN_TOUCHSCREEN) + return {} + + async def op_swipe(self, msg): + svc = await self._ensure_touch() + await _do_drag( + svc, int(msg["x1"]), int(msg["y1"]), int(msg["x2"]), int(msg["y2"]), + int(msg.get("steps", 19)), float(msg.get("duration", 0.3)), + tsid=DIGITIZER_SURFACE_MAIN_TOUCHSCREEN, + ) + return {} + + async def op_button(self, msg): + name = msg["name"] + if name not in _NAMED_BUTTONS: + raise RuntimeError(f"unknown button '{name}'") + usage_page, usage_code, hold = _NAMED_BUTTONS[name] + async with IndigoHIDService(self.rsd) as svc: + await _send_button_press(svc, usage_page, usage_code, "press", hold) + return {} + + async def op_homescreen(self, _): + sb = SpringBoardServicesService(lockdown=self.rsd) + icons = await _maybe_await(sb.get_icon_state()) + metrics = await _maybe_await(sb.get_homescreen_icon_metrics()) + return {"icon_state": icons, "metrics": metrics} + + async def op_ping(self, _): + return {"pong": True} + + async def dispatch(self, msg): + op = msg.get("op") + fn = getattr(self, f"op_{op}", None) + if fn is None: + raise RuntimeError(f"unknown op '{op}'") + return await fn(msg) + + async def close(self): + with contextlib.suppress(Exception): + await self.stack.aclose() + if self.rsd is not None: + with contextlib.suppress(Exception): + await self.rsd.close() + + +def _is_9021(text: str) -> bool: + import re + return bool(re.search(r"core\s*device\s*error\W*9021", text, re.I) or re.search(r"\b9021\b", text)) + + +async def main(): + udid = sys.argv[1] + port = int(sys.argv[2]) if len(sys.argv) > 2 else 49151 + agent = Agent(udid, port) + out = sys.stdout + + def emit(obj): + # default=str keeps a stray plist type (bytes/datetime in springboard + # icon state) from crashing serialization mid-session. + out.write(json.dumps(obj, default=str) + "\n") + out.flush() + + try: + await agent.connect() + except Exception as e: # noqa: BLE001 + emit({"ready": False, "error": str(e)}) + return + emit({"ready": True}) + + loop = asyncio.get_event_loop() + reader = asyncio.StreamReader() + await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(reader), sys.stdin) + + while True: + line = await reader.readline() + if not line: + break + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: # noqa: BLE001 + emit({"error": "bad json"}) + continue + mid = msg.get("id") + try: + result = await agent.dispatch(msg) + emit({"id": mid, "ok": True, **result}) + except Exception as e: # noqa: BLE001 + text = f"{type(e).__name__}: {e}" + emit({"id": mid, "error": text, "gated_9021": _is_9021(text)}) + + await agent.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/packages/tool-server/src/blueprints/coredevice-agent.ts b/packages/tool-server/src/blueprints/coredevice-agent.ts new file mode 100644 index 000000000..9242296f9 --- /dev/null +++ b/packages/tool-server/src/blueprints/coredevice-agent.ts @@ -0,0 +1,252 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { createHash } from "node:crypto"; +import { createInterface, type Interface } from "node:readline"; + +/** + * The persistent CoreDevice agent (a small pymobiledevice3 program) that a + * physical iPhone is driven through. It is base64-embedded rather than shipped + * as a loose `.py` so it survives esbuild bundling with zero build-pipeline + * wiring, and base64 (not a template literal) so its Python `\s`/`\n` escapes + * can't be mangled by JS string parsing. + * + * Source of truth: `coredevice-agent.py.b64` sits next to this file; regenerate + * with `base64 -i coredevice-agent.py | tr -d '\n'`. The decoded script speaks + * newline-delimited JSON on stdio (see AGENT_PROTOCOL below). + */ +const AGENT_SCRIPT_B64 = + "IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwoiIiJQZXJzaXN0ZW50IENvcmVEZXZpY2UgYWdlbnQg4oCUIG9uZSBsb25nLWxpdmVkIHByb2Nlc3MgcGVyIHBoeXNpY2FsIGlQaG9uZS4KClJlcGxhY2VzIHBlci1jYWxsIGBweW1vYmlsZWRldmljZTNgIENMSSBzcGF3bnMgKGVhY2ggfjAuOHMsIH4wLjVzIGp1c3QgdGhlClB5dGhvbiBpbXBvcnQpIHdpdGggYSBzaW5nbGUgcHJvY2VzcyB0aGF0IGNvbm5lY3RzIHRoZSBSU0QgdHVubmVsIG9uY2UsIGhvbGRzCnRoZSB0b3VjaHNjcmVlbiBtZWRpYS1zdHJlYW0gc2Vzc2lvbiArIHNjcmVlbnNob3Qgc2VydmljZSBvcGVuLCBhbmQgZXhlY3V0ZXMKbmV3bGluZS1kZWxpbWl0ZWQgSlNPTiBjb21tYW5kcyBvbiBzdGRpbiwgcmVwbHlpbmcgd2l0aCBvbmUgSlNPTiBsaW5lIGVhY2guCgpSZXVzZXMgcHltb2JpbGVkZXZpY2UzJ3Mgb3duIENMSSBoZWxwZXJzIHNvIGJlaGF2aW91ciBpcyBpZGVudGljYWwgdG8gdGhlCmBkZXZlbG9wZXIgY29yZS1kZXZpY2Ug4oCmYCBjb21tYW5kcyAoZHdlbGwtZHJhZyB0YXAsIG1haW5Ub3VjaHNjcmVlbiByZXBvcnRzLApJbmRpZ28gaGFyZHdhcmUgYnV0dG9ucywgc2NyZWVuLWNhcHR1cmUgUE5HKS4KClByb3RvY29sIChvbmUgSlNPTiBvYmplY3QgcGVyIGxpbmUpOgogIDwtIHsidWRpZCI6ICIuLi4iLCAicG9ydCI6IDQ5MTUxfSAgICAgICAgICAgICAgICAgKGFyZ3YsIG5vdCBzdGRpbikKICAtPiB7InJlYWR5IjogdHJ1ZX0gICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAob3IgeyJyZWFkeSI6IGZhbHNlLCAiZXJyb3IiOiAiLi4uIn0pCiAgPC0geyJpZCI6IDEsICJvcCI6ICJzY3JlZW5zaG90In0KICAtPiB7ImlkIjogMSwgIm9rIjogdHJ1ZSwgImltYWdlX2I2NCI6ICIuLi4ifQogIDwtIHsiaWQiOiAyLCAib3AiOiAidGFwIiwgIngiOiAzMjc2OCwgInkiOiAyMDAwMH0gICAgICAgICAgKHgveSBhbHJlYWR5IDAuLjY1NTM1KQogIDwtIHsiaWQiOiAzLCAib3AiOiAic3dpcGUiLCAieDEiOi4uLCAieTEiOi4uLCAieDIiOi4uLCAieTIiOi4uLCAic3RlcHMiOjE5LCAiZHVyYXRpb24iOjAuM30KICA8LSB7ImlkIjogNCwgIm9wIjogImJ1dHRvbiIsICJuYW1lIjogImhvbWUifQogIDwtIHsiaWQiOiA1LCAib3AiOiAiaG9tZXNjcmVlbiJ9ICAgICAgICAgICAgICAgICAgICAgICAgICAgIChzcHJpbmdib2FyZCBpY29uIGdyaWQpCiAgLT4geyJpZCI6IE4sICJvayI6IHRydWUsIC4uLn0gIHwgIHsiaWQiOiBOLCAiZXJyb3IiOiAiLi4uIiwgImdhdGVkXzkwMjEiOiBib29sfQoiIiIKaW1wb3J0IGFzeW5jaW8KaW1wb3J0IGJhc2U2NAppbXBvcnQgY29udGV4dGxpYgppbXBvcnQganNvbgppbXBvcnQgc3lzCgpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUucmVtb3RlX3NlcnZpY2VfZGlzY292ZXJ5IGltcG9ydCBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2UuaGlkX3NlcnZpY2UgaW1wb3J0ICgKICAgIHRvdWNoX3Nlc3Npb24sCiAgICBJbmRpZ29ISURTZXJ2aWNlLAogICAgRElHSVRJWkVSX1NVUkZBQ0VfTUFJTl9UT1VDSFNDUkVFTiwKKQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2Uuc2NyZWVuX2NhcHR1cmVfc2VydmljZSBpbXBvcnQgU2NyZWVuQ2FwdHVyZVNlcnZpY2UKZnJvbSBweW1vYmlsZWRldmljZTMuc2VydmljZXMuc3ByaW5nYm9hcmQgaW1wb3J0IFNwcmluZ0JvYXJkU2VydmljZXNTZXJ2aWNlCmZyb20gcHltb2JpbGVkZXZpY2UzLmNsaS5kZXZlbG9wZXIuY29yZV9kZXZpY2UgaW1wb3J0IF9kb19kcmFnLCBfc2VuZF9idXR0b25fcHJlc3MsIF9OQU1FRF9CVVRUT05TCgppbXBvcnQgdXJsbGliLnJlcXVlc3QKCgpkZWYgX3Jlc29sdmVfcnNkKHVkaWQ6IHN0ciwgcG9ydDogaW50KToKICAgIHBheWxvYWQgPSBqc29uLmxvYWQodXJsbGliLnJlcXVlc3QudXJsb3BlbihmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9LyIsIHRpbWVvdXQ9NCkpCiAgICBlbnRyeSA9IHBheWxvYWQuZ2V0KHVkaWQpIG9yIFtdCiAgICB0ID0gZW50cnlbMF0gaWYgZW50cnkgZWxzZSB7fQogICAgYWRkciwgdHBvcnQgPSB0LmdldCgidHVubmVsLWFkZHJlc3MiKSwgdC5nZXQoInR1bm5lbC1wb3J0IikKICAgIGlmIG5vdCBhZGRyIG9yIG5vdCB0cG9ydDoKICAgICAgICByYWlzZSBSdW50aW1lRXJyb3IoZiJubyBhY3RpdmUgdHVubmVsIHJlZ2lzdGVyZWQgZm9yIHt1ZGlkfSBvbiB0dW5uZWxkIDp7cG9ydH0iKQogICAgcmV0dXJuIGFkZHIsIGludCh0cG9ydCkKCgphc3luYyBkZWYgX21heWJlX2F3YWl0KHYpOgogICAgcmV0dXJuIGF3YWl0IHYgaWYgYXN5bmNpby5pc2Nvcm91dGluZSh2KSBlbHNlIHYKCgpjbGFzcyBBZ2VudDoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCB1ZGlkOiBzdHIsIHBvcnQ6IGludCk6CiAgICAgICAgc2VsZi51ZGlkID0gdWRpZAogICAgICAgIHNlbGYucG9ydCA9IHBvcnQKICAgICAgICBzZWxmLnJzZCA9IE5vbmUKICAgICAgICBzZWxmLnN0YWNrID0gY29udGV4dGxpYi5Bc3luY0V4aXRTdGFjaygpCiAgICAgICAgc2VsZi50b3VjaCA9IE5vbmUgICMgVW5pdmVyc2FsSElEU2VydmljZVNlcnZpY2UsIGhlbGQgb3BlbiAobWVkaWEgc3RyZWFtIHN0YXlzIHdhcm0pCgogICAgYXN5bmMgZGVmIGNvbm5lY3Qoc2VsZik6CiAgICAgICAgYWRkciwgdHBvcnQgPSBfcmVzb2x2ZV9yc2Qoc2VsZi51ZGlkLCBzZWxmLnBvcnQpCiAgICAgICAgc2VsZi5yc2QgPSBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZSgoYWRkciwgdHBvcnQpKQogICAgICAgIGF3YWl0IHNlbGYucnNkLmNvbm5lY3QoKQoKICAgIGFzeW5jIGRlZiBfZW5zdXJlX3RvdWNoKHNlbGYpOgogICAgICAgICMgTGF6aWx5IG9wZW4gKGFuZCBrZWVwKSB0aGUgdG91Y2hzY3JlZW4gbWVkaWEtc3RyZWFtIHNlc3Npb24g4oCUIHRoZSBhdXRoCiAgICAgICAgIyBnYXRlIGJhY2tib2FyZGQgbmVlZHMgZm9yIGluamVjdGVkIHRvdWNoZXMuIEtlcHQgd2FybSBzbyB0YXBzIGRvbid0IHBheQogICAgICAgICMgdGhlIG1lZGlhLXN0cmVhbSBzdGFydHVwIGVhY2ggdGltZS4KICAgICAgICBpZiBzZWxmLnRvdWNoIGlzIE5vbmU6CiAgICAgICAgICAgIHNlbGYudG91Y2ggPSBhd2FpdCBzZWxmLnN0YWNrLmVudGVyX2FzeW5jX2NvbnRleHQodG91Y2hfc2Vzc2lvbihzZWxmLnJzZCkpCiAgICAgICAgcmV0dXJuIHNlbGYudG91Y2gKCiAgICBhc3luYyBkZWYgb3Bfc2NyZWVuc2hvdChzZWxmLCBfKToKICAgICAgICAjIFNjcmVlbkNhcHR1cmVTZXJ2aWNlIGRlbGl2ZXJzIG9uZSBQTkcgcGVyIG9wZW4gKHRoZSBzdHJlYW0gZW5kcyBhZnRlcgogICAgICAgICMgdGhlIGZyYW1lKSwgc28gb3BlbiBhIGZyZXNoIG9uZSBlYWNoIGNhbGwg4oCUIGNoZWFwIG5vdyB0aGUgaW50ZXJwcmV0ZXIKICAgICAgICAjIGFuZCB0dW5uZWwgYXJlIGFscmVhZHkgd2FybS4KICAgICAgICBhc3luYyB3aXRoIFNjcmVlbkNhcHR1cmVTZXJ2aWNlKHNlbGYucnNkKSBhcyBzY3JlZW46CiAgICAgICAgICAgIHJlc3AgPSBhd2FpdCBzY3JlZW4uY2FwdHVyZV9zY3JlZW5zaG90KCkKICAgICAgICByZXR1cm4geyJpbWFnZV9iNjQiOiBiYXNlNjQuYjY0ZW5jb2RlKHJlc3BbImltYWdlIl0pLmRlY29kZSgiYXNjaWkiKX0KCiAgICBhc3luYyBkZWYgb3BfdGFwKHNlbGYsIG1zZyk6CiAgICAgICAgc3ZjID0gYXdhaXQgc2VsZi5fZW5zdXJlX3RvdWNoKCkKICAgICAgICB4LCB5ID0gaW50KG1zZ1sieCJdKSwgaW50KG1zZ1sieSJdKQogICAgICAgICMgWmVyby1kd2VsbCB0YXBzIGFyZSBkcm9wcGVkIGJ5IGlPUzsgZW1pdCBhIHNob3J0IGhlbGQgZHJhZyB3aXRoIGEgdGlueQogICAgICAgICMgbW92ZSBhd2F5IGZyb20gdGhlIGVkZ2UgKG1pcnJvcnMgY29yZS1kZXZpY2UudHMgLyB0aGUgQ0xJIGRyYWcgcGF0aCkuCiAgICAgICAgeTIgPSB5ICsgOTYgaWYgeSA8PSA2NTUzNSAtIDEyMCBlbHNlIHkgLSA5NgogICAgICAgIGF3YWl0IF9kb19kcmFnKHN2YywgeCwgeSwgeCwgeTIsIDMsIDAuMTUsIHRzaWQ9RElHSVRJWkVSX1NVUkZBQ0VfTUFJTl9UT1VDSFNDUkVFTikKICAgICAgICByZXR1cm4ge30KCiAgICBhc3luYyBkZWYgb3Bfc3dpcGUoc2VsZiwgbXNnKToKICAgICAgICBzdmMgPSBhd2FpdCBzZWxmLl9lbnN1cmVfdG91Y2goKQogICAgICAgIGF3YWl0IF9kb19kcmFnKAogICAgICAgICAgICBzdmMsIGludChtc2dbIngxIl0pLCBpbnQobXNnWyJ5MSJdKSwgaW50KG1zZ1sieDIiXSksIGludChtc2dbInkyIl0pLAogICAgICAgICAgICBpbnQobXNnLmdldCgic3RlcHMiLCAxOSkpLCBmbG9hdChtc2cuZ2V0KCJkdXJhdGlvbiIsIDAuMykpLAogICAgICAgICAgICB0c2lkPURJR0lUSVpFUl9TVVJGQUNFX01BSU5fVE9VQ0hTQ1JFRU4sCiAgICAgICAgKQogICAgICAgIHJldHVybiB7fQoKICAgIGFzeW5jIGRlZiBvcF9idXR0b24oc2VsZiwgbXNnKToKICAgICAgICBuYW1lID0gbXNnWyJuYW1lIl0KICAgICAgICBpZiBuYW1lIG5vdCBpbiBfTkFNRURfQlVUVE9OUzoKICAgICAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKGYidW5rbm93biBidXR0b24gJ3tuYW1lfSciKQogICAgICAgIHVzYWdlX3BhZ2UsIHVzYWdlX2NvZGUsIGhvbGQgPSBfTkFNRURfQlVUVE9OU1tuYW1lXQogICAgICAgIGFzeW5jIHdpdGggSW5kaWdvSElEU2VydmljZShzZWxmLnJzZCkgYXMgc3ZjOgogICAgICAgICAgICBhd2FpdCBfc2VuZF9idXR0b25fcHJlc3Moc3ZjLCB1c2FnZV9wYWdlLCB1c2FnZV9jb2RlLCAicHJlc3MiLCBob2xkKQogICAgICAgIHJldHVybiB7fQoKICAgIGFzeW5jIGRlZiBvcF9ob21lc2NyZWVuKHNlbGYsIF8pOgogICAgICAgIHNiID0gU3ByaW5nQm9hcmRTZXJ2aWNlc1NlcnZpY2UobG9ja2Rvd249c2VsZi5yc2QpCiAgICAgICAgaWNvbnMgPSBhd2FpdCBfbWF5YmVfYXdhaXQoc2IuZ2V0X2ljb25fc3RhdGUoKSkKICAgICAgICBtZXRyaWNzID0gYXdhaXQgX21heWJlX2F3YWl0KHNiLmdldF9ob21lc2NyZWVuX2ljb25fbWV0cmljcygpKQogICAgICAgIHJldHVybiB7Imljb25fc3RhdGUiOiBpY29ucywgIm1ldHJpY3MiOiBtZXRyaWNzfQoKICAgIGFzeW5jIGRlZiBvcF9waW5nKHNlbGYsIF8pOgogICAgICAgIHJldHVybiB7InBvbmciOiBUcnVlfQoKICAgIGFzeW5jIGRlZiBkaXNwYXRjaChzZWxmLCBtc2cpOgogICAgICAgIG9wID0gbXNnLmdldCgib3AiKQogICAgICAgIGZuID0gZ2V0YXR0cihzZWxmLCBmIm9wX3tvcH0iLCBOb25lKQogICAgICAgIGlmIGZuIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcihmInVua25vd24gb3AgJ3tvcH0nIikKICAgICAgICByZXR1cm4gYXdhaXQgZm4obXNnKQoKICAgIGFzeW5jIGRlZiBjbG9zZShzZWxmKToKICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgYXdhaXQgc2VsZi5zdGFjay5hY2xvc2UoKQogICAgICAgIGlmIHNlbGYucnNkIGlzIG5vdCBOb25lOgogICAgICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgICAgIGF3YWl0IHNlbGYucnNkLmNsb3NlKCkKCgpkZWYgX2lzXzkwMjEodGV4dDogc3RyKSAtPiBib29sOgogICAgaW1wb3J0IHJlCiAgICByZXR1cm4gYm9vbChyZS5zZWFyY2gociJjb3JlXHMqZGV2aWNlXHMqZXJyb3JcVyo5MDIxIiwgdGV4dCwgcmUuSSkgb3IgcmUuc2VhcmNoKHIiXGI5MDIxXGIiLCB0ZXh0KSkKCgphc3luYyBkZWYgbWFpbigpOgogICAgdWRpZCA9IHN5cy5hcmd2WzFdCiAgICBwb3J0ID0gaW50KHN5cy5hcmd2WzJdKSBpZiBsZW4oc3lzLmFyZ3YpID4gMiBlbHNlIDQ5MTUxCiAgICBhZ2VudCA9IEFnZW50KHVkaWQsIHBvcnQpCiAgICBvdXQgPSBzeXMuc3Rkb3V0CgogICAgZGVmIGVtaXQob2JqKToKICAgICAgICAjIGRlZmF1bHQ9c3RyIGtlZXBzIGEgc3RyYXkgcGxpc3QgdHlwZSAoYnl0ZXMvZGF0ZXRpbWUgaW4gc3ByaW5nYm9hcmQKICAgICAgICAjIGljb24gc3RhdGUpIGZyb20gY3Jhc2hpbmcgc2VyaWFsaXphdGlvbiBtaWQtc2Vzc2lvbi4KICAgICAgICBvdXQud3JpdGUoanNvbi5kdW1wcyhvYmosIGRlZmF1bHQ9c3RyKSArICJcbiIpCiAgICAgICAgb3V0LmZsdXNoKCkKCiAgICB0cnk6CiAgICAgICAgYXdhaXQgYWdlbnQuY29ubmVjdCgpCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6ICAjIG5vcWE6IEJMRTAwMQogICAgICAgIGVtaXQoeyJyZWFkeSI6IEZhbHNlLCAiZXJyb3IiOiBzdHIoZSl9KQogICAgICAgIHJldHVybgogICAgZW1pdCh7InJlYWR5IjogVHJ1ZX0pCgogICAgbG9vcCA9IGFzeW5jaW8uZ2V0X2V2ZW50X2xvb3AoKQogICAgcmVhZGVyID0gYXN5bmNpby5TdHJlYW1SZWFkZXIoKQogICAgYXdhaXQgbG9vcC5jb25uZWN0X3JlYWRfcGlwZShsYW1iZGE6IGFzeW5jaW8uU3RyZWFtUmVhZGVyUHJvdG9jb2wocmVhZGVyKSwgc3lzLnN0ZGluKQoKICAgIHdoaWxlIFRydWU6CiAgICAgICAgbGluZSA9IGF3YWl0IHJlYWRlci5yZWFkbGluZSgpCiAgICAgICAgaWYgbm90IGxpbmU6CiAgICAgICAgICAgIGJyZWFrCiAgICAgICAgbGluZSA9IGxpbmUuc3RyaXAoKQogICAgICAgIGlmIG5vdCBsaW5lOgogICAgICAgICAgICBjb250aW51ZQogICAgICAgIHRyeToKICAgICAgICAgICAgbXNnID0ganNvbi5sb2FkcyhsaW5lKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246ICAjIG5vcWE6IEJMRTAwMQogICAgICAgICAgICBlbWl0KHsiZXJyb3IiOiAiYmFkIGpzb24ifSkKICAgICAgICAgICAgY29udGludWUKICAgICAgICBtaWQgPSBtc2cuZ2V0KCJpZCIpCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXN1bHQgPSBhd2FpdCBhZ2VudC5kaXNwYXRjaChtc2cpCiAgICAgICAgICAgIGVtaXQoeyJpZCI6IG1pZCwgIm9rIjogVHJ1ZSwgKipyZXN1bHR9KQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZTogICMgbm9xYTogQkxFMDAxCiAgICAgICAgICAgIHRleHQgPSBmInt0eXBlKGUpLl9fbmFtZV9ffToge2V9IgogICAgICAgICAgICBlbWl0KHsiaWQiOiBtaWQsICJlcnJvciI6IHRleHQsICJnYXRlZF85MDIxIjogX2lzXzkwMjEodGV4dCl9KQoKICAgIGF3YWl0IGFnZW50LmNsb3NlKCkKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgYXN5bmNpby5ydW4obWFpbigpKQo="; + +/** Decode the embedded agent program to its Python source. */ +export function coreDeviceAgentScript(): string { + return Buffer.from(AGENT_SCRIPT_B64, "base64").toString("utf8"); +} + +/** + * Materialize the agent program to a stable temp path (content-hashed, so a new + * build's script lands at a new path and an old one is never re-run). Written + * once; concurrent callers race harmlessly on an identical payload. + */ +export async function materializeAgentScript(): Promise { + const script = coreDeviceAgentScript(); + const hash = createHash("sha256").update(script).digest("hex").slice(0, 16); + const path = join(tmpdir(), `argent-coredevice-agent-${hash}.py`); + if (!existsSync(path)) { + await writeFile(path, script, { mode: 0o600 }); + } + return path; +} + +/** + * Resolve the Python interpreter that has pymobiledevice3 importable. The + * `pymobiledevice3` CLI is a thin launcher whose shebang points at its venv + * interpreter (pipx/venv installs), so read the shebang. Falls back to a + * sibling `python`/`python3` in the same bin dir. + */ +export async function resolvePmd3Python(pmd3CliPath: string): Promise { + try { + const head = (await readFile(pmd3CliPath, "utf8")).slice(0, 256); + const firstLine = head.split("\n", 1)[0] ?? ""; + if (firstLine.startsWith("#!")) { + const interp = firstLine.slice(2).trim().split(/\s+/)[0]; + if (interp && interp.startsWith("/") && existsSync(interp)) return interp; + } + } catch { + // not a readable script (e.g. a compiled shim) — fall through to siblings + } + const binDir = dirname(pmd3CliPath); + for (const name of ["python3", "python"]) { + const p = join(binDir, name); + if (existsSync(p)) return p; + } + throw new Error( + `could not resolve the pymobiledevice3 Python interpreter from "${pmd3CliPath}" ` + + `(its shebang and sibling python were both unusable)` + ); +} + +/** A response line from the agent, keyed by the request `id` we sent. */ +interface AgentResponse { + id?: number; + ok?: boolean; + error?: string; + gated_9021?: boolean; + [k: string]: unknown; +} + +/** Raised when the agent replies with an `error` (carries the 9021 hint). */ +export class CoreDeviceAgentError extends Error { + readonly gated9021: boolean; + constructor(message: string, gated9021: boolean) { + super(message); + this.name = "CoreDeviceAgentError"; + this.gated9021 = gated9021; + } +} + +/** + * One long-lived pymobiledevice3 process per physical iPhone. Connects the RSD + * tunnel and opens the HID/screenshot services once, then serves JSON commands + * on stdio — so each tap/screenshot/button costs a socket write, not a fresh + * ~0.8s Python cold-start (of which ~0.5s is just `import pymobiledevice3`). + * + * Requests are id-correlated; the caller may have several in flight, though the + * device serializes them anyway. + */ +export class CoreDeviceAgent { + private proc: ChildProcessWithoutNullStreams | null = null; + private rl: Interface | null = null; + private nextId = 1; + private readonly pending = new Map< + number, + { resolve: (r: AgentResponse) => void; reject: (e: Error) => void } + >(); + private startError: Error | null = null; + private exited = false; + + constructor( + private readonly python: string, + private readonly scriptPath: string, + private readonly udid: string, + private readonly tunneldPort: number, + private readonly startTimeoutMs = 30_000 + ) {} + + /** Spawn the agent and wait for its `{"ready":true}` handshake line. */ + async start(): Promise { + const proc = spawn(this.python, [this.scriptPath, this.udid, String(this.tunneldPort)], { + stdio: ["pipe", "pipe", "pipe"], + }); + this.proc = proc; + + let stderr = ""; + proc.stderr.on("data", (d: Buffer) => { + // pymobiledevice3 logs to stderr; keep a bounded tail for diagnostics. + stderr = (stderr + d.toString()).slice(-4000); + }); + + const rl = createInterface({ input: proc.stdout }); + this.rl = rl; + + const ready = new Promise((resolve, reject) => { + // `onLine` and `timer` reference each other; `timer` is only read from + // inside `onLine`, which never runs before `timer` is assigned below. + const onLine = (line: string): void => { + const trimmed = line.trim(); + if (!trimmed) return; + let msg: AgentResponse; + try { + msg = JSON.parse(trimmed) as AgentResponse; + } catch { + return; // ignore any non-JSON noise before the handshake + } + if ("ready" in msg) { + clearTimeout(timer); + rl.off("line", onLine); + if (msg.ready) { + rl.on("line", (l) => this.onResponse(l)); + resolve(); + } else { + reject(new Error(String(msg.error ?? "agent failed to connect"))); + } + } + }; + const timer = setTimeout(() => { + rl.off("line", onLine); + reject( + new Error( + `CoreDevice agent did not become ready within ${this.startTimeoutMs}ms` + + (stderr ? `; last stderr: ${stderr.slice(-300)}` : "") + ) + ); + }, this.startTimeoutMs); + rl.on("line", onLine); + }); + + proc.on("exit", (code, signal) => { + this.exited = true; + const err = new Error( + `CoreDevice agent exited (code=${code}, signal=${signal})` + + (stderr ? `; stderr: ${stderr.slice(-300)}` : "") + ); + this.startError ??= err; + for (const { reject } of this.pending.values()) reject(err); + this.pending.clear(); + }); + + try { + await ready; + } catch (err) { + this.dispose(); + throw err instanceof Error ? err : new Error(String(err)); + } + } + + private onResponse(line: string): void { + const trimmed = line.trim(); + if (!trimmed) return; + let msg: AgentResponse; + try { + msg = JSON.parse(trimmed) as AgentResponse; + } catch { + return; + } + if (typeof msg.id !== "number") return; + const waiter = this.pending.get(msg.id); + if (!waiter) return; + this.pending.delete(msg.id); + if (msg.error) { + waiter.reject(new CoreDeviceAgentError(String(msg.error), Boolean(msg.gated_9021))); + } else { + waiter.resolve(msg); + } + } + + /** Send one op and await its correlated response. */ + request( + op: string, + args: Record = {}, + timeoutMs = 30_000 + ): Promise { + if (this.exited || !this.proc) { + return Promise.reject(this.startError ?? new Error("CoreDevice agent is not running")); + } + const id = this.nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (this.pending.delete(id)) { + reject(new Error(`CoreDevice ${op} timed out after ${timeoutMs}ms`)); + } + }, timeoutMs); + this.pending.set(id, { + resolve: (r) => { + clearTimeout(timer); + resolve(r); + }, + reject: (e) => { + clearTimeout(timer); + reject(e); + }, + }); + this.proc!.stdin.write(JSON.stringify({ id, op, ...args }) + "\n"); + }); + } + + dispose(): void { + this.exited = true; + this.rl?.close(); + this.rl = null; + if (this.proc) { + this.proc.stdin.end(); + this.proc.kill("SIGTERM"); + this.proc = null; + } + for (const { reject } of this.pending.values()) { + reject(new Error("CoreDevice agent disposed")); + } + this.pending.clear(); + } +} diff --git a/packages/tool-server/src/tools/describe/contract.ts b/packages/tool-server/src/tools/describe/contract.ts index cae35e884..f40b326a5 100644 --- a/packages/tool-server/src/tools/describe/contract.ts +++ b/packages/tool-server/src/tools/describe/contract.ts @@ -76,7 +76,8 @@ export type DescribeSource = | "android-devtools" | "cdp-dom" | "vega-automation" - | "tv-focus"; + | "tv-focus" + | "springboard"; // Internal shape produced by the per-platform adapters. The `tree` is consumed // by the formatter in `format-tree.ts` and then dropped before the tool replies diff --git a/packages/tool-server/src/tools/describe/platforms/ios/index.ts b/packages/tool-server/src/tools/describe/platforms/ios/index.ts index 7e51a4af1..ba486ee17 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/index.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/index.ts @@ -2,13 +2,14 @@ import type { DeviceInfo, Registry, ToolDependency } from "@argent/registry"; import { axServiceRef, AXServiceApi } from "../../../../blueprints/ax-service"; import { nativeDevtoolsRef, NativeDevtoolsApi } from "../../../../blueprints/native-devtools"; import { isPhysicalIos } from "../../../../utils/device-info"; -import { UnsupportedOperationError } from "../../../../utils/capability"; +import { coreDeviceRef, type CoreDeviceApi } from "../../../../blueprints/core-device"; import { resolveNativeTargetApp } from "../../../../utils/native-target-app"; import { isTvOsSimulator } from "../../../../utils/ios-devices"; import { parseNativeDescribeScreenResult } from "../../../native-devtools/native-describe-contract"; import { DescribeTreeData, parseDescribeResult, type DescribeNode } from "../../contract"; import { adaptAXDescribeToDescribeResult } from "./ios-ax-adapter"; import { adaptNativeDescribeToDescribeResult } from "./ios-native-adapter"; +import { adaptSpringboardToDescribeResult } from "./ios-springboard-adapter"; const DEGRADED_HINT = "This simulator was not booted through argent — system dialogs and native modals may not appear. You MUST call boot-device with force=true now to restart the simulator and apply full accessibility settings before continuing."; @@ -26,6 +27,17 @@ const TVOS_HINT = "(up/down/left/right/select/back/menu/home) to move focus, and `keyboard` to type. " + "See the argent-tv-interact skill."; +// Physical iPhones expose no app-free in-app accessibility tree (Apple gates the +// CoreDevice axAuditDaemon to trusted/AppleInternal callers). The one structured +// screen data we CAN read is SpringBoard's home-screen layout — so describe +// returns that, with this hint making its scope and precision explicit. +const PHYSICAL_IOS_SPRINGBOARD_HINT = + "This is the SpringBoard home-screen layout — the only app-free structured screen data on a " + + "physical iPhone — NOT necessarily the current screen. If an app is open, its content is not " + + "here: call screenshot instead. Icon frames are approximate (derived from the home-screen grid, " + + "not exact pixels), so confirm with screenshot before a precise tap. In-app accessibility is not " + + "reachable on physical iOS (Apple gates the CoreDevice accessibility service to trusted/AppleInternal callers)."; + function emptyTree(): DescribeNode { return parseDescribeResult({ role: "AXGroup", @@ -58,36 +70,33 @@ export async function describeIos( params: DescribeIosParams, options: DescribeIosOptions = {} ): Promise { - // Physical iPhones are driven over CoreDevice, and there is no app-free - // describe (accessibility tree) path on a real device today: - // - The two simulator backends can't run against hardware: ax-service - // shells `simctl spawn` and native-devtools injects a dylib via - // `simctl spawn`, both simulator-only. Their blueprint guards throw for - // kind === "device"; without this early return the fallback chain below - // would swallow those throws and return an empty tree + the "reboot the - // simulator" degraded hint — meaningless for hardware with no simulator. - // - The on-device accessibility tree is served by CoreDevice's - // axAuditDaemon, but Apple gates it to trusted/AppleInternal callers - // (hardware-verified on iOS 27, 2026-07): the DTX service - // `com.apple.accessibility.axAuditDaemon.remoteserver.shim.remote` - // requires the `com.apple.mobile.lockdown.remote.trusted` entitlement — - // over the developer (untrusted) CoreDevice tunnel pymobiledevice3 forms, - // the daemon accepts the socket but terminates it on the first request - // (every audit selector, and even the standard DTX capability handshake). - // The RemoteXPC replacement `…axAuditDaemon.remoteAXService` requires the - // `AppleInternal` entitlement, unreachable by any third party. DTX - // transport itself works over the same tunnel (`developer dvt proclist` - // succeeds), so this is Apple's auth wall, not a transport gap. Screenshot - // (CoreDevice `canViewDeviceDisplay`) is the only app-free screen capture. - // Reject explicitly with a clear, 400-mapped error and point at screenshot. + // Physical iPhones are driven over CoreDevice. There is no app-free *in-app* + // accessibility tree on a real device: the on-device tree is served by + // CoreDevice's axAuditDaemon, but Apple gates it to trusted/AppleInternal + // callers (hardware-verified on iOS 27, 2026-07). The DTX service + // `…axAuditDaemon.remoteserver.shim.remote` opens over the developer + // (untrusted) CoreDevice tunnel pymobiledevice3 forms, but the daemon + // terminates the connection on the first message (every audit selector, and + // even the standard DTX capability handshake); its RemoteXPC replacement + // `…axAuditDaemon.remoteAXService` requires the `AppleInternal` entitlement. + // (DTX transport itself works over the same tunnel — `dvt proclist` succeeds — + // so it's Apple's auth wall, not a transport gap.) The two simulator backends + // below can't run against hardware either (they shell `simctl spawn`). + // + // What we CAN read app-free is SpringBoard's home-screen layout, so `describe` + // on a physical device returns the home-screen app grid (icons + dock) via + // CoreDevice's springboardservices. Icon frames are derived from the icon-grid + // geometry and are approximate; the hint tells the agent this is the home + // screen (not necessarily the current app) and to confirm with screenshot. if (isPhysicalIos(device)) { - throw new UnsupportedOperationError( - "describe", - device, - "Apple restricts the physical device's CoreDevice accessibility service to " + - "trusted/AppleInternal callers, so there is no app-free accessibility tree; " + - "use screenshot to inspect the screen" - ); + const ref = coreDeviceRef(device); + const coreDevice = await registry.resolveService(ref.urn, ref.options); + const home = await coreDevice.homescreen(); + return { + tree: adaptSpringboardToDescribeResult(home), + source: "springboard", + hint: PHYSICAL_IOS_SPRINGBOARD_HINT, + }; } // tvOS short-circuit: the focus-engine accessibility tree is served by the diff --git a/packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts b/packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts new file mode 100644 index 000000000..06d3f27d5 --- /dev/null +++ b/packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts @@ -0,0 +1,242 @@ +import type { CoreDeviceHomescreen } from "../../../../blueprints/core-device"; +import { parseDescribeResult, type DescribeNode } from "../../contract"; + +/** + * Adapts a physical iPhone's SpringBoard icon state (from CoreDevice's + * `springboardservices`) into a describe tree for the **home screen**. + * + * This is the only app-free *structured* screen data reachable on a real device + * (in-app accessibility is Apple-gated — see index.ts). It describes what apps + * exist and roughly where their icons sit; the exact pixel positions of icons + * are not exposed by SpringBoard, so frames are computed from the icon-grid + * geometry (`getHomeScreenIconMetrics`) and are APPROXIMATE. The tool's hint + * tells the agent to confirm with `screenshot` before a precise tap. + * + * `iconState` shape (SpringBoard `getIconState`, format 2): + * [ dock:[icon…], page0:[item…], page1:[item…], … ] + * where a leaf icon is `{displayName, bundleIdentifier, …}` and a widget/folder + * is `{iconType, gridSize:{…}, iconLists|elements, …}` (no displayName). + */ + +interface Metrics { + cols: number; + rows: number; +} + +function readMetrics(m: Record): Metrics { + const cols = Math.round(m.homeScreenIconColumns ?? 4); + const rows = Math.round(m.homeScreenIconRows ?? 6); + return { + cols: cols > 0 ? cols : 4, + rows: rows > 0 ? rows : 6, + }; +} + +interface IconRecord { + displayName?: unknown; + bundleIdentifier?: unknown; + displayIdentifier?: unknown; + // A widget's footprint is a *string* size class ("small"/"medium"/"large"), + // not a {columns,rows} pair. + gridSize?: unknown; + [k: string]: unknown; +} + +function str(v: unknown): string | undefined { + return typeof v === "string" && v.length > 0 ? v : undefined; +} + +function isLeaf(item: IconRecord): boolean { + return str(item.displayName) !== undefined || str(item.bundleIdentifier) !== undefined; +} + +/** + * Cell footprint (columns × rows) an item occupies. Leaves are 1×1; a widget's + * `gridSize` is an iOS size class — "small" is 2×2, "medium" 4×2, "large" 4×4 + * (an extra-large 4×6 also exists). Anything unrecognised falls back to 1×1. + */ +function span(item: IconRecord): { w: number; h: number } { + switch (str(item.gridSize)) { + case "small": + return { w: 2, h: 2 }; + case "medium": + return { w: 4, h: 2 }; + case "large": + return { w: 4, h: 4 }; + case "extralarge": + return { w: 4, h: 6 }; + default: + return { w: 1, h: 1 }; + } +} + +/** Clamp to the unit interval so an approximate frame never fails validation. */ +function unit(v: number): number { + return Math.max(0, Math.min(1, v)); +} + +// The home-screen content band, calibrated (iPhone 15, 393×852pt) so a tap at a +// frame's centre lands on the icon: the grid starts just below the status bar +// and a row is ~0.115 of the height. SpringBoard doesn't publish exact insets, +// so these are approximate — good enough to hit the right icon on common iPhone +// sizes, which is all `describe` promises here (the hint says confirm with +// screenshot). The dock sits on its own row near the bottom. +const GRID_TOP = 0.085; +const GRID_ROW_PITCH = 0.115; +const GRID_MARGIN_X = 0.06; + +/** + * Normalized frame for a grid block at (col,row) spanning (w,h) cells. A 1×1 + * icon fills ~70%×50% of its cell (centred); a multi-cell widget fills its + * whole block so its bounding box reads true. + */ +function gridFrame( + col: number, + row: number, + w: number, + h: number, + cols: number +): DescribeNode["frame"] { + const cellW = (1 - 2 * GRID_MARGIN_X) / cols; + const cellH = GRID_ROW_PITCH; + const blockX = GRID_MARGIN_X + col * cellW; + const blockY = GRID_TOP + row * cellH; + const blockW = w * cellW; + const blockH = h * cellH; + if (w === 1 && h === 1) { + const fw = blockW * 0.7; + const fh = blockH * 0.5; + return { + x: unit(blockX + (blockW - fw) / 2), + y: unit(blockY + (blockH - fh) / 2), + width: unit(fw), + height: unit(fh), + }; + } + return { x: unit(blockX), y: unit(blockY), width: unit(blockW), height: unit(blockH) }; +} + +/** + * SpringBoard packs a page row-major, first-fit: each item takes the earliest + * top-left cell where its w×h block is free. Track a boolean occupancy grid and + * return each item's placed (col,row) so multi-row widgets don't shove the icons + * after them into the wrong slots. + */ +class GridPacker { + private readonly occ: boolean[]; + constructor( + private readonly cols: number, + private readonly rows: number + ) { + this.occ = new Array(cols * rows).fill(false); + } + private free(col: number, row: number, w: number, h: number): boolean { + if (col + w > this.cols || row + h > this.rows) return false; + for (let r = row; r < row + h; r++) { + for (let c = col; c < col + w; c++) { + if (this.occ[r * this.cols + c]) return false; + } + } + return true; + } + /** Place a w×h block; returns its top-left (col,row), or null if the page is full. */ + place(w: number, h: number): { col: number; row: number } | null { + const cw = Math.min(w, this.cols); + const ch = Math.min(h, this.rows); + for (let row = 0; row + ch <= this.rows; row++) { + for (let col = 0; col + cw <= this.cols; col++) { + if (this.free(col, row, cw, ch)) { + for (let r = row; r < row + ch; r++) { + for (let c = col; c < col + cw; c++) this.occ[r * this.cols + c] = true; + } + return { col, row }; + } + } + } + return null; + } +} + +/** Frame for the i-th of `count` dock slots along the bottom row. */ +function dockFrame(i: number, count: number): DescribeNode["frame"] { + const n = Math.max(count, 1); + const marginX = 0.06; + const slot = (1 - 2 * marginX) / n; + const cx = marginX + (i + 0.5) * slot; + const w = slot * 0.6; + const h = 0.05; + const cy = 0.945; + return { x: unit(cx - w / 2), y: unit(cy - h / 2), width: unit(w), height: unit(h) }; +} + +function iconNode(item: IconRecord, frame: DescribeNode["frame"], role: string): DescribeNode { + const node: DescribeNode = { role, frame, children: [] }; + const label = str(item.displayName); + if (label) node.label = label; + const id = str(item.bundleIdentifier) ?? str(item.displayIdentifier); + if (id) node.identifier = id; + return node; +} + +/** + * Build the home-screen describe tree: the first home page's icons laid out on + * the grid, plus the dock. Later pages aren't on screen, so they're omitted + * (the agent swipes to reach them, then re-describes). + */ +export function adaptSpringboardToDescribeResult(home: CoreDeviceHomescreen): DescribeNode { + const { cols, rows } = readMetrics(home.metrics); + const pages = Array.isArray(home.iconState) ? (home.iconState as IconRecord[][]) : []; + const children: DescribeNode[] = []; + + // iconState[0] is the dock; iconState[1] is the first home page. + const dock = Array.isArray(pages[0]) ? pages[0] : []; + const firstPage = Array.isArray(pages[1]) ? pages[1] : []; + + // First home page — pack items row-major first-fit (SpringBoard's own order), + // so multi-row widgets don't shove the icons after them into the wrong slots. + const packer = new GridPacker(cols, rows); + for (const item of firstPage) { + if (!item || typeof item !== "object") continue; + const { w, h } = span(item); + const pos = packer.place(w, h); + if (!pos) continue; // page is full + const frame = gridFrame(pos.col, pos.row, Math.min(w, cols), Math.min(h, rows), cols); + if (isLeaf(item)) { + children.push(iconNode(item, frame, "AXIcon")); + } else { + // A widget or folder: surface it as a group with any nested leaf icons. + const group = iconNode(item, frame, "AXGroup"); + for (const leaf of collectNested(item)) group.children.push(iconNode(leaf, frame, "AXIcon")); + children.push(group); + } + } + + // Dock along the bottom. + dock.forEach((item, i) => { + if (item && typeof item === "object") { + children.push(iconNode(item, dockFrame(i, dock.length), "AXIcon")); + } + }); + + return parseDescribeResult({ + role: "AXGroup", + frame: { x: 0, y: 0, width: 1, height: 1 }, + children, + }); +} + +/** Pull leaf icons out of a folder/widget's nested `iconLists`. */ +function collectNested(item: IconRecord): IconRecord[] { + const out: IconRecord[] = []; + const visit = (v: unknown): void => { + if (Array.isArray(v)) { + for (const el of v) visit(el); + } else if (v && typeof v === "object") { + const rec = v as IconRecord; + if (isLeaf(rec)) out.push(rec); + else if (Array.isArray(rec.iconLists)) visit(rec.iconLists); + } + }; + if (Array.isArray(item.iconLists)) visit(item.iconLists); + return out; +} diff --git a/packages/tool-server/test/physical-ios-coredevice-argv.test.ts b/packages/tool-server/test/physical-ios-coredevice-argv.test.ts deleted file mode 100644 index 12875af26..000000000 --- a/packages/tool-server/test/physical-ios-coredevice-argv.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Coverage for the actual pymobiledevice3 command line the CoreDevice backend - * shells out, plus the iOS-27 host-input gate translation. The rest of the - * physical-iOS suite mocks the CoreDeviceApi wholesale, so the real argv strings - * (`developer core-device universal-hid-service drag …`, `… hid button … press`, - * `… screen-capture screenshot …`) and the `conciseError` 9021 mapping were - * untested — a pmd3 CLI rename or a typo in those arrays would ship silently. - * - * These tests drive `coreDeviceBlueprint.factory` end to end, mocking only the - * `node:child_process` execFile boundary (so the genuine argv is asserted) and - * the tunneld HTTP lookup (so no real device/tunnel is needed). - */ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import type { DeviceInfo } from "@argent/registry"; - -// Hoisted so the vi.mock factory below can close over the same mock instance. -const { execFileMock } = vi.hoisted(() => ({ execFileMock: vi.fn() })); - -// The flag gate is exercised elsewhere; here it's always on so the factory runs. -vi.mock("@argent/configuration-core", () => ({ isFlagEnabled: () => true })); - -// Override only execFile; keep the rest of child_process real so unrelated -// imports in the graph are unaffected. `promisify(execFile)` (run at module -// load) then wraps this mock with the default callback convention. -vi.mock("node:child_process", async (importActual) => { - const actual = await importActual(); - return { ...actual, execFile: execFileMock }; -}); - -import { coreDeviceBlueprint } from "../src/blueprints/core-device"; - -const PHYSICAL_UDID = "00008120-000E6D0C0ABBA01E"; -const TUNNEL = { address: "fd11:2233::1", port: 54321 }; -const device: DeviceInfo = { id: PHYSICAL_UDID, platform: "ios", kind: "device" }; - -type CommandResult = { stdout?: string; stderr?: string } | { error: unknown }; - -/** - * Build the CoreDevice API with the factory, recording every real interaction - * command's argv. The setup probes (`version`, `mounter auto-mount`) are - * answered with success and not recorded; the per-interaction command (anything - * starting with `developer`) is recorded and answered by `onCommand`. - */ -async function makeApi(onCommand: (file: string, args: string[]) => CommandResult) { - const calls: Array<{ file: string; args: string[] }> = []; - execFileMock.mockImplementation((...all: unknown[]) => { - const cb = all[all.length - 1] as (err: unknown, res?: unknown) => void; - const file = all[0] as string; - const args = all[1] as string[]; - const sub = args[0]; - if (sub === "version" || sub === "mounter") { - cb(null, { stdout: "", stderr: "" }); - return; - } - calls.push({ file, args }); - const r = onCommand(file, args); - if ("error" in r) cb(r.error); - else cb(null, { stdout: r.stdout ?? "", stderr: r.stderr ?? "" }); - }); - const instance = await coreDeviceBlueprint.factory({}, "ignored" as never, { device }); - return { api: instance.api, calls }; -} - -beforeEach(() => { - execFileMock.mockReset(); - // tunneld REST lookup: report this device's RSD endpoint so resolveTunnel - // succeeds without a real tunnel. - vi.stubGlobal( - "fetch", - vi.fn(async () => ({ - ok: true, - json: async () => ({ - [PHYSICAL_UDID]: [{ "tunnel-address": TUNNEL.address, "tunnel-port": TUNNEL.port }], - }), - })) - ); -}); - -afterEach(() => { - vi.unstubAllGlobals(); -}); - -describe("CoreDevice pmd3 argv (the exact command line, untested elsewhere)", () => { - const rsd = ["--rsd", TUNNEL.address, String(TUNNEL.port)]; - - it("tap → `universal-hid-service drag` with a short held dwell move + RSD endpoint", async () => { - const { api, calls } = await makeApi(() => ({ stdout: "" })); - await api.tap(0.5, 0.5); - expect(calls).toHaveLength(1); - // 0.5 → 32768; the dwell move nudges y by +96 (away from the edge) so a - // zero-dwell contact isn't dropped by iOS. - expect(calls[0]!.args).toEqual([ - "developer", - "core-device", - "universal-hid-service", - "drag", - "32768", - "32768", - "32768", - "32864", - "--steps", - "3", - "--duration", - "0.15", - ...rsd, - ]); - }); - - it("button → `hid button press` with RSD endpoint", async () => { - const { api, calls } = await makeApi(() => ({ stdout: "" })); - await api.button("home"); - expect(calls[0]!.args).toEqual([ - "developer", - "core-device", - "hid", - "button", - "home", - "press", - ...rsd, - ]); - }); - - it("screenshot → `screen-capture screenshot ` with RSD endpoint", async () => { - const { api, calls } = await makeApi(() => ({ stdout: "" })); - const { path } = await api.screenshot(); - const args = calls[0]!.args; - expect(args.slice(0, 4)).toEqual(["developer", "core-device", "screen-capture", "screenshot"]); - expect(args[4]).toMatch(/argent-ios-shot-.*\.png$/); - expect(args.slice(5)).toEqual(rsd); - // The capture path returned to the caller is the one passed to pmd3. - expect(path).toBe(args[4]); - }); - - it("swipe → `universal-hid-service drag` from→to with clamped step count + RSD endpoint", async () => { - const { api, calls } = await makeApi(() => ({ stdout: "" })); - await api.swipe(0.2, 0.8, 0.6, 0.4, 300); - expect(calls[0]!.args).toEqual([ - "developer", - "core-device", - "universal-hid-service", - "drag", - "13107", // toHid(0.2) - "52428", // toHid(0.8) - "39321", // toHid(0.6) - "26214", // toHid(0.4) - "--steps", - "19", // round(300/16), capped at 60 - "--duration", - "0.300", - ...rsd, - ]); - }); -}); - -describe("iOS-27 host-input gate (CoreDeviceError 9021) is translated, screenshot unaffected", () => { - const gated = Object.assign(new Error("Command failed: pymobiledevice3 … 9021"), { - stderr: "CoreDeviceError 9021: The operation could not be completed.", - stdout: "", - code: 1, - }); - - it("tap surfaces an actionable iOS-27 message instead of the raw 9021 line", async () => { - const { api } = await makeApi(() => ({ error: gated })); - const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; - expect(e.message).toMatch(/requires iOS 27\+/); - // Hardware-verified: screenshot AND buttons work below iOS 27 — only touch - // (tap/swipe) is actually gated. The message must not overclaim button too. - expect(e.message).toMatch(/Screenshot and hardware buttons work/i); - expect(e.message).toContain("CoreDeviceError 9021"); - // Names the attempted interaction so the user knows what was gated. - expect(e.message).toContain("tap"); - // The raw pmd3 error is preserved as the cause for diagnostics. - expect((e as Error & { cause?: unknown }).cause).toBe(gated); - }); - - it("button is gated identically", async () => { - const { api } = await makeApi(() => ({ error: gated })); - const e = (await api.button("home").catch((x: unknown) => x)) as Error; - expect(e.message).toMatch(/requires iOS 27\+/); - }); - - it("screenshot still works on the same pre-iOS-27 device (screen-capture is not gated)", async () => { - // Same device: host input (hid) is rejected with 9021, but screen-capture - // succeeds — exactly the documented pre-27 behavior. - const { api } = await makeApi((_file, args) => - args[2] === "screen-capture" ? { stdout: "" } : { error: gated } - ); - await expect(api.tap(0.5, 0.5)).rejects.toThrow(/requires iOS 27\+/); - const { path } = await api.screenshot(); - expect(path).toMatch(/argent-ios-shot-.*\.png$/); - }); - - it("does NOT mistranslate a non-9021 failure (the mapping is 9021-specific)", async () => { - const refused = Object.assign(new Error("boom"), { - stderr: "Connection refused by the device", - stdout: "", - code: 1, - }); - const { api } = await makeApi(() => ({ error: refused })); - const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; - expect(e.message).toContain("CoreDevice tap failed:"); - expect(e.message).toContain("Connection refused"); - expect(e.message).not.toMatch(/iOS 27/); - }); - - it("picks the real exception, not a rich-boxed source line, from a stale-tunnel traceback", async () => { - // Real captured shape (a genuinely unreachable/stale RSD endpoint): pymobiledevice3 - // renders its traceback via Python's `rich`, framing SOURCE CODE context lines in a - // box — several of which coincidentally match the "fail"/"not " keyword heuristic - // (e.g. "if rsd is not None:") well before the actual OSError line at the bottom. - const staleTunnel = Object.assign(new Error("Command failed"), { - stderr: [ - "Traceback (most recent call last):", - "╭──────────────── Traceback (most recent call last) ────────────────╮", - "│ /pmd3/remote/core_device_tunnel_service.py:218 in connect │", - "│ │", - "│ 218 │ │ if rsd is not None: │", - "│ 219 │ │ │ raise ConnectionError('rsd is not available') │", - "╰────────────────────────────────────────────────────────────────────╯", - "OSError: [Errno 65] No route to host", - ].join("\n"), - stdout: "", - code: 1, - }); - const { api } = await makeApi(() => ({ error: staleTunnel })); - const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; - expect(e.message).toContain("CoreDevice tap failed:"); - expect(e.message).toContain("OSError: [Errno 65] No route to host"); - expect(e.message).not.toContain("if rsd is not None"); - }); - - it("does NOT mistranslate a standalone 9021 HID coordinate echoed in the argv message", async () => { - // execFile's error `message` echoes the full argv, and a HID coordinate maps - // [0,1]→[0,65535], so a real tap can produce the *standalone* token 9021 - // (toHid(0.13765) === 9021). The gate detector must inspect pmd3's stderr/stdout - // only — never `message` — or this non-gated handshake failure would be - // mislabeled "upgrade to iOS 27". The token here is a bare `9021` (not 29021), - // so this test genuinely fails if the detector ever starts scanning `message`. - const coordErr = Object.assign( - new Error( - "Command failed: pymobiledevice3 developer core-device universal-hid-service " + - "drag --rsd fd00::1 50000 --duration 0.05 9021 32768 9021 40000" - ), - { stderr: "the device handshake failed", stdout: "", code: 1 } - ); - const { api } = await makeApi(() => ({ error: coordErr })); - const e = (await api.tap(0.5, 0.5).catch((x: unknown) => x)) as Error; - expect(e.message).not.toMatch(/iOS 27/); - expect(e.message).toContain("handshake failed"); - }); -}); diff --git a/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts b/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts new file mode 100644 index 000000000..b6a143817 --- /dev/null +++ b/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts @@ -0,0 +1,173 @@ +/** + * Coverage for the physical-iOS (CoreDevice) backend after it moved from a + * per-call `pymobiledevice3` CLI spawn to a persistent stdio sidecar: + * + * - `adaptSpringboardToDescribeResult` — the SpringBoard home-screen → describe + * tree adapter that backs `describe` on a real iPhone (in-app AX is Apple- + * gated). Frames are grid-derived; this pins the layout math (widget spans + * push icons down; dock sits at the bottom; every frame stays in [0,1]). + * - `agentError` — the 9021 (iOS-27 host-input gate) message mapping. + * - `CoreDeviceAgent` — the stdio JSON protocol: ready handshake, id-correlated + * request/response, and error propagation (via a stand-in node process). + */ +import { describe, it, expect } from "vitest"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { FAILURE_CODES, getFailureSignal } from "@argent/registry"; +import { adaptSpringboardToDescribeResult } from "../src/tools/describe/platforms/ios/ios-springboard-adapter"; +import { agentError } from "../src/blueprints/core-device"; +import { CoreDeviceAgent, CoreDeviceAgentError } from "../src/blueprints/coredevice-agent"; + +const METRICS = { + homeScreenIconColumns: 4, + homeScreenIconRows: 6, + homeScreenWidth: 393, + homeScreenHeight: 852, + homeScreenIconDockMaxCount: 4, +}; + +const leaf = (name: string, id: string) => ({ displayName: name, bundleIdentifier: id }); +const widget = (size: string) => ({ iconType: "custom", gridSize: size }); + +// Two 2x2 "small" widgets fill the top two rows; the first app row therefore +// starts at grid row 2. A second page must NOT appear (it's off-screen). +const ICON_STATE = [ + [leaf("Phone", "com.apple.mobilephone"), leaf("Safari", "com.apple.mobilesafari")], // dock + [ + widget("small"), + widget("small"), + leaf("FaceTime", "com.apple.facetime"), + leaf("Maps", "com.apple.Maps"), + ], + [leaf("OffPage", "com.example.offpage")], // page 2, not rendered +]; + +interface Node { + role: string; + frame: { x: number; y: number; width: number; height: number }; + children: Node[]; + label?: string; + identifier?: string; +} +function flatten(n: Node, out: Node[] = []): Node[] { + out.push(n); + for (const c of n.children) flatten(c, out); + return out; +} +const center = (f: Node["frame"]) => ({ x: f.x + f.width / 2, y: f.y + f.height / 2 }); + +describe("adaptSpringboardToDescribeResult", () => { + const tree = adaptSpringboardToDescribeResult({ iconState: ICON_STATE, metrics: METRICS }); + const nodes = flatten(tree as Node); + const byLabel = (l: string) => nodes.find((n) => n.label === l); + + it("returns leaf icons with label + bundle identifier", () => { + const faceTime = byLabel("FaceTime"); + expect(faceTime?.identifier).toBe("com.apple.facetime"); + expect(byLabel("Phone")?.identifier).toBe("com.apple.mobilephone"); + }); + + it("packs the first app below the two 2x2 widgets (widgets occupy the top rows)", () => { + // FaceTime is item 3 on the page but the two small widgets take rows 0-1, + // so it lands on row 2 — visibly below the widgets, not at the very top. + const widgets = nodes.filter((n) => n.role === "AXGroup" && n !== tree); + expect(widgets).toHaveLength(2); + const widgetBottom = Math.max(...widgets.map((w) => w.frame.y + w.frame.height)); + const faceTime = byLabel("FaceTime")!; + expect(center(faceTime.frame).y).toBeGreaterThan(widgetBottom - 0.01); + }); + + it("places the dock along the bottom", () => { + expect(center(byLabel("Phone")!.frame).y).toBeGreaterThan(0.85); + expect(center(byLabel("Safari")!.frame).y).toBeGreaterThan(0.85); + }); + + it("omits off-screen pages (only the first home page + dock)", () => { + expect(byLabel("OffPage")).toBeUndefined(); + }); + + it("keeps every frame within the normalized [0,1] box", () => { + for (const n of nodes) { + const { x, y, width, height } = n.frame; + for (const v of [x, y, width, height]) { + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThanOrEqual(1); + } + expect(x + width).toBeLessThanOrEqual(1.0001); + expect(y + height).toBeLessThanOrEqual(1.0001); + } + }); + + it("does not throw on empty / malformed icon state", () => { + expect(() => adaptSpringboardToDescribeResult({ iconState: [], metrics: {} })).not.toThrow(); + expect(() => + adaptSpringboardToDescribeResult({ iconState: null, metrics: METRICS }) + ).not.toThrow(); + }); +}); + +describe("agentError — iOS-27 host-input gate (CoreDeviceError 9021)", () => { + it("maps a gated agent error to the actionable iOS-27 message", () => { + const e = agentError("tap", new CoreDeviceAgentError("… CoreDeviceError 9021 …", true)); + expect(e.message).toContain("requires iOS 27+"); + expect(getFailureSignal(e)?.error_code).toBe(FAILURE_CODES.CORE_DEVICE_IOS_VERSION_TOO_OLD); + }); + + it("maps a non-gated agent error to a generic command failure", () => { + const e = agentError("swipe", new CoreDeviceAgentError("some other failure", false)); + expect(e.message).toContain("CoreDevice swipe failed"); + expect(e.message).not.toContain("iOS 27"); + expect(getFailureSignal(e)?.error_code).toBe(FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED); + }); + + it("maps a plain Error to a command failure", () => { + const e = agentError("button", new Error("boom")); + expect(getFailureSignal(e)?.error_code).toBe(FAILURE_CODES.CORE_DEVICE_COMMAND_FAILED); + }); +}); + +describe("CoreDeviceAgent — stdio JSON protocol", () => { + // A stand-in for the python agent: emits the ready handshake, echoes ops, and + // returns a gated error for op "fail". Run with node so the test needs no + // device or pymobiledevice3. + const mock = join(tmpdir(), `argent-mock-coredevice-agent-${process.pid}.cjs`); + writeFileSync( + mock, + `const rl = require("readline").createInterface({ input: process.stdin }); +process.stdout.write(JSON.stringify({ ready: true }) + "\\n"); +rl.on("line", (l) => { + const m = JSON.parse(l); + if (m.op === "fail") { + process.stdout.write(JSON.stringify({ id: m.id, error: "CoreDeviceError 9021", gated_9021: true }) + "\\n"); + } else { + process.stdout.write(JSON.stringify({ id: m.id, ok: true, echo: m.op }) + "\\n"); + } +}); +` + ); + + it("handshakes, correlates responses by id, and propagates errors", async () => { + const agent = new CoreDeviceAgent(process.execPath, mock, "UDID", 49151, 5000); + await agent.start(); + try { + const r = await agent.request("ping"); + expect(r.ok).toBe(true); + expect(r.echo).toBe("ping"); + + await expect(agent.request("fail")).rejects.toMatchObject({ + name: "CoreDeviceAgentError", + gated9021: true, + }); + } finally { + agent.dispose(); + } + }); + + it("rejects a request made after dispose", async () => { + const agent = new CoreDeviceAgent(process.execPath, mock, "UDID", 49151, 5000); + await agent.start(); + agent.dispose(); + await expect(agent.request("ping")).rejects.toThrow(); + }); +}); diff --git a/packages/tool-server/test/physical-ios-followups.test.ts b/packages/tool-server/test/physical-ios-followups.test.ts index 9a448e102..fb8fca6d1 100644 --- a/packages/tool-server/test/physical-ios-followups.test.ts +++ b/packages/tool-server/test/physical-ios-followups.test.ts @@ -254,10 +254,28 @@ describe("tools unsupported on physical iOS reject with UnsupportedOperationErro ).rejects.toBeInstanceOf(UnsupportedOperationError); }); - it("describe — rejects instead of returning a misleading empty tree", async () => { - await expect(describeIos({} as never, device, {})).rejects.toBeInstanceOf( - UnsupportedOperationError - ); + // describe is NOT in the unsupported set: on a physical iPhone it returns the + // SpringBoard home-screen layout (the only app-free structured screen data; + // in-app AX is Apple-gated). It resolves the CoreDevice service for a + // homescreen() snapshot, so a stub service stands in for the device here. + it("describe — returns the SpringBoard home-screen tree (source springboard), not a rejection", async () => { + const registry = { + resolveService: async () => ({ + homescreen: async () => ({ + iconState: [ + [{ displayName: "Phone", bundleIdentifier: "com.apple.mobilephone" }], + [{ displayName: "Settings", bundleIdentifier: "com.apple.Preferences" }], + ], + metrics: { homeScreenIconColumns: 4, homeScreenIconRows: 6 }, + }), + }), + }; + const result = await describeIos(registry as never, device, {}); + expect(result.source).toBe("springboard"); + const flat = JSON.stringify(result.tree); + expect(flat).toContain("Settings"); + expect(flat).toContain("com.apple.Preferences"); + expect(result.hint).toContain("screenshot"); }); }); From 57e1ca602b227fe09006ca9dae1cfefe4e99ba34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignacy=20=C5=81=C4=85tka?= Date: Mon, 6 Jul 2026 22:49:52 +0200 Subject: [PATCH 16/16] feat(physical-ios): describe reads the real in-app accessibility tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the SpringBoard home-screen-only describe with the device's live on-screen accessibility tree, read app-free over CoreDevice via the iOS-26+ axAudit service. Works in ANY app and on the home screen (the same VoiceOver walk): labels, values, traits (mapped to roles) and reading order are exact. The unlock is the RSDCheckin handshake iOS 26 added to RSD DTX services. pmd3's DtxServiceProvider opens RSD services with a raw connect and skips RSDCheckin, so iOS 26/27 dropped the accessibility daemon on the first byte (which had read as an "Apple wall"); the sidecar monkeypatches the open to go through start_lockdown_service, which performs the checkin. Verified live on iPhone/iOS 27: describe Settings -> read the "Other…" button's frame -> gesture-tap it -> the Join-Wi-Fi form opened. Home screen lists every app + widget. Frames: Apple exposes no per-element geometry on hardware, so exact rects come only from the accessibility audit (a subset of elements, correlated to the tree by element id); the rest are interpolated from reading-order neighbours - enough to tap a list row, with a hint to confirm precise taps via screenshot. Pixel-exact in-app frames would need an on-device XCUITest runner (code-signing). - sidecar: RSDCheckin fix + an `axtree` op (iter_elements walk + run_audit rects + screen size), replacing `homescreen`. - new ios-coredevice-ax-adapter (caption->role/label parse, rect normalize, frame interpolation); removes the springboard adapter. Source: coredevice-ax. --- README.md | 20 +- .../tool-server/src/blueprints/core-device.ts | 36 ++- .../src/blueprints/coredevice-agent.py | 99 +++++++ .../src/blueprints/coredevice-agent.ts | 2 +- .../src/tools/describe/contract.ts | 2 +- .../src/tools/describe/platforms/ios/index.ts | 59 ++--- .../ios/ios-coredevice-ax-adapter.ts | 123 +++++++++ .../platforms/ios/ios-springboard-adapter.ts | 242 ------------------ .../physical-ios-coredevice-sidecar.test.ts | 99 ++++--- .../test/physical-ios-followups.test.ts | 23 +- 10 files changed, 339 insertions(+), 366 deletions(-) create mode 100644 packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts delete mode 100644 packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts diff --git a/README.md b/README.md index 085303dc2..f8de4c872 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Argent can drive a **physical iPhone** — no app installed on the device — ov CoreDevice "remote control" services (the same path Xcode's device window uses), via [`pymobiledevice3`](https://github.com/doronz88/pymobiledevice3). Supported interactions: `screenshot`, `gesture-tap`, `gesture-swipe`, `button`, `launch-app`, and `describe` (the -home-screen app grid — see the note below). The device shows up in `list-devices` with +live on-screen accessibility tree — see the note below). The device shows up in `list-devices` with `kind: "device"`. Interactions run through one persistent `pymobiledevice3` helper per device (connected once), so a tap/screenshot costs a socket write rather than a fresh Python cold-start. @@ -86,15 +86,15 @@ and leave it running: `sudo pymobiledevice3 remote tunneld`. **Limitations / notes** -- `describe` returns the **SpringBoard home-screen layout** (app icons + dock, via CoreDevice's - `springboardservices`) — the only app-free _structured_ screen data on a physical device. - Icon frames are derived from the home-screen grid geometry, so they're approximate: good - enough to tap the right icon, but confirm with `screenshot` before a precise tap. It always - reflects the home screen, so if an app is open its content is **not** there — use `screenshot`. - There is no app-free _in-app_ accessibility tree: that's served by CoreDevice's `axAuditDaemon`, - which Apple gates to trusted / `AppleInternal` callers (hardware-verified on iOS 27 — the DTX - service drops the connection on the first request over the developer tunnel, and its modern - RemoteXPC replacement needs the `AppleInternal` entitlement). +- `describe` returns the device's **live on-screen accessibility tree** — the frontmost app's + elements (or the home screen), read app-free via the iOS-26+ accessibility-audit service over + CoreDevice. Element labels, values, traits (roles) and reading order are exact. Frames are exact + for the elements the accessibility audit reports and **interpolated** from reading-order + neighbours for the rest (Apple doesn't expose per-element geometry on a physical device), so + they're good enough to tap a row in a vertical list — but confirm with `screenshot` before a + precise tap, especially for controls like toggles. (This needs the RSDCheckin handshake iOS 26 + added; the helper performs it. For pixel-exact in-app frames + taps you'd need an on-device + XCUITest runner, which requires code-signing.) - Not supported yet (return a clear "not supported" error): keyboard/typing, pinch & rotate (multi-touch), `open-url`, `reinstall-app`, `restart-app`, and the native inspection / profiling tools (`native-*`, `native-profiler-*`, `screenshot-diff`). `launch-app` (via diff --git a/packages/tool-server/src/blueprints/core-device.ts b/packages/tool-server/src/blueprints/core-device.ts index b6a81de64..882b52e03 100644 --- a/packages/tool-server/src/blueprints/core-device.ts +++ b/packages/tool-server/src/blueprints/core-device.ts @@ -70,11 +70,19 @@ type CoreDeviceFactoryOptions = Record & { device: DeviceInfo } * running CoreDevice tunnel (`sudo pymobiledevice3 remote tunneld`, which needs * root to create the tunnel interface — every command here then runs unprivileged). */ -export interface CoreDeviceHomescreen { - /** SpringBoard `getIconState` — the home-screen app/folder/widget layout. */ - iconState: unknown; - /** SpringBoard icon-grid geometry (columns/rows/icon size/screen points). */ - metrics: Record; +export interface CoreDeviceAxElement { + /** VoiceOver caption: label + value + traits, e.g. "Wi-Fi, 1, Button, Toggle". */ + caption: string; + /** Per-element identifier (hex) — stable within a single snapshot. */ + id: string; + /** On-screen rect "{{x, y}, {w, h}}" in points, when the audit reported one. */ + rect?: string; +} + +export interface CoreDeviceAxTree { + elements: CoreDeviceAxElement[]; + /** Screen size in points, for normalizing rects to 0..1. */ + screen?: { w?: number; h?: number }; } export interface CoreDeviceApi { @@ -87,11 +95,12 @@ export interface CoreDeviceApi { /** Press a hardware button by its pymobiledevice3 name (home/lock/volume-up/volume-down/...). */ button(name: string): Promise; /** - * The SpringBoard home-screen layout — the only app-free *structured* screen - * data reachable on a physical iPhone (in-app accessibility is Apple-gated; - * see describe/platforms/ios). Backs `describe` for the home screen. + * The on-screen accessibility tree of the frontmost app (or the home screen), + * via the iOS-26+ axAudit service — rich semantic captions + reading order for + * every element, plus on-screen rects for the subset the accessibility audit + * flags (Apple doesn't expose per-element geometry on hardware). Backs `describe`. */ - homescreen(): Promise; + axtree(): Promise; } export function coreDeviceRef(device: DeviceInfo): { @@ -587,11 +596,12 @@ export const coreDeviceBlueprint: ServiceBlueprint = async button(name) { await call("button", "button", { name }, 15_000); }, - async homescreen() { - const res = await call("homescreen", "homescreen", {}, 20_000); + async axtree() { + // The audit walk + rect pass runs on-device; give it room. + const res = await call("axtree", "axtree", {}, 45_000); return { - iconState: res.icon_state, - metrics: (res.metrics as Record) ?? {}, + elements: (res.elements as CoreDeviceAxTree["elements"]) ?? [], + screen: res.screen as CoreDeviceAxTree["screen"], }; }, }; diff --git a/packages/tool-server/src/blueprints/coredevice-agent.py b/packages/tool-server/src/blueprints/coredevice-agent.py index 7e1bacfca..974e15b11 100644 --- a/packages/tool-server/src/blueprints/coredevice-agent.py +++ b/packages/tool-server/src/blueprints/coredevice-agent.py @@ -36,10 +36,33 @@ from pymobiledevice3.remote.core_device.screen_capture_service import ScreenCaptureService from pymobiledevice3.services.springboard import SpringBoardServicesService from pymobiledevice3.cli.developer.core_device import _do_drag, _send_button_press, _NAMED_BUTTONS +from pymobiledevice3.dtx_service_provider import DtxServiceProvider +from pymobiledevice3.dtx.connection import DTXConnection +from pymobiledevice3.services.accessibilityaudit import AccessibilityAudit, deserialize_object import urllib.request +# --- RSDCheckin fix for the iOS-26+ accessibility (axAudit) DTX service -------- +# In iOS 26 Apple re-plumbed axauditd onto RemoteServiceDiscovery; the +# `…axAuditDaemon.remoteserver.shim.remote` DTX daemon now requires the RSDCheckin +# handshake before it accepts DTX framing. pymobiledevice3's DtxServiceProvider +# opens RSD services with a raw connect (create_service_connection) and skips +# RSDCheckin, so iOS 26/27 drops the connection on the first byte. start_lockdown_service +# performs RSDCheckin — route RSD DTX opens through it. (ref: littledivy/iphone-mirroring-axdump) +_orig_open_dtx = DtxServiceProvider._open_dtx_connection + + +async def _open_dtx_with_checkin(self, service_name, *, strip_ssl=False): + if isinstance(self.lockdown, RemoteServiceDiscoveryService): + svc = await self.lockdown.start_lockdown_service(service_name) + return DTXConnection(svc.reader, svc.writer) + return await _orig_open_dtx(self, service_name, strip_ssl=strip_ssl) + + +DtxServiceProvider._open_dtx_connection = _open_dtx_with_checkin + + def _resolve_rsd(udid: str, port: int): payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=4)) entry = payload.get(udid) or [] @@ -116,6 +139,82 @@ async def op_homescreen(self, _): metrics = await _maybe_await(sb.get_homescreen_icon_metrics()) return {"icon_state": icons, "metrics": metrics} + async def op_axtree(self, msg): + """The on-screen accessibility tree of whatever app is frontmost (or the + home screen), via the iOS-26+ axAudit service (RSDCheckin-unlocked above). + + Returns each element's accessible caption (label + value + traits) in + VoiceOver reading order plus a stable element id, and — where the + accessibility audit reports one — its on-screen rect in points. Per-element + geometry isn't exposed on hardware, so only audited elements carry a rect; + the TS layer fills the gaps. Screen size (points) comes from SpringBoard. + """ + limit = int(msg.get("limit", 120)) + + # 1) Walk the elements (caption + reading order + element id). + walk = AccessibilityAudit(self.rsd) + elements = [] + try: + async for el in walk.iter_elements(): + elements.append( + {"caption": el.caption, "id": el.element.identifier.hex()} + ) + if len(elements) >= limit: + break + finally: + with contextlib.suppress(Exception): + await walk.close() + + # 2) Audit for rects, keyed by element id (the only hardware frame source). + rects = {} + audit = AccessibilityAudit(self.rsd) + try: + types = await audit.supported_audits_types() + await audit._ensure_ready() + await audit._invoke("deviceBeginAuditTypes:", types, expects_reply=False) + while True: + name, args = await audit._event_queue.get() + if name != "hostDeviceDidCompleteAuditCategoriesWithAuditIssues:": + continue + issues = deserialize_object(audit._extract_event_payload(args)) + # iOS 27 returns the AXAuditIssue list directly; older wrap it in [{"value":…}]. + if ( + isinstance(issues, list) + and issues + and isinstance(issues[0], dict) + and "value" in issues[0] + ): + issues = issues[0]["value"] + for iss in issues or []: + try: + eid = iss._fields["AuditElementValue_v1"]._fields[ + "PlatformElementValue_v1" + ].hex() + rect = iss._fields.get("ElementRectValue_v1") + if eid and rect and eid not in rects: + rects[eid] = rect + except Exception: + continue + break + except Exception: + pass # audit is best-effort; the tree still returns without rects + finally: + with contextlib.suppress(Exception): + await audit.close() + + for el in elements: + if el["id"] in rects: + el["rect"] = rects[el["id"]] + + # 3) Screen size in points, for normalizing the rects. + screen = None + with contextlib.suppress(Exception): + sb = SpringBoardServicesService(lockdown=self.rsd) + m = await _maybe_await(sb.get_homescreen_icon_metrics()) + screen = {"w": m.get("homeScreenWidth"), "h": m.get("homeScreenHeight")} + + return {"elements": elements, "screen": screen} + async def op_ping(self, _): return {"pong": True} diff --git a/packages/tool-server/src/blueprints/coredevice-agent.ts b/packages/tool-server/src/blueprints/coredevice-agent.ts index 9242296f9..b44260bdd 100644 --- a/packages/tool-server/src/blueprints/coredevice-agent.ts +++ b/packages/tool-server/src/blueprints/coredevice-agent.ts @@ -18,7 +18,7 @@ import { createInterface, type Interface } from "node:readline"; * newline-delimited JSON on stdio (see AGENT_PROTOCOL below). */ const AGENT_SCRIPT_B64 = - "IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwoiIiJQZXJzaXN0ZW50IENvcmVEZXZpY2UgYWdlbnQg4oCUIG9uZSBsb25nLWxpdmVkIHByb2Nlc3MgcGVyIHBoeXNpY2FsIGlQaG9uZS4KClJlcGxhY2VzIHBlci1jYWxsIGBweW1vYmlsZWRldmljZTNgIENMSSBzcGF3bnMgKGVhY2ggfjAuOHMsIH4wLjVzIGp1c3QgdGhlClB5dGhvbiBpbXBvcnQpIHdpdGggYSBzaW5nbGUgcHJvY2VzcyB0aGF0IGNvbm5lY3RzIHRoZSBSU0QgdHVubmVsIG9uY2UsIGhvbGRzCnRoZSB0b3VjaHNjcmVlbiBtZWRpYS1zdHJlYW0gc2Vzc2lvbiArIHNjcmVlbnNob3Qgc2VydmljZSBvcGVuLCBhbmQgZXhlY3V0ZXMKbmV3bGluZS1kZWxpbWl0ZWQgSlNPTiBjb21tYW5kcyBvbiBzdGRpbiwgcmVwbHlpbmcgd2l0aCBvbmUgSlNPTiBsaW5lIGVhY2guCgpSZXVzZXMgcHltb2JpbGVkZXZpY2UzJ3Mgb3duIENMSSBoZWxwZXJzIHNvIGJlaGF2aW91ciBpcyBpZGVudGljYWwgdG8gdGhlCmBkZXZlbG9wZXIgY29yZS1kZXZpY2Ug4oCmYCBjb21tYW5kcyAoZHdlbGwtZHJhZyB0YXAsIG1haW5Ub3VjaHNjcmVlbiByZXBvcnRzLApJbmRpZ28gaGFyZHdhcmUgYnV0dG9ucywgc2NyZWVuLWNhcHR1cmUgUE5HKS4KClByb3RvY29sIChvbmUgSlNPTiBvYmplY3QgcGVyIGxpbmUpOgogIDwtIHsidWRpZCI6ICIuLi4iLCAicG9ydCI6IDQ5MTUxfSAgICAgICAgICAgICAgICAgKGFyZ3YsIG5vdCBzdGRpbikKICAtPiB7InJlYWR5IjogdHJ1ZX0gICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAob3IgeyJyZWFkeSI6IGZhbHNlLCAiZXJyb3IiOiAiLi4uIn0pCiAgPC0geyJpZCI6IDEsICJvcCI6ICJzY3JlZW5zaG90In0KICAtPiB7ImlkIjogMSwgIm9rIjogdHJ1ZSwgImltYWdlX2I2NCI6ICIuLi4ifQogIDwtIHsiaWQiOiAyLCAib3AiOiAidGFwIiwgIngiOiAzMjc2OCwgInkiOiAyMDAwMH0gICAgICAgICAgKHgveSBhbHJlYWR5IDAuLjY1NTM1KQogIDwtIHsiaWQiOiAzLCAib3AiOiAic3dpcGUiLCAieDEiOi4uLCAieTEiOi4uLCAieDIiOi4uLCAieTIiOi4uLCAic3RlcHMiOjE5LCAiZHVyYXRpb24iOjAuM30KICA8LSB7ImlkIjogNCwgIm9wIjogImJ1dHRvbiIsICJuYW1lIjogImhvbWUifQogIDwtIHsiaWQiOiA1LCAib3AiOiAiaG9tZXNjcmVlbiJ9ICAgICAgICAgICAgICAgICAgICAgICAgICAgIChzcHJpbmdib2FyZCBpY29uIGdyaWQpCiAgLT4geyJpZCI6IE4sICJvayI6IHRydWUsIC4uLn0gIHwgIHsiaWQiOiBOLCAiZXJyb3IiOiAiLi4uIiwgImdhdGVkXzkwMjEiOiBib29sfQoiIiIKaW1wb3J0IGFzeW5jaW8KaW1wb3J0IGJhc2U2NAppbXBvcnQgY29udGV4dGxpYgppbXBvcnQganNvbgppbXBvcnQgc3lzCgpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUucmVtb3RlX3NlcnZpY2VfZGlzY292ZXJ5IGltcG9ydCBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2UuaGlkX3NlcnZpY2UgaW1wb3J0ICgKICAgIHRvdWNoX3Nlc3Npb24sCiAgICBJbmRpZ29ISURTZXJ2aWNlLAogICAgRElHSVRJWkVSX1NVUkZBQ0VfTUFJTl9UT1VDSFNDUkVFTiwKKQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2Uuc2NyZWVuX2NhcHR1cmVfc2VydmljZSBpbXBvcnQgU2NyZWVuQ2FwdHVyZVNlcnZpY2UKZnJvbSBweW1vYmlsZWRldmljZTMuc2VydmljZXMuc3ByaW5nYm9hcmQgaW1wb3J0IFNwcmluZ0JvYXJkU2VydmljZXNTZXJ2aWNlCmZyb20gcHltb2JpbGVkZXZpY2UzLmNsaS5kZXZlbG9wZXIuY29yZV9kZXZpY2UgaW1wb3J0IF9kb19kcmFnLCBfc2VuZF9idXR0b25fcHJlc3MsIF9OQU1FRF9CVVRUT05TCgppbXBvcnQgdXJsbGliLnJlcXVlc3QKCgpkZWYgX3Jlc29sdmVfcnNkKHVkaWQ6IHN0ciwgcG9ydDogaW50KToKICAgIHBheWxvYWQgPSBqc29uLmxvYWQodXJsbGliLnJlcXVlc3QudXJsb3BlbihmImh0dHA6Ly8xMjcuMC4wLjE6e3BvcnR9LyIsIHRpbWVvdXQ9NCkpCiAgICBlbnRyeSA9IHBheWxvYWQuZ2V0KHVkaWQpIG9yIFtdCiAgICB0ID0gZW50cnlbMF0gaWYgZW50cnkgZWxzZSB7fQogICAgYWRkciwgdHBvcnQgPSB0LmdldCgidHVubmVsLWFkZHJlc3MiKSwgdC5nZXQoInR1bm5lbC1wb3J0IikKICAgIGlmIG5vdCBhZGRyIG9yIG5vdCB0cG9ydDoKICAgICAgICByYWlzZSBSdW50aW1lRXJyb3IoZiJubyBhY3RpdmUgdHVubmVsIHJlZ2lzdGVyZWQgZm9yIHt1ZGlkfSBvbiB0dW5uZWxkIDp7cG9ydH0iKQogICAgcmV0dXJuIGFkZHIsIGludCh0cG9ydCkKCgphc3luYyBkZWYgX21heWJlX2F3YWl0KHYpOgogICAgcmV0dXJuIGF3YWl0IHYgaWYgYXN5bmNpby5pc2Nvcm91dGluZSh2KSBlbHNlIHYKCgpjbGFzcyBBZ2VudDoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCB1ZGlkOiBzdHIsIHBvcnQ6IGludCk6CiAgICAgICAgc2VsZi51ZGlkID0gdWRpZAogICAgICAgIHNlbGYucG9ydCA9IHBvcnQKICAgICAgICBzZWxmLnJzZCA9IE5vbmUKICAgICAgICBzZWxmLnN0YWNrID0gY29udGV4dGxpYi5Bc3luY0V4aXRTdGFjaygpCiAgICAgICAgc2VsZi50b3VjaCA9IE5vbmUgICMgVW5pdmVyc2FsSElEU2VydmljZVNlcnZpY2UsIGhlbGQgb3BlbiAobWVkaWEgc3RyZWFtIHN0YXlzIHdhcm0pCgogICAgYXN5bmMgZGVmIGNvbm5lY3Qoc2VsZik6CiAgICAgICAgYWRkciwgdHBvcnQgPSBfcmVzb2x2ZV9yc2Qoc2VsZi51ZGlkLCBzZWxmLnBvcnQpCiAgICAgICAgc2VsZi5yc2QgPSBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZSgoYWRkciwgdHBvcnQpKQogICAgICAgIGF3YWl0IHNlbGYucnNkLmNvbm5lY3QoKQoKICAgIGFzeW5jIGRlZiBfZW5zdXJlX3RvdWNoKHNlbGYpOgogICAgICAgICMgTGF6aWx5IG9wZW4gKGFuZCBrZWVwKSB0aGUgdG91Y2hzY3JlZW4gbWVkaWEtc3RyZWFtIHNlc3Npb24g4oCUIHRoZSBhdXRoCiAgICAgICAgIyBnYXRlIGJhY2tib2FyZGQgbmVlZHMgZm9yIGluamVjdGVkIHRvdWNoZXMuIEtlcHQgd2FybSBzbyB0YXBzIGRvbid0IHBheQogICAgICAgICMgdGhlIG1lZGlhLXN0cmVhbSBzdGFydHVwIGVhY2ggdGltZS4KICAgICAgICBpZiBzZWxmLnRvdWNoIGlzIE5vbmU6CiAgICAgICAgICAgIHNlbGYudG91Y2ggPSBhd2FpdCBzZWxmLnN0YWNrLmVudGVyX2FzeW5jX2NvbnRleHQodG91Y2hfc2Vzc2lvbihzZWxmLnJzZCkpCiAgICAgICAgcmV0dXJuIHNlbGYudG91Y2gKCiAgICBhc3luYyBkZWYgb3Bfc2NyZWVuc2hvdChzZWxmLCBfKToKICAgICAgICAjIFNjcmVlbkNhcHR1cmVTZXJ2aWNlIGRlbGl2ZXJzIG9uZSBQTkcgcGVyIG9wZW4gKHRoZSBzdHJlYW0gZW5kcyBhZnRlcgogICAgICAgICMgdGhlIGZyYW1lKSwgc28gb3BlbiBhIGZyZXNoIG9uZSBlYWNoIGNhbGwg4oCUIGNoZWFwIG5vdyB0aGUgaW50ZXJwcmV0ZXIKICAgICAgICAjIGFuZCB0dW5uZWwgYXJlIGFscmVhZHkgd2FybS4KICAgICAgICBhc3luYyB3aXRoIFNjcmVlbkNhcHR1cmVTZXJ2aWNlKHNlbGYucnNkKSBhcyBzY3JlZW46CiAgICAgICAgICAgIHJlc3AgPSBhd2FpdCBzY3JlZW4uY2FwdHVyZV9zY3JlZW5zaG90KCkKICAgICAgICByZXR1cm4geyJpbWFnZV9iNjQiOiBiYXNlNjQuYjY0ZW5jb2RlKHJlc3BbImltYWdlIl0pLmRlY29kZSgiYXNjaWkiKX0KCiAgICBhc3luYyBkZWYgb3BfdGFwKHNlbGYsIG1zZyk6CiAgICAgICAgc3ZjID0gYXdhaXQgc2VsZi5fZW5zdXJlX3RvdWNoKCkKICAgICAgICB4LCB5ID0gaW50KG1zZ1sieCJdKSwgaW50KG1zZ1sieSJdKQogICAgICAgICMgWmVyby1kd2VsbCB0YXBzIGFyZSBkcm9wcGVkIGJ5IGlPUzsgZW1pdCBhIHNob3J0IGhlbGQgZHJhZyB3aXRoIGEgdGlueQogICAgICAgICMgbW92ZSBhd2F5IGZyb20gdGhlIGVkZ2UgKG1pcnJvcnMgY29yZS1kZXZpY2UudHMgLyB0aGUgQ0xJIGRyYWcgcGF0aCkuCiAgICAgICAgeTIgPSB5ICsgOTYgaWYgeSA8PSA2NTUzNSAtIDEyMCBlbHNlIHkgLSA5NgogICAgICAgIGF3YWl0IF9kb19kcmFnKHN2YywgeCwgeSwgeCwgeTIsIDMsIDAuMTUsIHRzaWQ9RElHSVRJWkVSX1NVUkZBQ0VfTUFJTl9UT1VDSFNDUkVFTikKICAgICAgICByZXR1cm4ge30KCiAgICBhc3luYyBkZWYgb3Bfc3dpcGUoc2VsZiwgbXNnKToKICAgICAgICBzdmMgPSBhd2FpdCBzZWxmLl9lbnN1cmVfdG91Y2goKQogICAgICAgIGF3YWl0IF9kb19kcmFnKAogICAgICAgICAgICBzdmMsIGludChtc2dbIngxIl0pLCBpbnQobXNnWyJ5MSJdKSwgaW50KG1zZ1sieDIiXSksIGludChtc2dbInkyIl0pLAogICAgICAgICAgICBpbnQobXNnLmdldCgic3RlcHMiLCAxOSkpLCBmbG9hdChtc2cuZ2V0KCJkdXJhdGlvbiIsIDAuMykpLAogICAgICAgICAgICB0c2lkPURJR0lUSVpFUl9TVVJGQUNFX01BSU5fVE9VQ0hTQ1JFRU4sCiAgICAgICAgKQogICAgICAgIHJldHVybiB7fQoKICAgIGFzeW5jIGRlZiBvcF9idXR0b24oc2VsZiwgbXNnKToKICAgICAgICBuYW1lID0gbXNnWyJuYW1lIl0KICAgICAgICBpZiBuYW1lIG5vdCBpbiBfTkFNRURfQlVUVE9OUzoKICAgICAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKGYidW5rbm93biBidXR0b24gJ3tuYW1lfSciKQogICAgICAgIHVzYWdlX3BhZ2UsIHVzYWdlX2NvZGUsIGhvbGQgPSBfTkFNRURfQlVUVE9OU1tuYW1lXQogICAgICAgIGFzeW5jIHdpdGggSW5kaWdvSElEU2VydmljZShzZWxmLnJzZCkgYXMgc3ZjOgogICAgICAgICAgICBhd2FpdCBfc2VuZF9idXR0b25fcHJlc3Moc3ZjLCB1c2FnZV9wYWdlLCB1c2FnZV9jb2RlLCAicHJlc3MiLCBob2xkKQogICAgICAgIHJldHVybiB7fQoKICAgIGFzeW5jIGRlZiBvcF9ob21lc2NyZWVuKHNlbGYsIF8pOgogICAgICAgIHNiID0gU3ByaW5nQm9hcmRTZXJ2aWNlc1NlcnZpY2UobG9ja2Rvd249c2VsZi5yc2QpCiAgICAgICAgaWNvbnMgPSBhd2FpdCBfbWF5YmVfYXdhaXQoc2IuZ2V0X2ljb25fc3RhdGUoKSkKICAgICAgICBtZXRyaWNzID0gYXdhaXQgX21heWJlX2F3YWl0KHNiLmdldF9ob21lc2NyZWVuX2ljb25fbWV0cmljcygpKQogICAgICAgIHJldHVybiB7Imljb25fc3RhdGUiOiBpY29ucywgIm1ldHJpY3MiOiBtZXRyaWNzfQoKICAgIGFzeW5jIGRlZiBvcF9waW5nKHNlbGYsIF8pOgogICAgICAgIHJldHVybiB7InBvbmciOiBUcnVlfQoKICAgIGFzeW5jIGRlZiBkaXNwYXRjaChzZWxmLCBtc2cpOgogICAgICAgIG9wID0gbXNnLmdldCgib3AiKQogICAgICAgIGZuID0gZ2V0YXR0cihzZWxmLCBmIm9wX3tvcH0iLCBOb25lKQogICAgICAgIGlmIGZuIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcihmInVua25vd24gb3AgJ3tvcH0nIikKICAgICAgICByZXR1cm4gYXdhaXQgZm4obXNnKQoKICAgIGFzeW5jIGRlZiBjbG9zZShzZWxmKToKICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgYXdhaXQgc2VsZi5zdGFjay5hY2xvc2UoKQogICAgICAgIGlmIHNlbGYucnNkIGlzIG5vdCBOb25lOgogICAgICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgICAgIGF3YWl0IHNlbGYucnNkLmNsb3NlKCkKCgpkZWYgX2lzXzkwMjEodGV4dDogc3RyKSAtPiBib29sOgogICAgaW1wb3J0IHJlCiAgICByZXR1cm4gYm9vbChyZS5zZWFyY2gociJjb3JlXHMqZGV2aWNlXHMqZXJyb3JcVyo5MDIxIiwgdGV4dCwgcmUuSSkgb3IgcmUuc2VhcmNoKHIiXGI5MDIxXGIiLCB0ZXh0KSkKCgphc3luYyBkZWYgbWFpbigpOgogICAgdWRpZCA9IHN5cy5hcmd2WzFdCiAgICBwb3J0ID0gaW50KHN5cy5hcmd2WzJdKSBpZiBsZW4oc3lzLmFyZ3YpID4gMiBlbHNlIDQ5MTUxCiAgICBhZ2VudCA9IEFnZW50KHVkaWQsIHBvcnQpCiAgICBvdXQgPSBzeXMuc3Rkb3V0CgogICAgZGVmIGVtaXQob2JqKToKICAgICAgICAjIGRlZmF1bHQ9c3RyIGtlZXBzIGEgc3RyYXkgcGxpc3QgdHlwZSAoYnl0ZXMvZGF0ZXRpbWUgaW4gc3ByaW5nYm9hcmQKICAgICAgICAjIGljb24gc3RhdGUpIGZyb20gY3Jhc2hpbmcgc2VyaWFsaXphdGlvbiBtaWQtc2Vzc2lvbi4KICAgICAgICBvdXQud3JpdGUoanNvbi5kdW1wcyhvYmosIGRlZmF1bHQ9c3RyKSArICJcbiIpCiAgICAgICAgb3V0LmZsdXNoKCkKCiAgICB0cnk6CiAgICAgICAgYXdhaXQgYWdlbnQuY29ubmVjdCgpCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6ICAjIG5vcWE6IEJMRTAwMQogICAgICAgIGVtaXQoeyJyZWFkeSI6IEZhbHNlLCAiZXJyb3IiOiBzdHIoZSl9KQogICAgICAgIHJldHVybgogICAgZW1pdCh7InJlYWR5IjogVHJ1ZX0pCgogICAgbG9vcCA9IGFzeW5jaW8uZ2V0X2V2ZW50X2xvb3AoKQogICAgcmVhZGVyID0gYXN5bmNpby5TdHJlYW1SZWFkZXIoKQogICAgYXdhaXQgbG9vcC5jb25uZWN0X3JlYWRfcGlwZShsYW1iZGE6IGFzeW5jaW8uU3RyZWFtUmVhZGVyUHJvdG9jb2wocmVhZGVyKSwgc3lzLnN0ZGluKQoKICAgIHdoaWxlIFRydWU6CiAgICAgICAgbGluZSA9IGF3YWl0IHJlYWRlci5yZWFkbGluZSgpCiAgICAgICAgaWYgbm90IGxpbmU6CiAgICAgICAgICAgIGJyZWFrCiAgICAgICAgbGluZSA9IGxpbmUuc3RyaXAoKQogICAgICAgIGlmIG5vdCBsaW5lOgogICAgICAgICAgICBjb250aW51ZQogICAgICAgIHRyeToKICAgICAgICAgICAgbXNnID0ganNvbi5sb2FkcyhsaW5lKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246ICAjIG5vcWE6IEJMRTAwMQogICAgICAgICAgICBlbWl0KHsiZXJyb3IiOiAiYmFkIGpzb24ifSkKICAgICAgICAgICAgY29udGludWUKICAgICAgICBtaWQgPSBtc2cuZ2V0KCJpZCIpCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXN1bHQgPSBhd2FpdCBhZ2VudC5kaXNwYXRjaChtc2cpCiAgICAgICAgICAgIGVtaXQoeyJpZCI6IG1pZCwgIm9rIjogVHJ1ZSwgKipyZXN1bHR9KQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZTogICMgbm9xYTogQkxFMDAxCiAgICAgICAgICAgIHRleHQgPSBmInt0eXBlKGUpLl9fbmFtZV9ffToge2V9IgogICAgICAgICAgICBlbWl0KHsiaWQiOiBtaWQsICJlcnJvciI6IHRleHQsICJnYXRlZF85MDIxIjogX2lzXzkwMjEodGV4dCl9KQoKICAgIGF3YWl0IGFnZW50LmNsb3NlKCkKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgYXN5bmNpby5ydW4obWFpbigpKQo="; + "IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwoiIiJQZXJzaXN0ZW50IENvcmVEZXZpY2UgYWdlbnQg4oCUIG9uZSBsb25nLWxpdmVkIHByb2Nlc3MgcGVyIHBoeXNpY2FsIGlQaG9uZS4KClJlcGxhY2VzIHBlci1jYWxsIGBweW1vYmlsZWRldmljZTNgIENMSSBzcGF3bnMgKGVhY2ggfjAuOHMsIH4wLjVzIGp1c3QgdGhlClB5dGhvbiBpbXBvcnQpIHdpdGggYSBzaW5nbGUgcHJvY2VzcyB0aGF0IGNvbm5lY3RzIHRoZSBSU0QgdHVubmVsIG9uY2UsIGhvbGRzCnRoZSB0b3VjaHNjcmVlbiBtZWRpYS1zdHJlYW0gc2Vzc2lvbiArIHNjcmVlbnNob3Qgc2VydmljZSBvcGVuLCBhbmQgZXhlY3V0ZXMKbmV3bGluZS1kZWxpbWl0ZWQgSlNPTiBjb21tYW5kcyBvbiBzdGRpbiwgcmVwbHlpbmcgd2l0aCBvbmUgSlNPTiBsaW5lIGVhY2guCgpSZXVzZXMgcHltb2JpbGVkZXZpY2UzJ3Mgb3duIENMSSBoZWxwZXJzIHNvIGJlaGF2aW91ciBpcyBpZGVudGljYWwgdG8gdGhlCmBkZXZlbG9wZXIgY29yZS1kZXZpY2Ug4oCmYCBjb21tYW5kcyAoZHdlbGwtZHJhZyB0YXAsIG1haW5Ub3VjaHNjcmVlbiByZXBvcnRzLApJbmRpZ28gaGFyZHdhcmUgYnV0dG9ucywgc2NyZWVuLWNhcHR1cmUgUE5HKS4KClByb3RvY29sIChvbmUgSlNPTiBvYmplY3QgcGVyIGxpbmUpOgogIDwtIHsidWRpZCI6ICIuLi4iLCAicG9ydCI6IDQ5MTUxfSAgICAgICAgICAgICAgICAgKGFyZ3YsIG5vdCBzdGRpbikKICAtPiB7InJlYWR5IjogdHJ1ZX0gICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAob3IgeyJyZWFkeSI6IGZhbHNlLCAiZXJyb3IiOiAiLi4uIn0pCiAgPC0geyJpZCI6IDEsICJvcCI6ICJzY3JlZW5zaG90In0KICAtPiB7ImlkIjogMSwgIm9rIjogdHJ1ZSwgImltYWdlX2I2NCI6ICIuLi4ifQogIDwtIHsiaWQiOiAyLCAib3AiOiAidGFwIiwgIngiOiAzMjc2OCwgInkiOiAyMDAwMH0gICAgICAgICAgKHgveSBhbHJlYWR5IDAuLjY1NTM1KQogIDwtIHsiaWQiOiAzLCAib3AiOiAic3dpcGUiLCAieDEiOi4uLCAieTEiOi4uLCAieDIiOi4uLCAieTIiOi4uLCAic3RlcHMiOjE5LCAiZHVyYXRpb24iOjAuM30KICA8LSB7ImlkIjogNCwgIm9wIjogImJ1dHRvbiIsICJuYW1lIjogImhvbWUifQogIDwtIHsiaWQiOiA1LCAib3AiOiAiaG9tZXNjcmVlbiJ9ICAgICAgICAgICAgICAgICAgICAgICAgICAgIChzcHJpbmdib2FyZCBpY29uIGdyaWQpCiAgLT4geyJpZCI6IE4sICJvayI6IHRydWUsIC4uLn0gIHwgIHsiaWQiOiBOLCAiZXJyb3IiOiAiLi4uIiwgImdhdGVkXzkwMjEiOiBib29sfQoiIiIKaW1wb3J0IGFzeW5jaW8KaW1wb3J0IGJhc2U2NAppbXBvcnQgY29udGV4dGxpYgppbXBvcnQganNvbgppbXBvcnQgc3lzCgpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUucmVtb3RlX3NlcnZpY2VfZGlzY292ZXJ5IGltcG9ydCBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2UuaGlkX3NlcnZpY2UgaW1wb3J0ICgKICAgIHRvdWNoX3Nlc3Npb24sCiAgICBJbmRpZ29ISURTZXJ2aWNlLAogICAgRElHSVRJWkVSX1NVUkZBQ0VfTUFJTl9UT1VDSFNDUkVFTiwKKQpmcm9tIHB5bW9iaWxlZGV2aWNlMy5yZW1vdGUuY29yZV9kZXZpY2Uuc2NyZWVuX2NhcHR1cmVfc2VydmljZSBpbXBvcnQgU2NyZWVuQ2FwdHVyZVNlcnZpY2UKZnJvbSBweW1vYmlsZWRldmljZTMuc2VydmljZXMuc3ByaW5nYm9hcmQgaW1wb3J0IFNwcmluZ0JvYXJkU2VydmljZXNTZXJ2aWNlCmZyb20gcHltb2JpbGVkZXZpY2UzLmNsaS5kZXZlbG9wZXIuY29yZV9kZXZpY2UgaW1wb3J0IF9kb19kcmFnLCBfc2VuZF9idXR0b25fcHJlc3MsIF9OQU1FRF9CVVRUT05TCmZyb20gcHltb2JpbGVkZXZpY2UzLmR0eF9zZXJ2aWNlX3Byb3ZpZGVyIGltcG9ydCBEdHhTZXJ2aWNlUHJvdmlkZXIKZnJvbSBweW1vYmlsZWRldmljZTMuZHR4LmNvbm5lY3Rpb24gaW1wb3J0IERUWENvbm5lY3Rpb24KZnJvbSBweW1vYmlsZWRldmljZTMuc2VydmljZXMuYWNjZXNzaWJpbGl0eWF1ZGl0IGltcG9ydCBBY2Nlc3NpYmlsaXR5QXVkaXQsIGRlc2VyaWFsaXplX29iamVjdAoKaW1wb3J0IHVybGxpYi5yZXF1ZXN0CgoKIyAtLS0gUlNEQ2hlY2tpbiBmaXggZm9yIHRoZSBpT1MtMjYrIGFjY2Vzc2liaWxpdHkgKGF4QXVkaXQpIERUWCBzZXJ2aWNlIC0tLS0tLS0tCiMgSW4gaU9TIDI2IEFwcGxlIHJlLXBsdW1iZWQgYXhhdWRpdGQgb250byBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5OyB0aGUKIyBg4oCmYXhBdWRpdERhZW1vbi5yZW1vdGVzZXJ2ZXIuc2hpbS5yZW1vdGVgIERUWCBkYWVtb24gbm93IHJlcXVpcmVzIHRoZSBSU0RDaGVja2luCiMgaGFuZHNoYWtlIGJlZm9yZSBpdCBhY2NlcHRzIERUWCBmcmFtaW5nLiBweW1vYmlsZWRldmljZTMncyBEdHhTZXJ2aWNlUHJvdmlkZXIKIyBvcGVucyBSU0Qgc2VydmljZXMgd2l0aCBhIHJhdyBjb25uZWN0IChjcmVhdGVfc2VydmljZV9jb25uZWN0aW9uKSBhbmQgc2tpcHMKIyBSU0RDaGVja2luLCBzbyBpT1MgMjYvMjcgZHJvcHMgdGhlIGNvbm5lY3Rpb24gb24gdGhlIGZpcnN0IGJ5dGUuIHN0YXJ0X2xvY2tkb3duX3NlcnZpY2UKIyBwZXJmb3JtcyBSU0RDaGVja2luIOKAlCByb3V0ZSBSU0QgRFRYIG9wZW5zIHRocm91Z2ggaXQuIChyZWY6IGxpdHRsZWRpdnkvaXBob25lLW1pcnJvcmluZy1heGR1bXApCl9vcmlnX29wZW5fZHR4ID0gRHR4U2VydmljZVByb3ZpZGVyLl9vcGVuX2R0eF9jb25uZWN0aW9uCgoKYXN5bmMgZGVmIF9vcGVuX2R0eF93aXRoX2NoZWNraW4oc2VsZiwgc2VydmljZV9uYW1lLCAqLCBzdHJpcF9zc2w9RmFsc2UpOgogICAgaWYgaXNpbnN0YW5jZShzZWxmLmxvY2tkb3duLCBSZW1vdGVTZXJ2aWNlRGlzY292ZXJ5U2VydmljZSk6CiAgICAgICAgc3ZjID0gYXdhaXQgc2VsZi5sb2NrZG93bi5zdGFydF9sb2NrZG93bl9zZXJ2aWNlKHNlcnZpY2VfbmFtZSkKICAgICAgICByZXR1cm4gRFRYQ29ubmVjdGlvbihzdmMucmVhZGVyLCBzdmMud3JpdGVyKQogICAgcmV0dXJuIGF3YWl0IF9vcmlnX29wZW5fZHR4KHNlbGYsIHNlcnZpY2VfbmFtZSwgc3RyaXBfc3NsPXN0cmlwX3NzbCkKCgpEdHhTZXJ2aWNlUHJvdmlkZXIuX29wZW5fZHR4X2Nvbm5lY3Rpb24gPSBfb3Blbl9kdHhfd2l0aF9jaGVja2luCgoKZGVmIF9yZXNvbHZlX3JzZCh1ZGlkOiBzdHIsIHBvcnQ6IGludCk6CiAgICBwYXlsb2FkID0ganNvbi5sb2FkKHVybGxpYi5yZXF1ZXN0LnVybG9wZW4oZiJodHRwOi8vMTI3LjAuMC4xOntwb3J0fS8iLCB0aW1lb3V0PTQpKQogICAgZW50cnkgPSBwYXlsb2FkLmdldCh1ZGlkKSBvciBbXQogICAgdCA9IGVudHJ5WzBdIGlmIGVudHJ5IGVsc2Uge30KICAgIGFkZHIsIHRwb3J0ID0gdC5nZXQoInR1bm5lbC1hZGRyZXNzIiksIHQuZ2V0KCJ0dW5uZWwtcG9ydCIpCiAgICBpZiBub3QgYWRkciBvciBub3QgdHBvcnQ6CiAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKGYibm8gYWN0aXZlIHR1bm5lbCByZWdpc3RlcmVkIGZvciB7dWRpZH0gb24gdHVubmVsZCA6e3BvcnR9IikKICAgIHJldHVybiBhZGRyLCBpbnQodHBvcnQpCgoKYXN5bmMgZGVmIF9tYXliZV9hd2FpdCh2KToKICAgIHJldHVybiBhd2FpdCB2IGlmIGFzeW5jaW8uaXNjb3JvdXRpbmUodikgZWxzZSB2CgoKY2xhc3MgQWdlbnQ6CiAgICBkZWYgX19pbml0X18oc2VsZiwgdWRpZDogc3RyLCBwb3J0OiBpbnQpOgogICAgICAgIHNlbGYudWRpZCA9IHVkaWQKICAgICAgICBzZWxmLnBvcnQgPSBwb3J0CiAgICAgICAgc2VsZi5yc2QgPSBOb25lCiAgICAgICAgc2VsZi5zdGFjayA9IGNvbnRleHRsaWIuQXN5bmNFeGl0U3RhY2soKQogICAgICAgIHNlbGYudG91Y2ggPSBOb25lICAjIFVuaXZlcnNhbEhJRFNlcnZpY2VTZXJ2aWNlLCBoZWxkIG9wZW4gKG1lZGlhIHN0cmVhbSBzdGF5cyB3YXJtKQoKICAgIGFzeW5jIGRlZiBjb25uZWN0KHNlbGYpOgogICAgICAgIGFkZHIsIHRwb3J0ID0gX3Jlc29sdmVfcnNkKHNlbGYudWRpZCwgc2VsZi5wb3J0KQogICAgICAgIHNlbGYucnNkID0gUmVtb3RlU2VydmljZURpc2NvdmVyeVNlcnZpY2UoKGFkZHIsIHRwb3J0KSkKICAgICAgICBhd2FpdCBzZWxmLnJzZC5jb25uZWN0KCkKCiAgICBhc3luYyBkZWYgX2Vuc3VyZV90b3VjaChzZWxmKToKICAgICAgICAjIExhemlseSBvcGVuIChhbmQga2VlcCkgdGhlIHRvdWNoc2NyZWVuIG1lZGlhLXN0cmVhbSBzZXNzaW9uIOKAlCB0aGUgYXV0aAogICAgICAgICMgZ2F0ZSBiYWNrYm9hcmRkIG5lZWRzIGZvciBpbmplY3RlZCB0b3VjaGVzLiBLZXB0IHdhcm0gc28gdGFwcyBkb24ndCBwYXkKICAgICAgICAjIHRoZSBtZWRpYS1zdHJlYW0gc3RhcnR1cCBlYWNoIHRpbWUuCiAgICAgICAgaWYgc2VsZi50b3VjaCBpcyBOb25lOgogICAgICAgICAgICBzZWxmLnRvdWNoID0gYXdhaXQgc2VsZi5zdGFjay5lbnRlcl9hc3luY19jb250ZXh0KHRvdWNoX3Nlc3Npb24oc2VsZi5yc2QpKQogICAgICAgIHJldHVybiBzZWxmLnRvdWNoCgogICAgYXN5bmMgZGVmIG9wX3NjcmVlbnNob3Qoc2VsZiwgXyk6CiAgICAgICAgIyBTY3JlZW5DYXB0dXJlU2VydmljZSBkZWxpdmVycyBvbmUgUE5HIHBlciBvcGVuICh0aGUgc3RyZWFtIGVuZHMgYWZ0ZXIKICAgICAgICAjIHRoZSBmcmFtZSksIHNvIG9wZW4gYSBmcmVzaCBvbmUgZWFjaCBjYWxsIOKAlCBjaGVhcCBub3cgdGhlIGludGVycHJldGVyCiAgICAgICAgIyBhbmQgdHVubmVsIGFyZSBhbHJlYWR5IHdhcm0uCiAgICAgICAgYXN5bmMgd2l0aCBTY3JlZW5DYXB0dXJlU2VydmljZShzZWxmLnJzZCkgYXMgc2NyZWVuOgogICAgICAgICAgICByZXNwID0gYXdhaXQgc2NyZWVuLmNhcHR1cmVfc2NyZWVuc2hvdCgpCiAgICAgICAgcmV0dXJuIHsiaW1hZ2VfYjY0IjogYmFzZTY0LmI2NGVuY29kZShyZXNwWyJpbWFnZSJdKS5kZWNvZGUoImFzY2lpIil9CgogICAgYXN5bmMgZGVmIG9wX3RhcChzZWxmLCBtc2cpOgogICAgICAgIHN2YyA9IGF3YWl0IHNlbGYuX2Vuc3VyZV90b3VjaCgpCiAgICAgICAgeCwgeSA9IGludChtc2dbIngiXSksIGludChtc2dbInkiXSkKICAgICAgICAjIFplcm8tZHdlbGwgdGFwcyBhcmUgZHJvcHBlZCBieSBpT1M7IGVtaXQgYSBzaG9ydCBoZWxkIGRyYWcgd2l0aCBhIHRpbnkKICAgICAgICAjIG1vdmUgYXdheSBmcm9tIHRoZSBlZGdlIChtaXJyb3JzIGNvcmUtZGV2aWNlLnRzIC8gdGhlIENMSSBkcmFnIHBhdGgpLgogICAgICAgIHkyID0geSArIDk2IGlmIHkgPD0gNjU1MzUgLSAxMjAgZWxzZSB5IC0gOTYKICAgICAgICBhd2FpdCBfZG9fZHJhZyhzdmMsIHgsIHksIHgsIHkyLCAzLCAwLjE1LCB0c2lkPURJR0lUSVpFUl9TVVJGQUNFX01BSU5fVE9VQ0hTQ1JFRU4pCiAgICAgICAgcmV0dXJuIHt9CgogICAgYXN5bmMgZGVmIG9wX3N3aXBlKHNlbGYsIG1zZyk6CiAgICAgICAgc3ZjID0gYXdhaXQgc2VsZi5fZW5zdXJlX3RvdWNoKCkKICAgICAgICBhd2FpdCBfZG9fZHJhZygKICAgICAgICAgICAgc3ZjLCBpbnQobXNnWyJ4MSJdKSwgaW50KG1zZ1sieTEiXSksIGludChtc2dbIngyIl0pLCBpbnQobXNnWyJ5MiJdKSwKICAgICAgICAgICAgaW50KG1zZy5nZXQoInN0ZXBzIiwgMTkpKSwgZmxvYXQobXNnLmdldCgiZHVyYXRpb24iLCAwLjMpKSwKICAgICAgICAgICAgdHNpZD1ESUdJVElaRVJfU1VSRkFDRV9NQUlOX1RPVUNIU0NSRUVOLAogICAgICAgICkKICAgICAgICByZXR1cm4ge30KCiAgICBhc3luYyBkZWYgb3BfYnV0dG9uKHNlbGYsIG1zZyk6CiAgICAgICAgbmFtZSA9IG1zZ1sibmFtZSJdCiAgICAgICAgaWYgbmFtZSBub3QgaW4gX05BTUVEX0JVVFRPTlM6CiAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcihmInVua25vd24gYnV0dG9uICd7bmFtZX0nIikKICAgICAgICB1c2FnZV9wYWdlLCB1c2FnZV9jb2RlLCBob2xkID0gX05BTUVEX0JVVFRPTlNbbmFtZV0KICAgICAgICBhc3luYyB3aXRoIEluZGlnb0hJRFNlcnZpY2Uoc2VsZi5yc2QpIGFzIHN2YzoKICAgICAgICAgICAgYXdhaXQgX3NlbmRfYnV0dG9uX3ByZXNzKHN2YywgdXNhZ2VfcGFnZSwgdXNhZ2VfY29kZSwgInByZXNzIiwgaG9sZCkKICAgICAgICByZXR1cm4ge30KCiAgICBhc3luYyBkZWYgb3BfaG9tZXNjcmVlbihzZWxmLCBfKToKICAgICAgICBzYiA9IFNwcmluZ0JvYXJkU2VydmljZXNTZXJ2aWNlKGxvY2tkb3duPXNlbGYucnNkKQogICAgICAgIGljb25zID0gYXdhaXQgX21heWJlX2F3YWl0KHNiLmdldF9pY29uX3N0YXRlKCkpCiAgICAgICAgbWV0cmljcyA9IGF3YWl0IF9tYXliZV9hd2FpdChzYi5nZXRfaG9tZXNjcmVlbl9pY29uX21ldHJpY3MoKSkKICAgICAgICByZXR1cm4geyJpY29uX3N0YXRlIjogaWNvbnMsICJtZXRyaWNzIjogbWV0cmljc30KCiAgICBhc3luYyBkZWYgb3BfYXh0cmVlKHNlbGYsIG1zZyk6CiAgICAgICAgIiIiVGhlIG9uLXNjcmVlbiBhY2Nlc3NpYmlsaXR5IHRyZWUgb2Ygd2hhdGV2ZXIgYXBwIGlzIGZyb250bW9zdCAob3IgdGhlCiAgICAgICAgaG9tZSBzY3JlZW4pLCB2aWEgdGhlIGlPUy0yNisgYXhBdWRpdCBzZXJ2aWNlIChSU0RDaGVja2luLXVubG9ja2VkIGFib3ZlKS4KCiAgICAgICAgUmV0dXJucyBlYWNoIGVsZW1lbnQncyBhY2Nlc3NpYmxlIGNhcHRpb24gKGxhYmVsICsgdmFsdWUgKyB0cmFpdHMpIGluCiAgICAgICAgVm9pY2VPdmVyIHJlYWRpbmcgb3JkZXIgcGx1cyBhIHN0YWJsZSBlbGVtZW50IGlkLCBhbmQg4oCUIHdoZXJlIHRoZQogICAgICAgIGFjY2Vzc2liaWxpdHkgYXVkaXQgcmVwb3J0cyBvbmUg4oCUIGl0cyBvbi1zY3JlZW4gcmVjdCBpbiBwb2ludHMuIFBlci1lbGVtZW50CiAgICAgICAgZ2VvbWV0cnkgaXNuJ3QgZXhwb3NlZCBvbiBoYXJkd2FyZSwgc28gb25seSBhdWRpdGVkIGVsZW1lbnRzIGNhcnJ5IGEgcmVjdDsKICAgICAgICB0aGUgVFMgbGF5ZXIgZmlsbHMgdGhlIGdhcHMuIFNjcmVlbiBzaXplIChwb2ludHMpIGNvbWVzIGZyb20gU3ByaW5nQm9hcmQuCiAgICAgICAgIiIiCiAgICAgICAgbGltaXQgPSBpbnQobXNnLmdldCgibGltaXQiLCAxMjApKQoKICAgICAgICAjIDEpIFdhbGsgdGhlIGVsZW1lbnRzIChjYXB0aW9uICsgcmVhZGluZyBvcmRlciArIGVsZW1lbnQgaWQpLgogICAgICAgIHdhbGsgPSBBY2Nlc3NpYmlsaXR5QXVkaXQoc2VsZi5yc2QpCiAgICAgICAgZWxlbWVudHMgPSBbXQogICAgICAgIHRyeToKICAgICAgICAgICAgYXN5bmMgZm9yIGVsIGluIHdhbGsuaXRlcl9lbGVtZW50cygpOgogICAgICAgICAgICAgICAgZWxlbWVudHMuYXBwZW5kKAogICAgICAgICAgICAgICAgICAgIHsiY2FwdGlvbiI6IGVsLmNhcHRpb24sICJpZCI6IGVsLmVsZW1lbnQuaWRlbnRpZmllci5oZXgoKX0KICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGxlbihlbGVtZW50cykgPj0gbGltaXQ6CiAgICAgICAgICAgICAgICAgICAgYnJlYWsKICAgICAgICBmaW5hbGx5OgogICAgICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgICAgIGF3YWl0IHdhbGsuY2xvc2UoKQoKICAgICAgICAjIDIpIEF1ZGl0IGZvciByZWN0cywga2V5ZWQgYnkgZWxlbWVudCBpZCAodGhlIG9ubHkgaGFyZHdhcmUgZnJhbWUgc291cmNlKS4KICAgICAgICByZWN0cyA9IHt9CiAgICAgICAgYXVkaXQgPSBBY2Nlc3NpYmlsaXR5QXVkaXQoc2VsZi5yc2QpCiAgICAgICAgdHJ5OgogICAgICAgICAgICB0eXBlcyA9IGF3YWl0IGF1ZGl0LnN1cHBvcnRlZF9hdWRpdHNfdHlwZXMoKQogICAgICAgICAgICBhd2FpdCBhdWRpdC5fZW5zdXJlX3JlYWR5KCkKICAgICAgICAgICAgYXdhaXQgYXVkaXQuX2ludm9rZSgiZGV2aWNlQmVnaW5BdWRpdFR5cGVzOiIsIHR5cGVzLCBleHBlY3RzX3JlcGx5PUZhbHNlKQogICAgICAgICAgICB3aGlsZSBUcnVlOgogICAgICAgICAgICAgICAgbmFtZSwgYXJncyA9IGF3YWl0IGF1ZGl0Ll9ldmVudF9xdWV1ZS5nZXQoKQogICAgICAgICAgICAgICAgaWYgbmFtZSAhPSAiaG9zdERldmljZURpZENvbXBsZXRlQXVkaXRDYXRlZ29yaWVzV2l0aEF1ZGl0SXNzdWVzOiI6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlzc3VlcyA9IGRlc2VyaWFsaXplX29iamVjdChhdWRpdC5fZXh0cmFjdF9ldmVudF9wYXlsb2FkKGFyZ3MpKQogICAgICAgICAgICAgICAgIyBpT1MgMjcgcmV0dXJucyB0aGUgQVhBdWRpdElzc3VlIGxpc3QgZGlyZWN0bHk7IG9sZGVyIHdyYXAgaXQgaW4gW3sidmFsdWUiOuKApn1dLgogICAgICAgICAgICAgICAgaWYgKAogICAgICAgICAgICAgICAgICAgIGlzaW5zdGFuY2UoaXNzdWVzLCBsaXN0KQogICAgICAgICAgICAgICAgICAgIGFuZCBpc3N1ZXMKICAgICAgICAgICAgICAgICAgICBhbmQgaXNpbnN0YW5jZShpc3N1ZXNbMF0sIGRpY3QpCiAgICAgICAgICAgICAgICAgICAgYW5kICJ2YWx1ZSIgaW4gaXNzdWVzWzBdCiAgICAgICAgICAgICAgICApOgogICAgICAgICAgICAgICAgICAgIGlzc3VlcyA9IGlzc3Vlc1swXVsidmFsdWUiXQogICAgICAgICAgICAgICAgZm9yIGlzcyBpbiBpc3N1ZXMgb3IgW106CiAgICAgICAgICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgICAgICAgICBlaWQgPSBpc3MuX2ZpZWxkc1siQXVkaXRFbGVtZW50VmFsdWVfdjEiXS5fZmllbGRzWwogICAgICAgICAgICAgICAgICAgICAgICAgICAgIlBsYXRmb3JtRWxlbWVudFZhbHVlX3YxIgogICAgICAgICAgICAgICAgICAgICAgICBdLmhleCgpCiAgICAgICAgICAgICAgICAgICAgICAgIHJlY3QgPSBpc3MuX2ZpZWxkcy5nZXQoIkVsZW1lbnRSZWN0VmFsdWVfdjEiKQogICAgICAgICAgICAgICAgICAgICAgICBpZiBlaWQgYW5kIHJlY3QgYW5kIGVpZCBub3QgaW4gcmVjdHM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICByZWN0c1tlaWRdID0gcmVjdAogICAgICAgICAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgICAgICBicmVhawogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246CiAgICAgICAgICAgIHBhc3MgICMgYXVkaXQgaXMgYmVzdC1lZmZvcnQ7IHRoZSB0cmVlIHN0aWxsIHJldHVybnMgd2l0aG91dCByZWN0cwogICAgICAgIGZpbmFsbHk6CiAgICAgICAgICAgIHdpdGggY29udGV4dGxpYi5zdXBwcmVzcyhFeGNlcHRpb24pOgogICAgICAgICAgICAgICAgYXdhaXQgYXVkaXQuY2xvc2UoKQoKICAgICAgICBmb3IgZWwgaW4gZWxlbWVudHM6CiAgICAgICAgICAgIGlmIGVsWyJpZCJdIGluIHJlY3RzOgogICAgICAgICAgICAgICAgZWxbInJlY3QiXSA9IHJlY3RzW2VsWyJpZCJdXQoKICAgICAgICAjIDMpIFNjcmVlbiBzaXplIGluIHBvaW50cywgZm9yIG5vcm1hbGl6aW5nIHRoZSByZWN0cy4KICAgICAgICBzY3JlZW4gPSBOb25lCiAgICAgICAgd2l0aCBjb250ZXh0bGliLnN1cHByZXNzKEV4Y2VwdGlvbik6CiAgICAgICAgICAgIHNiID0gU3ByaW5nQm9hcmRTZXJ2aWNlc1NlcnZpY2UobG9ja2Rvd249c2VsZi5yc2QpCiAgICAgICAgICAgIG0gPSBhd2FpdCBfbWF5YmVfYXdhaXQoc2IuZ2V0X2hvbWVzY3JlZW5faWNvbl9tZXRyaWNzKCkpCiAgICAgICAgICAgIHNjcmVlbiA9IHsidyI6IG0uZ2V0KCJob21lU2NyZWVuV2lkdGgiKSwgImgiOiBtLmdldCgiaG9tZVNjcmVlbkhlaWdodCIpfQoKICAgICAgICByZXR1cm4geyJlbGVtZW50cyI6IGVsZW1lbnRzLCAic2NyZWVuIjogc2NyZWVufQoKICAgIGFzeW5jIGRlZiBvcF9waW5nKHNlbGYsIF8pOgogICAgICAgIHJldHVybiB7InBvbmciOiBUcnVlfQoKICAgIGFzeW5jIGRlZiBkaXNwYXRjaChzZWxmLCBtc2cpOgogICAgICAgIG9wID0gbXNnLmdldCgib3AiKQogICAgICAgIGZuID0gZ2V0YXR0cihzZWxmLCBmIm9wX3tvcH0iLCBOb25lKQogICAgICAgIGlmIGZuIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFJ1bnRpbWVFcnJvcihmInVua25vd24gb3AgJ3tvcH0nIikKICAgICAgICByZXR1cm4gYXdhaXQgZm4obXNnKQoKICAgIGFzeW5jIGRlZiBjbG9zZShzZWxmKToKICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgYXdhaXQgc2VsZi5zdGFjay5hY2xvc2UoKQogICAgICAgIGlmIHNlbGYucnNkIGlzIG5vdCBOb25lOgogICAgICAgICAgICB3aXRoIGNvbnRleHRsaWIuc3VwcHJlc3MoRXhjZXB0aW9uKToKICAgICAgICAgICAgICAgIGF3YWl0IHNlbGYucnNkLmNsb3NlKCkKCgpkZWYgX2lzXzkwMjEodGV4dDogc3RyKSAtPiBib29sOgogICAgaW1wb3J0IHJlCiAgICByZXR1cm4gYm9vbChyZS5zZWFyY2gociJjb3JlXHMqZGV2aWNlXHMqZXJyb3JcVyo5MDIxIiwgdGV4dCwgcmUuSSkgb3IgcmUuc2VhcmNoKHIiXGI5MDIxXGIiLCB0ZXh0KSkKCgphc3luYyBkZWYgbWFpbigpOgogICAgdWRpZCA9IHN5cy5hcmd2WzFdCiAgICBwb3J0ID0gaW50KHN5cy5hcmd2WzJdKSBpZiBsZW4oc3lzLmFyZ3YpID4gMiBlbHNlIDQ5MTUxCiAgICBhZ2VudCA9IEFnZW50KHVkaWQsIHBvcnQpCiAgICBvdXQgPSBzeXMuc3Rkb3V0CgogICAgZGVmIGVtaXQob2JqKToKICAgICAgICAjIGRlZmF1bHQ9c3RyIGtlZXBzIGEgc3RyYXkgcGxpc3QgdHlwZSAoYnl0ZXMvZGF0ZXRpbWUgaW4gc3ByaW5nYm9hcmQKICAgICAgICAjIGljb24gc3RhdGUpIGZyb20gY3Jhc2hpbmcgc2VyaWFsaXphdGlvbiBtaWQtc2Vzc2lvbi4KICAgICAgICBvdXQud3JpdGUoanNvbi5kdW1wcyhvYmosIGRlZmF1bHQ9c3RyKSArICJcbiIpCiAgICAgICAgb3V0LmZsdXNoKCkKCiAgICB0cnk6CiAgICAgICAgYXdhaXQgYWdlbnQuY29ubmVjdCgpCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6ICAjIG5vcWE6IEJMRTAwMQogICAgICAgIGVtaXQoeyJyZWFkeSI6IEZhbHNlLCAiZXJyb3IiOiBzdHIoZSl9KQogICAgICAgIHJldHVybgogICAgZW1pdCh7InJlYWR5IjogVHJ1ZX0pCgogICAgbG9vcCA9IGFzeW5jaW8uZ2V0X2V2ZW50X2xvb3AoKQogICAgcmVhZGVyID0gYXN5bmNpby5TdHJlYW1SZWFkZXIoKQogICAgYXdhaXQgbG9vcC5jb25uZWN0X3JlYWRfcGlwZShsYW1iZGE6IGFzeW5jaW8uU3RyZWFtUmVhZGVyUHJvdG9jb2wocmVhZGVyKSwgc3lzLnN0ZGluKQoKICAgIHdoaWxlIFRydWU6CiAgICAgICAgbGluZSA9IGF3YWl0IHJlYWRlci5yZWFkbGluZSgpCiAgICAgICAgaWYgbm90IGxpbmU6CiAgICAgICAgICAgIGJyZWFrCiAgICAgICAgbGluZSA9IGxpbmUuc3RyaXAoKQogICAgICAgIGlmIG5vdCBsaW5lOgogICAgICAgICAgICBjb250aW51ZQogICAgICAgIHRyeToKICAgICAgICAgICAgbXNnID0ganNvbi5sb2FkcyhsaW5lKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb246ICAjIG5vcWE6IEJMRTAwMQogICAgICAgICAgICBlbWl0KHsiZXJyb3IiOiAiYmFkIGpzb24ifSkKICAgICAgICAgICAgY29udGludWUKICAgICAgICBtaWQgPSBtc2cuZ2V0KCJpZCIpCiAgICAgICAgdHJ5OgogICAgICAgICAgICByZXN1bHQgPSBhd2FpdCBhZ2VudC5kaXNwYXRjaChtc2cpCiAgICAgICAgICAgIGVtaXQoeyJpZCI6IG1pZCwgIm9rIjogVHJ1ZSwgKipyZXN1bHR9KQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZTogICMgbm9xYTogQkxFMDAxCiAgICAgICAgICAgIHRleHQgPSBmInt0eXBlKGUpLl9fbmFtZV9ffToge2V9IgogICAgICAgICAgICBlbWl0KHsiaWQiOiBtaWQsICJlcnJvciI6IHRleHQsICJnYXRlZF85MDIxIjogX2lzXzkwMjEodGV4dCl9KQoKICAgIGF3YWl0IGFnZW50LmNsb3NlKCkKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgYXN5bmNpby5ydW4obWFpbigpKQo="; /** Decode the embedded agent program to its Python source. */ export function coreDeviceAgentScript(): string { diff --git a/packages/tool-server/src/tools/describe/contract.ts b/packages/tool-server/src/tools/describe/contract.ts index f40b326a5..fbc7dba28 100644 --- a/packages/tool-server/src/tools/describe/contract.ts +++ b/packages/tool-server/src/tools/describe/contract.ts @@ -77,7 +77,7 @@ export type DescribeSource = | "cdp-dom" | "vega-automation" | "tv-focus" - | "springboard"; + | "coredevice-ax"; // Internal shape produced by the per-platform adapters. The `tree` is consumed // by the formatter in `format-tree.ts` and then dropped before the tool replies diff --git a/packages/tool-server/src/tools/describe/platforms/ios/index.ts b/packages/tool-server/src/tools/describe/platforms/ios/index.ts index ba486ee17..d0caf226e 100644 --- a/packages/tool-server/src/tools/describe/platforms/ios/index.ts +++ b/packages/tool-server/src/tools/describe/platforms/ios/index.ts @@ -9,7 +9,7 @@ import { parseNativeDescribeScreenResult } from "../../../native-devtools/native import { DescribeTreeData, parseDescribeResult, type DescribeNode } from "../../contract"; import { adaptAXDescribeToDescribeResult } from "./ios-ax-adapter"; import { adaptNativeDescribeToDescribeResult } from "./ios-native-adapter"; -import { adaptSpringboardToDescribeResult } from "./ios-springboard-adapter"; +import { adaptCoreDeviceAxToDescribeResult } from "./ios-coredevice-ax-adapter"; const DEGRADED_HINT = "This simulator was not booted through argent — system dialogs and native modals may not appear. You MUST call boot-device with force=true now to restart the simulator and apply full accessibility settings before continuing."; @@ -27,16 +27,18 @@ const TVOS_HINT = "(up/down/left/right/select/back/menu/home) to move focus, and `keyboard` to type. " + "See the argent-tv-interact skill."; -// Physical iPhones expose no app-free in-app accessibility tree (Apple gates the -// CoreDevice axAuditDaemon to trusted/AppleInternal callers). The one structured -// screen data we CAN read is SpringBoard's home-screen layout — so describe -// returns that, with this hint making its scope and precision explicit. -const PHYSICAL_IOS_SPRINGBOARD_HINT = - "This is the SpringBoard home-screen layout — the only app-free structured screen data on a " + - "physical iPhone — NOT necessarily the current screen. If an app is open, its content is not " + - "here: call screenshot instead. Icon frames are approximate (derived from the home-screen grid, " + - "not exact pixels), so confirm with screenshot before a precise tap. In-app accessibility is not " + - "reachable on physical iOS (Apple gates the CoreDevice accessibility service to trusted/AppleInternal callers)."; +// Physical iPhones expose their real on-screen accessibility tree app-free via +// the iOS-26+ axAudit service (read over CoreDevice). Captions (label + value + +// traits) and reading order are exact for every element; frames are exact only +// for the subset the accessibility audit flags, and interpolated for the rest — +// this hint makes that precision boundary explicit. +const PHYSICAL_IOS_AX_HINT = + "This is the live accessibility tree of the frontmost app (or the home screen), read over " + + "CoreDevice. Element labels, values, traits and reading order are exact. Frames are exact for " + + "elements the accessibility audit reported and APPROXIMATE (interpolated from neighbours) for the " + + "rest — good enough to tap a row in a vertical list, but confirm with screenshot before a precise " + + "tap, especially for controls like toggles. Apple does not expose per-element geometry on a " + + "physical device, so screenshot remains authoritative for exact positions."; function emptyTree(): DescribeNode { return parseDescribeResult({ @@ -70,32 +72,23 @@ export async function describeIos( params: DescribeIosParams, options: DescribeIosOptions = {} ): Promise { - // Physical iPhones are driven over CoreDevice. There is no app-free *in-app* - // accessibility tree on a real device: the on-device tree is served by - // CoreDevice's axAuditDaemon, but Apple gates it to trusted/AppleInternal - // callers (hardware-verified on iOS 27, 2026-07). The DTX service - // `…axAuditDaemon.remoteserver.shim.remote` opens over the developer - // (untrusted) CoreDevice tunnel pymobiledevice3 forms, but the daemon - // terminates the connection on the first message (every audit selector, and - // even the standard DTX capability handshake); its RemoteXPC replacement - // `…axAuditDaemon.remoteAXService` requires the `AppleInternal` entitlement. - // (DTX transport itself works over the same tunnel — `dvt proclist` succeeds — - // so it's Apple's auth wall, not a transport gap.) The two simulator backends - // below can't run against hardware either (they shell `simctl spawn`). - // - // What we CAN read app-free is SpringBoard's home-screen layout, so `describe` - // on a physical device returns the home-screen app grid (icons + dock) via - // CoreDevice's springboardservices. Icon frames are derived from the icon-grid - // geometry and are approximate; the hint tells the agent this is the home - // screen (not necessarily the current app) and to confirm with screenshot. + // Physical iPhones are driven over CoreDevice. describe reads the device's real + // on-screen accessibility tree app-free via the iOS-26+ axAudit service (the + // `…axAuditDaemon.remoteserver.shim.remote` DTX daemon). This works in ANY app + // and on the home screen — the same VoiceOver-style walk. It needs the RSDCheckin + // handshake that iOS 26 added (the sidecar performs it); without it the daemon + // drops the connection on the first byte. Apple exposes no per-element geometry + // on hardware, so frames are exact only for elements the accessibility audit + // flags and interpolated for the rest (see the adapter + hint). The two + // simulator backends below can't run against hardware (they shell `simctl spawn`). if (isPhysicalIos(device)) { const ref = coreDeviceRef(device); const coreDevice = await registry.resolveService(ref.urn, ref.options); - const home = await coreDevice.homescreen(); + const axtree = await coreDevice.axtree(); return { - tree: adaptSpringboardToDescribeResult(home), - source: "springboard", - hint: PHYSICAL_IOS_SPRINGBOARD_HINT, + tree: adaptCoreDeviceAxToDescribeResult(axtree), + source: "coredevice-ax", + hint: PHYSICAL_IOS_AX_HINT, }; } diff --git a/packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts b/packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts new file mode 100644 index 000000000..77abfc0a5 --- /dev/null +++ b/packages/tool-server/src/tools/describe/platforms/ios/ios-coredevice-ax-adapter.ts @@ -0,0 +1,123 @@ +import type { CoreDeviceAxTree } from "../../../../blueprints/core-device"; +import { parseDescribeResult, type DescribeNode } from "../../contract"; + +/** + * Adapts a physical iPhone's on-screen accessibility tree (from the iOS-26+ + * axAudit service, read app-free over CoreDevice) into a describe tree. + * + * The audit gives a rich VoiceOver caption (label + value + traits) and reading + * order for EVERY on-screen element, but Apple doesn't expose per-element + * geometry on hardware — only the subset of elements the accessibility audit + * flags carry an on-screen rect. So real frames are used where present, and the + * rest are interpolated from their reading-order neighbours (good enough to tap + * a row in a vertical list; the tool's hint says to confirm with screenshot). + */ + +// AX trait tokens that appear (comma-separated) at the tail of a VoiceOver +// caption, mapped to a describe role. Order matters: the first structural trait +// found wins. +const TRAIT_ROLE: Array<[RegExp, string]> = [ + [/^Button$/i, "AXButton"], + [/^Link$/i, "AXLink"], + [/^Header$/i, "AXHeader"], + [/^(Toggle|Switch)$/i, "AXSwitch"], + [/^Adjustable$/i, "AXSlider"], + [/^Search Field$/i, "AXSearchField"], + [/^Text Field$/i, "AXTextField"], + [/^Tab$/i, "AXTab"], + [/^Image$/i, "AXImage"], +]; +// Trailing tokens that are traits/states (stripped from the label). +const TRAIT_TOKEN = + /^(Button|Link|Header|Toggle|Switch|Adjustable|Search Field|Text Field|Tab|Image|Selected|Not Selected|Dimmed|Disabled)$/i; + +function parseCaption(caption: string): { label: string; role: string } { + const tokens = caption.split(/,\s*/).filter((t) => t.length > 0); + let role = "AXStaticText"; + for (const [re, r] of TRAIT_ROLE) { + if (tokens.some((t) => re.test(t))) { + role = r; + break; + } + } + // Drop trailing trait/state tokens to get a cleaner label; keep the full + // caption if that would leave nothing. + let end = tokens.length; + while (end > 0 && TRAIT_TOKEN.test(tokens[end - 1])) end--; + const label = (end > 0 ? tokens.slice(0, end) : tokens).join(", ") || caption; + return { label, role }; +} + +const RECT_RE = /-?\d+(?:\.\d+)?/g; + +/** Parse "{{x, y}, {w, h}}" (points) → normalized frame, or null. */ +function parseRect(rect: string | undefined, sw: number, sh: number): DescribeNode["frame"] | null { + if (!rect || sw <= 0 || sh <= 0) return null; + const nums = rect.match(RECT_RE); + if (!nums || nums.length < 4) return null; + const [x, y, w, h] = nums.slice(0, 4).map(Number); + const clamp = (v: number) => Math.max(0, Math.min(1, v)); + return { + x: clamp(x / sw), + y: clamp(y / sh), + width: clamp(w / sw), + height: clamp(h / sh), + }; +} + +const MARGIN_X = 0.04; +const APPROX_HEIGHT = 0.05; + +/** Full-width approximate frame centred at normalized y. */ +function approxFrame(yCenter: number): DescribeNode["frame"] { + const y = Math.max(0, Math.min(1 - APPROX_HEIGHT, yCenter - APPROX_HEIGHT / 2)); + return { x: MARGIN_X, y, width: 1 - 2 * MARGIN_X, height: APPROX_HEIGHT }; +} + +/** + * Fill frames for elements the audit didn't give a rect: interpolate each gap's + * y-centres between the nearest real rects above and below (reading order), so a + * list row lands roughly where it should. Falls back to an even top-to-bottom + * spread when there are no anchoring rects. + */ +function fillFrames(frames: Array): DescribeNode["frame"][] { + const n = frames.length; + const yc = (f: DescribeNode["frame"]) => f.y + f.height / 2; + const out = frames.slice(); + for (let i = 0; i < n; i++) { + if (out[i]) continue; + let prev = i - 1; + while (prev >= 0 && !out[prev]) prev--; + let next = i + 1; + while (next < n && !out[next]) next++; + const top = prev >= 0 ? yc(out[prev]!) : 0.06; + const bottom = next < n ? yc(out[next]!) : 0.94; + const span = next < n ? next : n; // denominator for even spread in the run + const start = prev >= 0 ? prev : -1; + const frac = (i - start) / (span - start); + out[i] = approxFrame(top + (bottom - top) * frac); + } + return out as DescribeNode["frame"][]; +} + +export function adaptCoreDeviceAxToDescribeResult(tree: CoreDeviceAxTree): DescribeNode { + const sw = tree.screen?.w ?? 0; + const sh = tree.screen?.h ?? 0; + const els = tree.elements ?? []; + + const rectFrames = els.map((e) => parseRect(e.rect, sw, sh)); + const frames = fillFrames(rectFrames); + + const children: DescribeNode[] = els.map((e, i) => { + const { label, role } = parseCaption(e.caption ?? ""); + const node: DescribeNode = { role, frame: frames[i], children: [] }; + if (label) node.label = label; + return node; + }); + + return parseDescribeResult({ + role: "AXGroup", + frame: { x: 0, y: 0, width: 1, height: 1 }, + children, + }); +} diff --git a/packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts b/packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts deleted file mode 100644 index 06d3f27d5..000000000 --- a/packages/tool-server/src/tools/describe/platforms/ios/ios-springboard-adapter.ts +++ /dev/null @@ -1,242 +0,0 @@ -import type { CoreDeviceHomescreen } from "../../../../blueprints/core-device"; -import { parseDescribeResult, type DescribeNode } from "../../contract"; - -/** - * Adapts a physical iPhone's SpringBoard icon state (from CoreDevice's - * `springboardservices`) into a describe tree for the **home screen**. - * - * This is the only app-free *structured* screen data reachable on a real device - * (in-app accessibility is Apple-gated — see index.ts). It describes what apps - * exist and roughly where their icons sit; the exact pixel positions of icons - * are not exposed by SpringBoard, so frames are computed from the icon-grid - * geometry (`getHomeScreenIconMetrics`) and are APPROXIMATE. The tool's hint - * tells the agent to confirm with `screenshot` before a precise tap. - * - * `iconState` shape (SpringBoard `getIconState`, format 2): - * [ dock:[icon…], page0:[item…], page1:[item…], … ] - * where a leaf icon is `{displayName, bundleIdentifier, …}` and a widget/folder - * is `{iconType, gridSize:{…}, iconLists|elements, …}` (no displayName). - */ - -interface Metrics { - cols: number; - rows: number; -} - -function readMetrics(m: Record): Metrics { - const cols = Math.round(m.homeScreenIconColumns ?? 4); - const rows = Math.round(m.homeScreenIconRows ?? 6); - return { - cols: cols > 0 ? cols : 4, - rows: rows > 0 ? rows : 6, - }; -} - -interface IconRecord { - displayName?: unknown; - bundleIdentifier?: unknown; - displayIdentifier?: unknown; - // A widget's footprint is a *string* size class ("small"/"medium"/"large"), - // not a {columns,rows} pair. - gridSize?: unknown; - [k: string]: unknown; -} - -function str(v: unknown): string | undefined { - return typeof v === "string" && v.length > 0 ? v : undefined; -} - -function isLeaf(item: IconRecord): boolean { - return str(item.displayName) !== undefined || str(item.bundleIdentifier) !== undefined; -} - -/** - * Cell footprint (columns × rows) an item occupies. Leaves are 1×1; a widget's - * `gridSize` is an iOS size class — "small" is 2×2, "medium" 4×2, "large" 4×4 - * (an extra-large 4×6 also exists). Anything unrecognised falls back to 1×1. - */ -function span(item: IconRecord): { w: number; h: number } { - switch (str(item.gridSize)) { - case "small": - return { w: 2, h: 2 }; - case "medium": - return { w: 4, h: 2 }; - case "large": - return { w: 4, h: 4 }; - case "extralarge": - return { w: 4, h: 6 }; - default: - return { w: 1, h: 1 }; - } -} - -/** Clamp to the unit interval so an approximate frame never fails validation. */ -function unit(v: number): number { - return Math.max(0, Math.min(1, v)); -} - -// The home-screen content band, calibrated (iPhone 15, 393×852pt) so a tap at a -// frame's centre lands on the icon: the grid starts just below the status bar -// and a row is ~0.115 of the height. SpringBoard doesn't publish exact insets, -// so these are approximate — good enough to hit the right icon on common iPhone -// sizes, which is all `describe` promises here (the hint says confirm with -// screenshot). The dock sits on its own row near the bottom. -const GRID_TOP = 0.085; -const GRID_ROW_PITCH = 0.115; -const GRID_MARGIN_X = 0.06; - -/** - * Normalized frame for a grid block at (col,row) spanning (w,h) cells. A 1×1 - * icon fills ~70%×50% of its cell (centred); a multi-cell widget fills its - * whole block so its bounding box reads true. - */ -function gridFrame( - col: number, - row: number, - w: number, - h: number, - cols: number -): DescribeNode["frame"] { - const cellW = (1 - 2 * GRID_MARGIN_X) / cols; - const cellH = GRID_ROW_PITCH; - const blockX = GRID_MARGIN_X + col * cellW; - const blockY = GRID_TOP + row * cellH; - const blockW = w * cellW; - const blockH = h * cellH; - if (w === 1 && h === 1) { - const fw = blockW * 0.7; - const fh = blockH * 0.5; - return { - x: unit(blockX + (blockW - fw) / 2), - y: unit(blockY + (blockH - fh) / 2), - width: unit(fw), - height: unit(fh), - }; - } - return { x: unit(blockX), y: unit(blockY), width: unit(blockW), height: unit(blockH) }; -} - -/** - * SpringBoard packs a page row-major, first-fit: each item takes the earliest - * top-left cell where its w×h block is free. Track a boolean occupancy grid and - * return each item's placed (col,row) so multi-row widgets don't shove the icons - * after them into the wrong slots. - */ -class GridPacker { - private readonly occ: boolean[]; - constructor( - private readonly cols: number, - private readonly rows: number - ) { - this.occ = new Array(cols * rows).fill(false); - } - private free(col: number, row: number, w: number, h: number): boolean { - if (col + w > this.cols || row + h > this.rows) return false; - for (let r = row; r < row + h; r++) { - for (let c = col; c < col + w; c++) { - if (this.occ[r * this.cols + c]) return false; - } - } - return true; - } - /** Place a w×h block; returns its top-left (col,row), or null if the page is full. */ - place(w: number, h: number): { col: number; row: number } | null { - const cw = Math.min(w, this.cols); - const ch = Math.min(h, this.rows); - for (let row = 0; row + ch <= this.rows; row++) { - for (let col = 0; col + cw <= this.cols; col++) { - if (this.free(col, row, cw, ch)) { - for (let r = row; r < row + ch; r++) { - for (let c = col; c < col + cw; c++) this.occ[r * this.cols + c] = true; - } - return { col, row }; - } - } - } - return null; - } -} - -/** Frame for the i-th of `count` dock slots along the bottom row. */ -function dockFrame(i: number, count: number): DescribeNode["frame"] { - const n = Math.max(count, 1); - const marginX = 0.06; - const slot = (1 - 2 * marginX) / n; - const cx = marginX + (i + 0.5) * slot; - const w = slot * 0.6; - const h = 0.05; - const cy = 0.945; - return { x: unit(cx - w / 2), y: unit(cy - h / 2), width: unit(w), height: unit(h) }; -} - -function iconNode(item: IconRecord, frame: DescribeNode["frame"], role: string): DescribeNode { - const node: DescribeNode = { role, frame, children: [] }; - const label = str(item.displayName); - if (label) node.label = label; - const id = str(item.bundleIdentifier) ?? str(item.displayIdentifier); - if (id) node.identifier = id; - return node; -} - -/** - * Build the home-screen describe tree: the first home page's icons laid out on - * the grid, plus the dock. Later pages aren't on screen, so they're omitted - * (the agent swipes to reach them, then re-describes). - */ -export function adaptSpringboardToDescribeResult(home: CoreDeviceHomescreen): DescribeNode { - const { cols, rows } = readMetrics(home.metrics); - const pages = Array.isArray(home.iconState) ? (home.iconState as IconRecord[][]) : []; - const children: DescribeNode[] = []; - - // iconState[0] is the dock; iconState[1] is the first home page. - const dock = Array.isArray(pages[0]) ? pages[0] : []; - const firstPage = Array.isArray(pages[1]) ? pages[1] : []; - - // First home page — pack items row-major first-fit (SpringBoard's own order), - // so multi-row widgets don't shove the icons after them into the wrong slots. - const packer = new GridPacker(cols, rows); - for (const item of firstPage) { - if (!item || typeof item !== "object") continue; - const { w, h } = span(item); - const pos = packer.place(w, h); - if (!pos) continue; // page is full - const frame = gridFrame(pos.col, pos.row, Math.min(w, cols), Math.min(h, rows), cols); - if (isLeaf(item)) { - children.push(iconNode(item, frame, "AXIcon")); - } else { - // A widget or folder: surface it as a group with any nested leaf icons. - const group = iconNode(item, frame, "AXGroup"); - for (const leaf of collectNested(item)) group.children.push(iconNode(leaf, frame, "AXIcon")); - children.push(group); - } - } - - // Dock along the bottom. - dock.forEach((item, i) => { - if (item && typeof item === "object") { - children.push(iconNode(item, dockFrame(i, dock.length), "AXIcon")); - } - }); - - return parseDescribeResult({ - role: "AXGroup", - frame: { x: 0, y: 0, width: 1, height: 1 }, - children, - }); -} - -/** Pull leaf icons out of a folder/widget's nested `iconLists`. */ -function collectNested(item: IconRecord): IconRecord[] { - const out: IconRecord[] = []; - const visit = (v: unknown): void => { - if (Array.isArray(v)) { - for (const el of v) visit(el); - } else if (v && typeof v === "object") { - const rec = v as IconRecord; - if (isLeaf(rec)) out.push(rec); - else if (Array.isArray(rec.iconLists)) visit(rec.iconLists); - } - }; - if (Array.isArray(item.iconLists)) visit(item.iconLists); - return out; -} diff --git a/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts b/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts index b6a143817..38b12f74f 100644 --- a/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts +++ b/packages/tool-server/test/physical-ios-coredevice-sidecar.test.ts @@ -2,10 +2,10 @@ * Coverage for the physical-iOS (CoreDevice) backend after it moved from a * per-call `pymobiledevice3` CLI spawn to a persistent stdio sidecar: * - * - `adaptSpringboardToDescribeResult` — the SpringBoard home-screen → describe - * tree adapter that backs `describe` on a real iPhone (in-app AX is Apple- - * gated). Frames are grid-derived; this pins the layout math (widget spans - * push icons down; dock sits at the bottom; every frame stays in [0,1]). + * - `adaptCoreDeviceAxToDescribeResult` — the axAudit accessibility tree → + * describe adapter that backs `describe` on a real iPhone. Pins caption→role + * parsing, label cleanup, rect normalization, and frame interpolation for the + * elements the audit didn't rect (every frame stays in [0,1]). * - `agentError` — the 9021 (iOS-27 host-input gate) message mapping. * - `CoreDeviceAgent` — the stdio JSON protocol: ready handshake, id-correlated * request/response, and error propagation (via a stand-in node process). @@ -15,40 +15,15 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { FAILURE_CODES, getFailureSignal } from "@argent/registry"; -import { adaptSpringboardToDescribeResult } from "../src/tools/describe/platforms/ios/ios-springboard-adapter"; +import { adaptCoreDeviceAxToDescribeResult } from "../src/tools/describe/platforms/ios/ios-coredevice-ax-adapter"; import { agentError } from "../src/blueprints/core-device"; import { CoreDeviceAgent, CoreDeviceAgentError } from "../src/blueprints/coredevice-agent"; -const METRICS = { - homeScreenIconColumns: 4, - homeScreenIconRows: 6, - homeScreenWidth: 393, - homeScreenHeight: 852, - homeScreenIconDockMaxCount: 4, -}; - -const leaf = (name: string, id: string) => ({ displayName: name, bundleIdentifier: id }); -const widget = (size: string) => ({ iconType: "custom", gridSize: size }); - -// Two 2x2 "small" widgets fill the top two rows; the first app row therefore -// starts at grid row 2. A second page must NOT appear (it's off-screen). -const ICON_STATE = [ - [leaf("Phone", "com.apple.mobilephone"), leaf("Safari", "com.apple.mobilesafari")], // dock - [ - widget("small"), - widget("small"), - leaf("FaceTime", "com.apple.facetime"), - leaf("Maps", "com.apple.Maps"), - ], - [leaf("OffPage", "com.example.offpage")], // page 2, not rendered -]; - interface Node { role: string; frame: { x: number; y: number; width: number; height: number }; children: Node[]; label?: string; - identifier?: string; } function flatten(n: Node, out: Node[] = []): Node[] { out.push(n); @@ -57,34 +32,48 @@ function flatten(n: Node, out: Node[] = []): Node[] { } const center = (f: Node["frame"]) => ({ x: f.x + f.width / 2, y: f.y + f.height / 2 }); -describe("adaptSpringboardToDescribeResult", () => { - const tree = adaptSpringboardToDescribeResult({ iconState: ICON_STATE, metrics: METRICS }); +// A realistic axAudit snapshot: some elements carry an audit rect (points on a +// 393x852 screen), others don't (interpolated by the adapter). +const AXTREE = { + screen: { w: 393, h: 852 }, + elements: [ + { caption: "Settings, Button", id: "a1", rect: "{{318, 63}, {55, 36}}" }, + { caption: "Wi-Fi, Header", id: "a2", rect: "{{32, 168}, {55, 26}}" }, + { caption: "Wi-Fi, 1, Button, Toggle", id: "a3" }, // no rect -> interpolated + { caption: "Other…, Button", id: "a4", rect: "{{16, 553}, {361, 52}}" }, + { caption: "Known networks will be joined automatically.", id: "a5" }, // static text + ], +}; + +describe("adaptCoreDeviceAxToDescribeResult", () => { + const tree = adaptCoreDeviceAxToDescribeResult(AXTREE); const nodes = flatten(tree as Node); const byLabel = (l: string) => nodes.find((n) => n.label === l); - it("returns leaf icons with label + bundle identifier", () => { - const faceTime = byLabel("FaceTime"); - expect(faceTime?.identifier).toBe("com.apple.facetime"); - expect(byLabel("Phone")?.identifier).toBe("com.apple.mobilephone"); - }); - - it("packs the first app below the two 2x2 widgets (widgets occupy the top rows)", () => { - // FaceTime is item 3 on the page but the two small widgets take rows 0-1, - // so it lands on row 2 — visibly below the widgets, not at the very top. - const widgets = nodes.filter((n) => n.role === "AXGroup" && n !== tree); - expect(widgets).toHaveLength(2); - const widgetBottom = Math.max(...widgets.map((w) => w.frame.y + w.frame.height)); - const faceTime = byLabel("FaceTime")!; - expect(center(faceTime.frame).y).toBeGreaterThan(widgetBottom - 0.01); + it("parses roles from caption traits and strips them from the label", () => { + expect(byLabel("Settings")?.role).toBe("AXButton"); + expect(byLabel("Wi-Fi")?.role).toBe("AXHeader"); + // Button trait wins the role; trailing Button/Toggle stripped from the label. + expect(byLabel("Wi-Fi, 1")?.role).toBe("AXButton"); + // No trait -> static text, full caption kept as label. + const stat = nodes.find((n) => n.label?.startsWith("Known networks")); + expect(stat?.role).toBe("AXStaticText"); }); - it("places the dock along the bottom", () => { - expect(center(byLabel("Phone")!.frame).y).toBeGreaterThan(0.85); - expect(center(byLabel("Safari")!.frame).y).toBeGreaterThan(0.85); + it("normalizes an audited rect (points) into a [0,1] frame", () => { + const other = byLabel("Other…")!; + // {{16, 553}, {361, 52}} on 393x852 + expect(other.frame.x).toBeCloseTo(16 / 393, 3); + expect(other.frame.y).toBeCloseTo(553 / 852, 3); + expect(other.frame.width).toBeCloseTo(361 / 393, 3); }); - it("omits off-screen pages (only the first home page + dock)", () => { - expect(byLabel("OffPage")).toBeUndefined(); + it("interpolates a rect-less element between its neighbours (reading order)", () => { + const wifiHeader = center(byLabel("Wi-Fi")!.frame).y; // ~168/852 + const other = center(byLabel("Other…")!.frame).y; // ~553/852 + const toggle = center(byLabel("Wi-Fi, 1")!.frame).y; // no rect, between the two + expect(toggle).toBeGreaterThan(wifiHeader); + expect(toggle).toBeLessThan(other); }); it("keeps every frame within the normalized [0,1] box", () => { @@ -99,10 +88,12 @@ describe("adaptSpringboardToDescribeResult", () => { } }); - it("does not throw on empty / malformed icon state", () => { - expect(() => adaptSpringboardToDescribeResult({ iconState: [], metrics: {} })).not.toThrow(); + it("does not throw on an empty / screen-less tree", () => { + expect(() => adaptCoreDeviceAxToDescribeResult({ elements: [] })).not.toThrow(); expect(() => - adaptSpringboardToDescribeResult({ iconState: null, metrics: METRICS }) + adaptCoreDeviceAxToDescribeResult({ + elements: [{ caption: "x", id: "1" }], + }) ).not.toThrow(); }); }); diff --git a/packages/tool-server/test/physical-ios-followups.test.ts b/packages/tool-server/test/physical-ios-followups.test.ts index fb8fca6d1..17517e8a0 100644 --- a/packages/tool-server/test/physical-ios-followups.test.ts +++ b/packages/tool-server/test/physical-ios-followups.test.ts @@ -255,26 +255,25 @@ describe("tools unsupported on physical iOS reject with UnsupportedOperationErro }); // describe is NOT in the unsupported set: on a physical iPhone it returns the - // SpringBoard home-screen layout (the only app-free structured screen data; - // in-app AX is Apple-gated). It resolves the CoreDevice service for a - // homescreen() snapshot, so a stub service stands in for the device here. - it("describe — returns the SpringBoard home-screen tree (source springboard), not a rejection", async () => { + // real on-screen accessibility tree via the CoreDevice axtree() snapshot + // (works in-app and on the home screen). A stub service stands in here. + it("describe — returns the CoreDevice accessibility tree (source coredevice-ax), not a rejection", async () => { const registry = { resolveService: async () => ({ - homescreen: async () => ({ - iconState: [ - [{ displayName: "Phone", bundleIdentifier: "com.apple.mobilephone" }], - [{ displayName: "Settings", bundleIdentifier: "com.apple.Preferences" }], + axtree: async () => ({ + screen: { w: 393, h: 852 }, + elements: [ + { caption: "General, Button", id: "e1", rect: "{{16, 100}, {361, 44}}" }, + { caption: "Accessibility, Button", id: "e2" }, ], - metrics: { homeScreenIconColumns: 4, homeScreenIconRows: 6 }, }), }), }; const result = await describeIos(registry as never, device, {}); - expect(result.source).toBe("springboard"); + expect(result.source).toBe("coredevice-ax"); const flat = JSON.stringify(result.tree); - expect(flat).toContain("Settings"); - expect(flat).toContain("com.apple.Preferences"); + expect(flat).toContain("General"); + expect(flat).toContain("Accessibility"); expect(result.hint).toContain("screenshot"); }); });