feat(ios): control physical iOS devices over CoreDevice (experimental)

README.md

@@ -46,6 +46,65 @@ Argent drives a growing set of targets through a single toolkit, each with the r
 
 ---
 
+## Physical iOS devices (experimental)
+
+Argent can drive a **physical iPhone** — no app installed on the device — over Apple's
+CoreDevice "remote control" services (the same path Xcode's device window uses), via
+[`pymobiledevice3`](https://github.com/doronz88/pymobiledevice3). Supported interactions:
+`screenshot`, `gesture-tap`, `gesture-swipe`, `button`, `launch-app`, and `describe` (the
+live on-screen accessibility tree — see the note below). The device shows up in `list-devices` with
+`kind: "device"`. Interactions run through one persistent `pymobiledevice3` helper per
+device (connected once), so a tap/screenshot costs a socket write rather than a fresh
+Python cold-start.
+
+**Requirements**
+
+- **iOS 27 or later for tap/swipe** — Apple gates host-driven touch input to iOS 27+; on
+  earlier versions those commands report `CoreDeviceError 9021`. Screenshot and hardware
+  buttons work on earlier iOS versions too.
+- macOS with Xcode, and `pymobiledevice3` installed (e.g. `pipx install pymobiledevice3`).
+- The iPhone connected, unlocked, trusted, with **Developer Mode** on.
+
+**Setup**
+
+1. Enable the feature flag:
+   ```sh
+   argent enable physical-ios-devices
+   ```
+2. Connect the iPhone (unlocked, trusted, Developer Mode on).
+
+`list-devices` then includes the iPhone, and the supported tools work against its UDID.
+The first interaction (or `boot-device`) starts the required CoreDevice tunnel
+automatically: Argent shows a standard macOS authorization prompt (Touch ID / password)
+to launch `pymobiledevice3 remote tunneld` as root (creating the tunnel interface needs
+root once; every other command runs unprivileged). No manual `sudo`. When the signed
+`argent-device-auth` helper is installed, the prompt is branded as Argent; otherwise it's
+the system's default admin prompt.
+
+If the prompt is declined or there's no GUI session (headless), start the tunnel manually
+and leave it running: `sudo pymobiledevice3 remote tunneld`.
+
+**Limitations / notes**
+
+- `describe` returns the device's **live on-screen accessibility tree** — the frontmost app's
+  elements (or the home screen), read app-free via the iOS-26+ accessibility-audit service over
+  CoreDevice. Element labels, values, traits (roles) and reading order are exact. Frames are exact
+  for the elements the accessibility audit reports and **interpolated** from reading-order
+  neighbours for the rest (Apple doesn't expose per-element geometry on a physical device), so
+  they're good enough to tap a row in a vertical list — but confirm with `screenshot` before a
+  precise tap, especially for controls like toggles. (This needs the RSDCheckin handshake iOS 26
+  added; the helper performs it. For pixel-exact in-app frames + taps you'd need an on-device
+  XCUITest runner, which requires code-signing.)
+- Not supported yet (return a clear "not supported" error): keyboard/typing, pinch & rotate
+  (multi-touch), `open-url`, `reinstall-app`, `restart-app`, and the native inspection /
+  profiling tools (`native-*`, `native-profiler-*`, `screenshot-diff`). `launch-app` (via
+  `devicectl`) works independently of the CoreDevice tunnel — it can succeed even before the
+  tunnel setup above has run.
+- Overrides: `ARGENT_PYMOBILEDEVICE3` (path to the binary), `ARGENT_PMD3_TUNNELD_PORT`
+  (defaults to `49151`).
+
+---
+
 ## Installation
 
 #### Prerequisites

packages/argent/scripts/bundle-tools.cjs

@@ -104,6 +104,17 @@ const AX_TCP_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/tcp/ax-service");
 const BIN_DIR = path.resolve(__dirname, "../bin");
 const AX_BIN_DEST = path.resolve(BIN_DIR, "darwin/ax-service");
 const AX_TCP_BIN_DEST = path.resolve(BIN_DIR, "darwin/tcp/ax-service");
+// argent-device-auth: macOS host helper for the branded physical-iOS tunnel
+// auth prompt. Best-effort — only present once argent-private publishes the
+// signed binary; until then physical iOS falls back to the osascript prompt.
+const DEVICE_AUTH_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/argent-device-auth");
+const DEVICE_AUTH_BIN_DEST = path.resolve(BIN_DIR, "darwin/argent-device-auth");
+// Argent icon shown in that prompt (committed under native-devtools-ios/assets).
+const DEVICE_ICON_SRC = path.resolve(
+  WORKSPACE_ROOT,
+  "packages/native-devtools-ios/assets/argent-icon.png"
+);
+const DEVICE_ICON_DEST = path.resolve(__dirname, "../assets/argent-icon.png");
 // tvOS control binaries. Both are macOS-only: tvos-ax-service runs inside an
 // appletvsimulator, tvos-hid-daemon runs on the host. Unix-socket only.
 const TVOS_AX_BIN_SRC = path.resolve(BIN_SRC_ROOT, "darwin/tvos-ax-service");
@@ -208,6 +219,27 @@ const ASSETS = [
     copiedLabel: "ax-service (tcp) binary",
     missLabel: "ax-service (tcp) binary",
   },
+  // macOS host helper for the branded physical-iOS tunnel auth prompt.
+  // Best-effort: present once argent-private publishes the signed binary;
+  // until then physical iOS falls back to the (unbranded) osascript prompt.
+  {
+    kind: "file",
+    src: DEVICE_AUTH_BIN_SRC,
+    dest: DEVICE_AUTH_BIN_DEST,
+    mode: 0o755,
+    required: false,
+    copiedLabel: "argent-device-auth binary",
+    missLabel: "argent-device-auth binary",
+  },
+  // Argent icon shown in the device-auth prompt (committed; best-effort copy).
+  {
+    kind: "file",
+    src: DEVICE_ICON_SRC,
+    dest: DEVICE_ICON_DEST,
+    required: false,
+    copiedLabel: "device-auth icon",
+    missLabel: "device-auth icon",
+  },
   // tvOS AX reader — spawned inside an appletvsimulator via simctl to read
   // the focus-engine accessibility tree. macOS-only, unix-socket transport.
   {

packages/configuration-core/src/flags.ts

@@ -57,6 +57,11 @@ export const FLAG_REGISTRY: readonly FlagDefinition[] = [
     name: "artifacts-list-endpoint",
     description: "Expose GET /artifacts for remote artifact inventory consumers.",
   },
+  {
+    name: "physical-ios-devices",
+    description:
+      "Discover and control physical iOS devices (iOS 27+) over Apple's CoreDevice tunnel via pymobiledevice3. The required tunnel is auto-started via a macOS admin prompt (or run `sudo pymobiledevice3 remote tunneld` manually). Supports screenshot, tap, swipe, hardware buttons, and launch-app.",
+  },
   {
     name: "tool-server-event-log",
     description: "Write structured tool-server lifecycle events to a JSONL file.",

packages/native-devtools-ios/src/index.ts

@@ -9,6 +9,9 @@ import * as fs from "node:fs";
 const DYLIB_DIR = process.env.ARGENT_NATIVE_DEVTOOLS_DIR ?? path.join(__dirname, "..", "dylibs");
 const BIN_DIR = process.env.ARGENT_SIMULATOR_SERVER_DIR ?? path.join(__dirname, "..", "bin");
 const DYLIB_TCP_DIR = process.env.ARGENT_NATIVE_DEVTOOLS_TCP_DIR ?? path.join(DYLIB_DIR, "tcp");
+// Committed static assets (e.g. the Argent icon shown in the device-auth prompt).
+const ASSETS_DIR =
+  process.env.ARGENT_NATIVE_DEVTOOLS_ASSETS_DIR ?? path.join(__dirname, "..", "assets");
 const DYLIB_TVOS_DIR = path.join(DYLIB_DIR, "tvos");
 
 // The *local* and *tvos* accessors are darwin-only: the dylibs they return get
@@ -146,6 +149,27 @@ export function axServiceBinaryPathTcp(): string {
   return requireBinIn(platformTcpBinDir(), "ax-service");
 }
 
+// argent-device-auth is the macOS host helper that pops the branded macOS
+// authorization modal (password / Touch ID) and runs a command as root —
+// Argent uses it to start the physical-iOS CoreDevice tunnel without a manual
+// `sudo`. Unlike the resolvers above it returns null (rather than throwing)
+// when absent, so callers can fall back to a less-branded escalation path.
+// Override the binary with ARGENT_DEVICE_AUTH_HELPER (absolute path).
+export function deviceAuthHelperPath(): string | null {
+  if (process.platform !== "darwin") return null;
+  const override = process.env.ARGENT_DEVICE_AUTH_HELPER;
+  if (override) return fs.existsSync(override) ? override : null;
+  const p = path.join(platformBinDir(), "argent-device-auth");
+  return fs.existsSync(p) ? p : null;
+}
+
+// Path to the Argent icon shown in the device-auth prompt, or null if missing.
+// Override with ARGENT_DEVICE_ICON (absolute path to a PNG/icns).
+export function argentIconPath(): string | null {
+  const p = process.env.ARGENT_DEVICE_ICON ?? path.join(ASSETS_DIR, "argent-icon.png");
+  return fs.existsSync(p) ? p : null;
+}
+
 // tvOS control binaries. tvos-ax-service is `simctl spawn`d into an
 // appletvsimulator to read the focus-engine AX state; tvos-hid-daemon runs on
 // the host and injects Siri-remote HID via SimulatorKit. Both are darwin-only.

packages/registry/src/errors.ts

@@ -26,11 +26,13 @@ export const FAILURE_COMMANDS = [
   "emulator",
   "vega",
   "xcrun_simctl",
+  "xcrun_devicectl",
   "xctrace",
   "native_devtools",
   "android_devtools",
   "ax_service",
   "simulator_server",
+  "pymobiledevice3",
   "cdp",
   "electron",
   "npm",

packages/registry/src/failure-codes.ts

@@ -79,13 +79,15 @@ export const FAILURE_CODES = {
   SIMULATOR_SERVER_READY_TIMEOUT: "SIMULATOR_SERVER_READY_TIMEOUT",
   SIMULATOR_SERVER_PROCESS_ERROR: "SIMULATOR_SERVER_PROCESS_ERROR",
   SIMULATOR_SERVER_TERMINATED: "SIMULATOR_SERVER_TERMINATED",
+  SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED: "SIMULATOR_SERVER_PHYSICAL_DEVICE_UNSUPPORTED",
 
   AX_QUERY_TIMEOUT: "AX_QUERY_TIMEOUT",
   AX_DAEMON_READY_TIMEOUT: "AX_DAEMON_READY_TIMEOUT",
   AX_DAEMON_EXITED_BEFORE_READY: "AX_DAEMON_EXITED_BEFORE_READY",
   AX_DAEMON_PROCESS_ERROR: "AX_DAEMON_PROCESS_ERROR",
   AX_FACTORY_OPTIONS_MISSING: "AX_FACTORY_OPTIONS_MISSING",
   AX_WRONG_PLATFORM: "AX_WRONG_PLATFORM",
+  AX_PHYSICAL_DEVICE_UNSUPPORTED: "AX_PHYSICAL_DEVICE_UNSUPPORTED",
   AX_DEVICE_ID_INVALID: "AX_DEVICE_ID_INVALID",
   AX_DESCRIBE_ERROR: "AX_DESCRIBE_ERROR",
   AX_QUERY_FAILED: "AX_QUERY_FAILED",
@@ -96,9 +98,22 @@ export const FAILURE_CODES = {
   ANDROID_REINSTALL_INSTALL_FAILED: "ANDROID_REINSTALL_INSTALL_FAILED",
   ANDROID_RESTART_FAILED: "ANDROID_RESTART_FAILED",
   IOS_LAUNCH_SIMCTL_FAILED: "IOS_LAUNCH_SIMCTL_FAILED",
+  IOS_LAUNCH_DEVICECTL_FAILED: "IOS_LAUNCH_DEVICECTL_FAILED",
   IOS_OPEN_URL_FAILED: "IOS_OPEN_URL_FAILED",
   IOS_REINSTALL_INSTALL_FAILED: "IOS_REINSTALL_INSTALL_FAILED",
   IOS_RESTART_LAUNCH_FAILED: "IOS_RESTART_LAUNCH_FAILED",
+
+  CORE_DEVICE_FLAG_DISABLED: "CORE_DEVICE_FLAG_DISABLED",
+  CORE_DEVICE_FACTORY_OPTIONS_MISSING: "CORE_DEVICE_FACTORY_OPTIONS_MISSING",
+  CORE_DEVICE_WRONG_DEVICE: "CORE_DEVICE_WRONG_DEVICE",
+  CORE_DEVICE_PMD3_NOT_FOUND: "CORE_DEVICE_PMD3_NOT_FOUND",
+  CORE_DEVICE_TUNNEL_UNREACHABLE: "CORE_DEVICE_TUNNEL_UNREACHABLE",
+  CORE_DEVICE_TUNNEL_NOT_REGISTERED: "CORE_DEVICE_TUNNEL_NOT_REGISTERED",
+  CORE_DEVICE_TUNNEL_AUTH_DECLINED: "CORE_DEVICE_TUNNEL_AUTH_DECLINED",
+  CORE_DEVICE_TUNNEL_TIMEOUT: "CORE_DEVICE_TUNNEL_TIMEOUT",
+  CORE_DEVICE_IOS_VERSION_TOO_OLD: "CORE_DEVICE_IOS_VERSION_TOO_OLD",
+  CORE_DEVICE_COMMAND_FAILED: "CORE_DEVICE_COMMAND_FAILED",
+
   NATIVE_DEVTOOLS_DESCRIBE_ERROR: "NATIVE_DEVTOOLS_DESCRIBE_ERROR",
   NATIVE_DEVTOOLS_VIEW_AT_POINT_ERROR: "NATIVE_DEVTOOLS_VIEW_AT_POINT_ERROR",
   NATIVE_DEVTOOLS_USER_INTERACTABLE_VIEW_AT_POINT_ERROR:

packages/tool-server/src/blueprints/ax-service.ts

@@ -235,6 +235,24 @@ export const axServiceBlueprint: ServiceBlueprint<AXServiceApi, DeviceInfo> = {
         }
       );
     }
+    if (device.kind === "device") {
+      // ax-service uses `xcrun simctl spawn`, which only works on simulators.
+      // Physical iPhones are driven over CoreDevice; their in-app accessibility
+      // tree is Apple-gated (the axAuditDaemon rejects trusted/AppleInternal
+      // callers — see describe/platforms/ios/index.ts). `describe` handles a
+      // physical device itself (via the SpringBoard home-screen layout) and
+      // never resolves ax-service for it — this is a backstop for any other
+      // direct ax-service resolution.
+      throw new FailureError(
+        `${AX_SERVICE_NAMESPACE} is iOS-simulator-only. The physical device '${device.id}' is driven over CoreDevice; its in-app accessibility tree is Apple-gated, so use describe (SpringBoard home screen) or screenshot instead.`,
+        {
+          error_code: FAILURE_CODES.AX_PHYSICAL_DEVICE_UNSUPPORTED,
+          failure_stage: "ax_service_factory_platform",
+          failure_area: "tool_server",
+          error_kind: "unsupported",
+        }
+      );
+    }
     // Reject before spawning. An undefined `device.id` slips through when an
     // inner tool is invoked via a wrapper that doesn't re-validate the inner
     // schema. Without this guard `getSocketPath(undefined).slice` would crash