fix(tool-server): cap concurrent simctl device listing

packages/tool-server/src/tools/devices/list-devices.ts

@@ -150,8 +150,10 @@ async function resolveVvdShadowAdbSerials<T extends { serial: string }>(
 //     entirely — see listVegaDevices — so the wedged-VVD case is just ~6s.)
 //   - Android: one bounded `adb devices` call (6s) + ~5s concurrent getprop
 //     enrichment = ~11s.
-//   - iOS / AVD-list / Chromium self-bound by their own subprocess/socket timeouts
-//     (iOS `simctl` ~10s, AVD-list ~5s, Chromium <1s) — all comfortably under 25s.
+//   - iOS: waits up to 12s for another argent process' host-wide simctl-list lock
+//     and then runs a 10s bounded `simctl list devices` probe — still under 25s.
+//   - AVD-list / Chromium self-bound by their own subprocess/socket timeouts
+//     (AVD-list ~5s, Chromium <1s) — both comfortably under 25s.
 // The Vega binary resolution (`resolveVegaBinary`) runs first but is memoized and
 // returns the instant `vega` is found, so it adds ~0 in practice; only a pathological
 // cold-session `command -v` shell-fork hang would add up to ~4s on top of the 20s,
@@ -210,11 +212,15 @@ Booted/ready devices are listed first. Platforms whose CLI is unavailable are si
     // out to potentially-wedged devices; iOS / AVD-list / Chromium already self-bound
     // with their own short subprocess/socket timeouts, but wrapping them too makes the
     // "no branch can hang the fan-out" guarantee universal at near-zero cost (the
-    // timer is cleared on the fast happy path). The deadline only substitutes a
-    // fallback on *slowness*; a rejection still propagates exactly as before — so the
-    // `.catch(() => [])` wrappers (and the lack of one on iOS/AVDs) are unchanged.
+    // timer is cleared on the fast happy path). Branch-level discovery failures
+    // degrade to that branch's empty result so one platform issue does not hide
+    // working devices from the others.
     const [ios, iosRemote, android, avds, chromium, vega] = await Promise.all([
-      withDeadline(listIosSimulators(), [], "ios"),
+      withDeadline(
+        listIosSimulators().catch(() => []),
+        [],
+        "ios"
+      ),
       withDeadline(listRemoteIosSimulators(), [], "ios-remote"),
       withDeadline(
         // Opt into runtimeKind enrichment (list-devices surfaces TV vs mobile to

packages/tool-server/src/utils/ios-devices.ts

@@ -1,5 +1,15 @@
 import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { open, readFile, stat, unlink, type FileHandle } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { promisify } from "node:util";
+import {
+  SIMCTL_LIST_DEVICES_LOCK_RETRY_MS,
+  SIMCTL_LIST_DEVICES_LOCK_STALE_MS,
+  SIMCTL_LIST_DEVICES_LOCK_WAIT_MS,
+  SIMCTL_LIST_DEVICES_TIMEOUT_MS,
+} from "./simctl-config";
 
 const execFileAsync = promisify(execFile);
 
@@ -19,21 +29,265 @@ interface SimctlDevice {
   isAvailable: boolean;
 }
 
-interface SimctlOutput {
+export interface SimctlOutput {
   devices: Record<string, SimctlDevice[]>;
 }
 
+const DEFAULT_SIMCTL_LIST_DEVICES_LOCK_PATH = join(tmpdir(), "argent-simctl-list-devices.lock");
+const SIMCTL_LIST_DEVICES_LOCK_LIVE_OWNER_MAX_MS = SIMCTL_LIST_DEVICES_LOCK_STALE_MS * 4;
+let simctlListDevicesLockPath: string | null = DEFAULT_SIMCTL_LIST_DEVICES_LOCK_PATH;
+
+interface SimctlListDevicesLockMetadata {
+  createdAt: number;
+  ownerId: string;
+  pid: number;
+}
+
+class SimctlListDevicesLockError extends Error {
+  constructor(
+    message: string,
+    readonly cause?: unknown
+  ) {
+    super(message);
+    this.name = "SimctlListDevicesLockError";
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isNodeError(err: unknown): err is NodeJS.ErrnoException {
+  return err instanceof Error && "code" in err;
+}
+
+function isSimctlListDevicesLockError(err: unknown): err is SimctlListDevicesLockError {
+  return err instanceof SimctlListDevicesLockError;
+}
+
+function isProcessAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    return isNodeError(err) && err.code === "EPERM";
+  }
+}
+
+function isLockOwnedByLiveProcess(metadata: SimctlListDevicesLockMetadata, now: number): boolean {
+  if (now - metadata.createdAt >= SIMCTL_LIST_DEVICES_LOCK_LIVE_OWNER_MAX_MS) return false;
+  return isProcessAlive(metadata.pid);
+}
+
+function parseSimctlListDevicesLockMetadata(raw: string): SimctlListDevicesLockMetadata | null {
+  try {
+    const parsed = JSON.parse(raw) as {
+      createdAt?: unknown;
+      ownerId?: unknown;
+      pid?: unknown;
+    };
+    if (
+      typeof parsed.createdAt === "number" &&
+      typeof parsed.ownerId === "string" &&
+      typeof parsed.pid === "number"
+    ) {
+      return {
+        createdAt: parsed.createdAt,
+        ownerId: parsed.ownerId,
+        pid: parsed.pid,
+      };
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
+async function readSimctlListDevicesLockMetadata(
+  lockPath: string
+): Promise<SimctlListDevicesLockMetadata | null> {
+  try {
+    return parseSimctlListDevicesLockMetadata(await readFile(lockPath, "utf8"));
+  } catch (err) {
+    if (isNodeError(err) && err.code === "ENOENT") return null;
+    throw err;
+  }
+}
+
+async function getSimctlListDevicesLockCreatedAt(lockPath: string): Promise<number | undefined> {
+  const metadata = await readSimctlListDevicesLockMetadata(lockPath);
+  if (metadata) return metadata.createdAt;
+
+  try {
+    return (await stat(lockPath)).mtimeMs;
+  } catch (err) {
+    if (isNodeError(err) && err.code === "ENOENT") return undefined;
+    throw err;
+  }
+}
+
+async function acquireSimctlListDevicesLock(
+  lockPath: string,
+  now: number
+): Promise<SimctlListDevicesLockMetadata | null> {
+  const metadata: SimctlListDevicesLockMetadata = {
+    createdAt: now,
+    ownerId: randomUUID(),
+    pid: process.pid,
+  };
+  let handle: FileHandle | undefined;
+
+  try {
+    handle = await open(lockPath, "wx");
+    await handle.writeFile(JSON.stringify(metadata));
+    return metadata;
+  } catch (err) {
+    if (handle) {
+      await handle.close().catch(() => {});
+      await unlink(lockPath).catch(() => {});
+    }
+    if (isNodeError(err) && err.code === "EEXIST") return null;
+    throw new SimctlListDevicesLockError(
+      `failed to acquire simctl list devices lock at ${lockPath}`,
+      err
+    );
+  } finally {
+    await handle?.close().catch(() => {});
+  }
+}
+
+async function releaseSimctlListDevicesLock(
+  lockPath: string,
+  metadata: SimctlListDevicesLockMetadata
+): Promise<void> {
+  const currentMetadata = await readSimctlListDevicesLockMetadata(lockPath);
+  if (currentMetadata?.ownerId !== metadata.ownerId) return;
+
+  try {
+    await unlink(lockPath);
+  } catch (err) {
+    if (!isNodeError(err) || err.code !== "ENOENT") {
+      throw new SimctlListDevicesLockError(
+        `failed to release simctl list devices lock at ${lockPath}`,
+        err
+      );
+    }
+  }
+}
+
+async function removeStaleCleanupLock(cleanupLockPath: string, now: number): Promise<void> {
+  const createdAt = await getSimctlListDevicesLockCreatedAt(cleanupLockPath);
+  if (createdAt === undefined || now - createdAt < SIMCTL_LIST_DEVICES_LOCK_STALE_MS) return;
+
+  const metadata = await readSimctlListDevicesLockMetadata(cleanupLockPath);
+  if (metadata && isLockOwnedByLiveProcess(metadata, now)) return;
+
+  try {
+    await unlink(cleanupLockPath);
+  } catch (err) {
+    if (!isNodeError(err) || err.code !== "ENOENT") {
+      throw new SimctlListDevicesLockError(
+        `failed to remove stale simctl list devices cleanup lock at ${cleanupLockPath}`,
+        err
+      );
+    }
+  }
+}
+
+async function removeStaleSimctlListDevicesLock(lockPath: string, now: number): Promise<void> {
+  const cleanupLockPath = `${lockPath}.cleanup`;
+  let cleanupMetadata = await acquireSimctlListDevicesLock(cleanupLockPath, now);
+  if (!cleanupMetadata) {
+    await removeStaleCleanupLock(cleanupLockPath, now);
+    cleanupMetadata = await acquireSimctlListDevicesLock(cleanupLockPath, Date.now());
+    if (!cleanupMetadata) return;
+  }
+
+  try {
+    await removeStaleSimctlListDevicesLockWithCleanupLock(lockPath, now);
+  } finally {
+    await releaseSimctlListDevicesLock(cleanupLockPath, cleanupMetadata);
+  }
+}
+
+async function removeStaleSimctlListDevicesLockWithCleanupLock(
+  lockPath: string,
+  now: number
+): Promise<void> {
+  const createdAt = await getSimctlListDevicesLockCreatedAt(lockPath);
+  if (createdAt === undefined) return;
+
+  if (now - createdAt < SIMCTL_LIST_DEVICES_LOCK_STALE_MS) return;
+
+  const metadata = await readSimctlListDevicesLockMetadata(lockPath);
+  if (metadata && isLockOwnedByLiveProcess(metadata, now)) return;
+
+  try {
+    if (metadata) await releaseSimctlListDevicesLock(lockPath, metadata);
+    else await unlink(lockPath);
+  } catch (err) {
+    if (!isNodeError(err) || err.code !== "ENOENT") {
+      throw new SimctlListDevicesLockError(
+        `failed to remove stale simctl list devices lock at ${lockPath}`,
+        err
+      );
+    }
+  }
+}
+
+async function withSimctlListDevicesLock<T>(fn: () => Promise<T>): Promise<T> {
+  const lockPath = simctlListDevicesLockPath;
+  if (!lockPath) return await fn();
+
+  const started = Date.now();
+
+  while (true) {
+    const metadata = await acquireSimctlListDevicesLock(lockPath, Date.now());
+    if (metadata) {
+      try {
+        return await fn();
+      } finally {
+        await releaseSimctlListDevicesLock(lockPath, metadata);
+      }
+    }
+
+    const now = Date.now();
+    await removeStaleSimctlListDevicesLock(lockPath, now);
+    const elapsed = now - started;
+    if (elapsed >= SIMCTL_LIST_DEVICES_LOCK_WAIT_MS) {
+      throw new SimctlListDevicesLockError(
+        `timed out waiting for simctl list devices lock after ${SIMCTL_LIST_DEVICES_LOCK_WAIT_MS}ms`
+      );
+    }
+    await sleep(
+      Math.min(SIMCTL_LIST_DEVICES_LOCK_RETRY_MS, SIMCTL_LIST_DEVICES_LOCK_WAIT_MS - elapsed)
+    );
+  }
+}
+
+/**
+ * Read CoreSimulator's device inventory with a host-wide cap: even if multiple
+ * tool-server processes are alive, only one of them should run
+ * `xcrun simctl list devices --json` at a time.
+ */
+export async function readSimctlDevices(): Promise<SimctlOutput> {
+  return await withSimctlListDevicesLock(async () => {
+    const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "devices", "--json"], {
+      timeout: SIMCTL_LIST_DEVICES_TIMEOUT_MS,
+    });
+    return JSON.parse(stdout) as SimctlOutput;
+  });
+}
+
 /**
  * List all available iOS and tvOS simulators via `xcrun simctl list devices --json`.
  * Returns an empty array when xcrun is missing or the call fails so the
  * rest of the tool surface stays usable on non-mac hosts.
  */
 export async function listIosSimulators(): Promise<IosSimulator[]> {
   try {
-    const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "devices", "--json"], {
-      timeout: 10_000,
-    });
-    const data: SimctlOutput = JSON.parse(stdout);
+    const data = await readSimctlDevices();
     const out: IosSimulator[] = [];
     for (const [runtimeId, devices] of Object.entries(data.devices)) {
       // Accept both iOS and tvOS runtimes
@@ -51,7 +305,8 @@ export async function listIosSimulators(): Promise<IosSimulator[]> {
       }
     }
     return out;
-  } catch {
+  } catch (err) {
+    if (isSimctlListDevicesLockError(err)) throw err;
     return [];
   }
 }
@@ -113,3 +368,20 @@ export function getCachedSimulatorRuntimeKind(udid: string): "mobile" | "tv" | u
 export function __resetSimulatorRuntimeKindCacheForTesting(): void {
   runtimeKindCache.clear();
 }
+
+/** Test-only: isolate the cross-process simctl lock so parallel test workers do
+ * not contend on the real host lock. */
+export function __setSimctlListDevicesLockPathForTesting(path: string | null): void {
+  simctlListDevicesLockPath = path;
+}
+
+export function __resetSimctlListDevicesLockPathForTesting(): void {
+  simctlListDevicesLockPath = DEFAULT_SIMCTL_LIST_DEVICES_LOCK_PATH;
+}
+
+export async function __removeStaleSimctlListDevicesLockForTesting(
+  lockPath: string,
+  now: number
+): Promise<void> {
+  await removeStaleSimctlListDevicesLock(lockPath, now);
+}

packages/tool-server/src/utils/simctl-config.ts

@@ -5,3 +5,19 @@
  * CoreSimulatorService blocking simctl forever, so the watcher's backoff
  * would never fire). */
 export const SIMCTL_SPAWN_TIMEOUT_MS = 10_000;
+
+/** Ceiling for `xcrun simctl list devices --json`, the hot-path discovery call
+ * used by list-devices, runtime-kind resolution, and the simulator watcher. */
+export const SIMCTL_LIST_DEVICES_TIMEOUT_MS = 10_000;
+
+/** Wait briefly for another argent process' simctl-list probe to finish instead
+ * of spawning another one. This stays below list-devices' branch deadline when
+ * combined with SIMCTL_LIST_DEVICES_TIMEOUT_MS. */
+export const SIMCTL_LIST_DEVICES_LOCK_WAIT_MS = 12_000;
+
+/** Recover from a crashed process that acquired the discovery lock but never
+ * removed it. This is above the subprocess timeout so a healthy holder is not
+ * treated as stale under normal load. */
+export const SIMCTL_LIST_DEVICES_LOCK_STALE_MS = 15_000;
+
+export const SIMCTL_LIST_DEVICES_LOCK_RETRY_MS = 50;

packages/tool-server/src/utils/simulator-watcher.ts

@@ -1,21 +1,15 @@
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
 import type { Registry } from "@argent/registry";
 import {
   NATIVE_DEVTOOLS_NAMESPACE,
   nativeDevtoolsRef,
   type NativeDevtoolsApi,
 } from "../blueprints/native-devtools";
-
-const execFileAsync = promisify(execFile);
+import { readSimctlDevices } from "./ios-devices";
 
 const POLL_INTERVAL_MS = 10_000;
 
 async function getBootedUdids(): Promise<Set<string>> {
-  const { stdout } = await execFileAsync("xcrun", ["simctl", "list", "devices", "--json"]);
-  const data = JSON.parse(stdout) as {
-    devices: Record<string, Array<{ udid: string; state: string }>>;
-  };
+  const data = await readSimctlDevices();
   const udids = new Set<string>();
   for (const devices of Object.values(data.devices)) {
     for (const device of devices) {