fix(engine): retry transient auth errors in withRateLimitRetry

.changeset/fix-transient-auth-scope-classification.md

@@ -0,0 +1,7 @@
+---
+"@runfusion/fusion": patch
+---
+
+summary: Retry transient OAuth token-rotation errors so in-flight agent calls survive rotation without failing the task.
+category: fix
+dev: withRateLimitRetry now retries transient auth errors (authentication_error, invalid credentials, token_expired) on a separate ~5s flat-delay budget that does not consume rate-limit attempts. OAuth scope/permission failures are explicitly excluded (operator must re-authorize) so they surface immediately instead of retrying pointlessly.

packages/engine/src/__tests__/rate-limit-retry.test.ts

@@ -218,4 +218,132 @@ describe("withRateLimitRetry", () => {
     const result = await promise;
     expect(result).toBe("ok");
   });
+
+  it("retries a transient auth error and succeeds after credential rotation", async () => {
+    const authErr = new Error(
+      '401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
+    );
+    const fn = vi
+      .fn()
+      .mockRejectedValueOnce(authErr)
+      .mockResolvedValueOnce("recovered");
+    const onRetry = vi.fn();
+
+    const promise = withRateLimitRetry(fn, { onRetry });
+
+    // Auth retry uses a flat ~5s delay (5000ms ±10 %), not the 30s backoff
+    await vi.advanceTimersByTimeAsync(6000);
+
+    const result = await promise;
+    expect(result).toBe("recovered");
+    expect(fn).toHaveBeenCalledTimes(2);
+    expect(onRetry).toHaveBeenCalledTimes(1);
+    expect(onRetry).toHaveBeenCalledWith(1, expect.any(Number), authErr);
+  });
+
+  it("throws after the transient-auth retry budget is exhausted", async () => {
+    const authErr = new Error("invalid authentication credentials");
+    const fn = vi.fn().mockRejectedValue(authErr);
+    const onRetry = vi.fn();
+
+    const promise = withRateLimitRetry(fn, { onRetry });
+    const assertion = expect(promise).rejects.toThrow(
+      "invalid authentication credentials",
+    );
+
+    for (let i = 0; i < 4; i++) {
+      await vi.advanceTimersByTimeAsync(6000);
+    }
+
+    await assertion;
+    expect(fn).toHaveBeenCalledTimes(3); // initial + 2 auth retries
+    expect(onRetry).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not let auth retries consume rate-limit attempts", async () => {
+    const fn = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("token_expired"))
+      .mockRejectedValueOnce(new Error("429 too many requests"))
+      .mockResolvedValueOnce("ok");
+
+    // maxRetries: 1 — if the auth retry consumed the single rate-limit
+    // attempt, the 429 on the next call would exhaust the budget and throw.
+    const promise = withRateLimitRetry(fn, {
+      maxRetries: 1,
+      baseDelayMs: 100,
+      maxDelayMs: 1000,
+    });
+
+    await vi.advanceTimersByTimeAsync(6000); // auth retry delay
+    await vi.advanceTimersByTimeAsync(500); // rate-limit backoff
+
+    const result = await promise;
+    expect(result).toBe("ok");
+    expect(fn).toHaveBeenCalledTimes(3);
+  });
+
+  it("classifies various transient auth error patterns correctly", async () => {
+    const patterns = [
+      '{"type":"error","error":{"type":"authentication_error"}}',
+      "Invalid authentication credentials",
+      "token_expired",
+      "token expired",
+    ];
+
+    for (const msg of patterns) {
+      const fn = vi
+        .fn()
+        .mockRejectedValueOnce(new Error(msg))
+        .mockResolvedValueOnce("ok");
+
+      const promise = withRateLimitRetry(fn);
+      await vi.advanceTimersByTimeAsync(6000);
+      const result = await promise;
+      expect(result).toBe("ok");
+      expect(fn).toHaveBeenCalledTimes(2);
+    }
+  });
+
+  it("does not retry OAuth scope errors even when wrapped in an authentication_error envelope", async () => {
+    const scopeErrors = [
+      "OAuth token does not meet scope requirements",
+      "insufficient_scope",
+      '{"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}',
+    ];
+
+    for (const msg of scopeErrors) {
+      const fn = vi.fn().mockRejectedValue(new Error(msg));
+      const onRetry = vi.fn();
+
+      await expect(withRateLimitRetry(fn, { onRetry })).rejects.toThrow(msg);
+
+      // Permanent scope failures must surface immediately — no retries.
+      expect(fn).toHaveBeenCalledTimes(1);
+      expect(onRetry).not.toHaveBeenCalled();
+    }
+  });
+
+  it("cancels transient-auth retry sleep when abort signal fires", async () => {
+    const authErr = new Error(
+      '401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}',
+    );
+    const fn = vi.fn().mockRejectedValue(authErr);
+    const ac = new AbortController();
+
+    const promise = withRateLimitRetry(fn, {
+      maxRetries: 5,
+      signal: ac.signal,
+    });
+
+    // Let the first call fail and enter the auth-retry sleep (~5s).
+    await vi.advanceTimersByTimeAsync(10);
+
+    // Abort before the 5s auth delay elapses.
+    ac.abort(new Error("Task paused"));
+
+    await expect(promise).rejects.toThrow("Task paused");
+    // Only the initial call — the auth retry never fires.
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
 });

packages/engine/src/rate-limit-retry.ts

@@ -14,13 +14,54 @@
  * pending retries when a task is paused, cancelled, or the engine is shutting
  * down — so agents don't sit in a 2-minute sleep unnecessarily.
  *
- * **Scope:** Only rate-limit errors (as classified by `isUsageLimitError`) are
- * retried. All other error types are re-thrown immediately so existing error
- * handling (transient-error retry, failure marking, etc.) is unaffected.
+ * **Scope:** Rate-limit errors (as classified by `isUsageLimitError`) are
+ * retried with the backoff curve above. Transient authentication errors (as
+ * classified by `isTransientAuthError` — e.g. an OAuth access token rotating
+ * mid-run) get their own small retry budget with a short flat delay. All other
+ * error types are re-thrown immediately so existing error handling
+ * (transient-error retry, failure marking, etc.) is unaffected.
  */
 
 import { isUsageLimitError } from "./usage-limit-detector.js";
 
+/*
+FNXC:EngineAuthRetry 2026-07-05-06:07:
+A long-running agent session holds its OAuth access token in memory. Claude Max access tokens rotate mid-run (~8 h lifetime); the in-flight call fails with a 401 authentication_error even though the credentials file has already been refreshed, and the very next call succeeds. Retry these a few times so a token-boundary rotation does not surface as a spurious task-failure alert. This budget is separate from the rate-limit retry budget and must not consume rate-limit attempts.
+*/
+
+/**
+ * Matches transient authentication failures caused by credential rotation —
+ * e.g. a Claude Max OAuth access token expiring mid-run (~8 h lifetime). A
+ * long-running agent session holds the old token in memory; the very next
+ * call after the provider refreshes credentials succeeds, so these are worth
+ * a couple of quick retries before propagating as a task failure.
+ */
+const TRANSIENT_AUTH_ERROR_RE =
+  /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired/i;
+
+/*
+FNXC:EngineAuthRetry 2026-07-05-06:07:
+OAuth scope/permission-grant failures are NOT transient — the token is valid but lacks required grants, so the operator must re-authorize the connection. Retrying would repeat the failing call for ~10 s before surfacing the real (operator-actionable) error. This exclusion runs BEFORE the transient match because providers wrap scope errors inside a generic {"type":"authentication_error"} envelope that would otherwise match TRANSIENT_AUTH_ERROR_RE and retry pointlessly.
+*/
+const SCOPE_ERROR_RE =
+  /oauth token does not meet scope|insufficient[_\s-]?scope|invalid[_\s-]?scope/i;
+
+function isTransientAuthError(message: string | undefined): boolean {
+  const msg = message ?? "";
+  // Permanent scope failures must surface immediately instead of retrying.
+  if (SCOPE_ERROR_RE.test(msg)) return false;
+  return TRANSIENT_AUTH_ERROR_RE.test(msg);
+}
+
+/** Transient-auth retry budget — separate from the rate-limit `maxRetries`. */
+const AUTH_MAX_RETRIES = 2;
+/**
+ * Flat delay before a transient-auth retry. A credential refresh completes
+ * within seconds, so the rate-limit backoff curve (30 s → 2 min) would just
+ * prolong the outage.
+ */
+const AUTH_RETRY_DELAY_MS = 5_000;
+
 export interface RateLimitRetryOptions {
   /** Maximum number of retry attempts before re-throwing (default: 3). */
   maxRetries?: number;
@@ -45,7 +86,10 @@ export interface RateLimitRetryOptions {
  *
  * The wrapper calls `fn()`. If it throws a rate-limit error (detected via
  * `isUsageLimitError`), it sleeps with exponential backoff and retries up to
- * `maxRetries` times. Non-rate-limit errors are re-thrown immediately.
+ * `maxRetries` times. If it throws a transient authentication error (detected
+ * via `isTransientAuthError`), it retries up to `AUTH_MAX_RETRIES` times after
+ * a short flat delay — this budget is separate and does not consume rate-limit
+ * attempts. All other errors are re-thrown immediately.
  *
  * After all retries are exhausted, the **original** error is thrown so the
  * caller's existing catch block can trigger the global pause via
@@ -73,20 +117,43 @@ export async function withRateLimitRetry<T>(
   } = options;
 
   let lastError: Error | undefined;
+  let authRetries = 0;
 
   for (let attempt = 0; attempt <= maxRetries; attempt++) {
     try {
       return await fn();
     } catch (err: unknown) {
       const error = err instanceof Error ? err : new Error(String(err));
+      const authError = isTransientAuthError(error.message);
 
-      // Non-rate-limit errors: re-throw immediately — no retry
-      if (!isUsageLimitError(error.message)) {
+      // Non-retryable errors: re-throw immediately — no retry
+      if (!isUsageLimitError(error.message) && !authError) {
         throw error;
       }
 
       lastError = error;
 
+      /*
+      FNXC:EngineAuthRetry 2026-07-05-06:07:
+      Transient-auth retries use a separate, smaller budget (AUTH_MAX_RETRIES) at a flat ~5s delay, and decrement `attempt` so they never burn a rate-limit attempt. Credential rotation completes in seconds, so the rate-limit backoff curve (30s -> 2min) would only prolong the outage. An already-aborted signal short-circuits to throw the original auth error without sleeping, matching the rate-limit path.
+      */
+      if (authError) {
+        if (authRetries >= AUTH_MAX_RETRIES || signal?.aborted) {
+          throw lastError;
+        }
+        authRetries++;
+        // Don't consume a rate-limit attempt for an auth retry
+        attempt--;
+
+        const jitter = AUTH_RETRY_DELAY_MS * 0.1 * (2 * Math.random() - 1); // ±10 %
+        const delay = Math.max(0, Math.round(AUTH_RETRY_DELAY_MS + jitter));
+
+        onRetry?.(authRetries, delay, error);
+
+        await sleep(delay, signal);
+        continue;
+      }
+
       // All retries exhausted — throw so caller can trigger global pause
       if (attempt >= maxRetries) {
         throw lastError;