From df88cb7289ef3f3608fb8e84e73eabdc1c2ef4eb Mon Sep 17 00:00:00 2001 From: fusion-merge-train Date: Sun, 5 Jul 2026 05:30:12 +0000 Subject: [PATCH 1/2] fix(engine): retry transient auth errors in withRateLimitRetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A long-running agent session holds its OAuth access token in memory. When the token rotates mid-run (Claude Max tokens have an ~8h lifetime), the next API call fails with 401 authentication_error and withRateLimitRetry re-throws it immediately — the task is marked failed and the operator is paged, even though the refreshed credentials make the very next call succeed. Observed recurring at every ~8h token boundary, hitting whatever task or heartbeat is in flight. Add isTransientAuthError (authentication_error / invalid authentication credentials / token_expired / oauth scope) with its own small retry budget: 2 retries at a flat ~5s delay (credential refresh completes within seconds, so the 30s -> 2min rate-limit backoff curve would just prolong the outage). Auth retries decrement the loop counter so they never consume rate-limit attempts, and the existing rate-limit path is byte-for-byte unchanged. Genuinely bad credentials still propagate after ~10s. Co-Authored-By: Claude Fable 5 --- .../src/__tests__/rate-limit-retry.test.ts | 87 +++++++++++++++++++ packages/engine/src/rate-limit-retry.ts | 60 +++++++++++-- 2 files changed, 141 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/__tests__/rate-limit-retry.test.ts b/packages/engine/src/__tests__/rate-limit-retry.test.ts index 8dadec9fb5..01f86ec81c 100644 --- a/packages/engine/src/__tests__/rate-limit-retry.test.ts +++ b/packages/engine/src/__tests__/rate-limit-retry.test.ts @@ -218,4 +218,91 @@ describe("withRateLimitRetry", () => { const result = await promise; expect(result).toBe("ok"); }); + + it("retries a transient auth error and succeeds after credential rotation", async () => { + const authErr = new Error( + '401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + ); + const fn = vi + .fn() + .mockRejectedValueOnce(authErr) + .mockResolvedValueOnce("recovered"); + const onRetry = vi.fn(); + + const promise = withRateLimitRetry(fn, { onRetry }); + + // Auth retry uses a flat ~5s delay (5000ms ±10 %), not the 30s backoff + await vi.advanceTimersByTimeAsync(6000); + + const result = await promise; + expect(result).toBe("recovered"); + expect(fn).toHaveBeenCalledTimes(2); + expect(onRetry).toHaveBeenCalledTimes(1); + expect(onRetry).toHaveBeenCalledWith(1, expect.any(Number), authErr); + }); + + it("throws after the transient-auth retry budget is exhausted", async () => { + const authErr = new Error("invalid authentication credentials"); + const fn = vi.fn().mockRejectedValue(authErr); + const onRetry = vi.fn(); + + const promise = withRateLimitRetry(fn, { onRetry }); + const assertion = expect(promise).rejects.toThrow( + "invalid authentication credentials", + ); + + for (let i = 0; i < 4; i++) { + await vi.advanceTimersByTimeAsync(6000); + } + + await assertion; + expect(fn).toHaveBeenCalledTimes(3); // initial + 2 auth retries + expect(onRetry).toHaveBeenCalledTimes(2); + }); + + it("does not let auth retries consume rate-limit attempts", async () => { + const fn = vi + .fn() + .mockRejectedValueOnce(new Error("token_expired")) + .mockRejectedValueOnce(new Error("429 too many requests")) + .mockResolvedValueOnce("ok"); + + // maxRetries: 1 — if the auth retry consumed the single rate-limit + // attempt, the 429 on the next call would exhaust the budget and throw. + const promise = withRateLimitRetry(fn, { + maxRetries: 1, + baseDelayMs: 100, + maxDelayMs: 1000, + }); + + await vi.advanceTimersByTimeAsync(6000); // auth retry delay + await vi.advanceTimersByTimeAsync(500); // rate-limit backoff + + const result = await promise; + expect(result).toBe("ok"); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it("classifies various transient auth error patterns correctly", async () => { + const patterns = [ + '{"type":"error","error":{"type":"authentication_error"}}', + "Invalid authentication credentials", + "token_expired", + "token expired", + "OAuth token does not meet scope requirements", + ]; + + for (const msg of patterns) { + const fn = vi + .fn() + .mockRejectedValueOnce(new Error(msg)) + .mockResolvedValueOnce("ok"); + + const promise = withRateLimitRetry(fn); + await vi.advanceTimersByTimeAsync(6000); + const result = await promise; + expect(result).toBe("ok"); + expect(fn).toHaveBeenCalledTimes(2); + } + }); }); diff --git a/packages/engine/src/rate-limit-retry.ts b/packages/engine/src/rate-limit-retry.ts index 5bf4629a22..905112f3bd 100644 --- a/packages/engine/src/rate-limit-retry.ts +++ b/packages/engine/src/rate-limit-retry.ts @@ -14,13 +14,39 @@ * pending retries when a task is paused, cancelled, or the engine is shutting * down — so agents don't sit in a 2-minute sleep unnecessarily. * - * **Scope:** Only rate-limit errors (as classified by `isUsageLimitError`) are - * retried. All other error types are re-thrown immediately so existing error - * handling (transient-error retry, failure marking, etc.) is unaffected. + * **Scope:** Rate-limit errors (as classified by `isUsageLimitError`) are + * retried with the backoff curve above. Transient authentication errors (as + * classified by `isTransientAuthError` — e.g. an OAuth access token rotating + * mid-run) get their own small retry budget with a short flat delay. All other + * error types are re-thrown immediately so existing error handling + * (transient-error retry, failure marking, etc.) is unaffected. */ import { isUsageLimitError } from "./usage-limit-detector.js"; +/** + * Matches transient authentication failures caused by credential rotation — + * e.g. a Claude Max OAuth access token expiring mid-run (~8 h lifetime). A + * long-running agent session holds the old token in memory; the very next + * call after the provider refreshes credentials succeeds, so these are worth + * a couple of quick retries before propagating as a task failure. + */ +const TRANSIENT_AUTH_ERROR_RE = + /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired|oauth token does not meet scope/i; + +function isTransientAuthError(message: string | undefined): boolean { + return TRANSIENT_AUTH_ERROR_RE.test(message ?? ""); +} + +/** Transient-auth retry budget — separate from the rate-limit `maxRetries`. */ +const AUTH_MAX_RETRIES = 2; +/** + * Flat delay before a transient-auth retry. A credential refresh completes + * within seconds, so the rate-limit backoff curve (30 s → 2 min) would just + * prolong the outage. + */ +const AUTH_RETRY_DELAY_MS = 5_000; + export interface RateLimitRetryOptions { /** Maximum number of retry attempts before re-throwing (default: 3). */ maxRetries?: number; @@ -45,7 +71,10 @@ export interface RateLimitRetryOptions { * * The wrapper calls `fn()`. If it throws a rate-limit error (detected via * `isUsageLimitError`), it sleeps with exponential backoff and retries up to - * `maxRetries` times. Non-rate-limit errors are re-thrown immediately. + * `maxRetries` times. If it throws a transient authentication error (detected + * via `isTransientAuthError`), it retries up to `AUTH_MAX_RETRIES` times after + * a short flat delay — this budget is separate and does not consume rate-limit + * attempts. All other errors are re-thrown immediately. * * After all retries are exhausted, the **original** error is thrown so the * caller's existing catch block can trigger the global pause via @@ -73,20 +102,39 @@ export async function withRateLimitRetry( } = options; let lastError: Error | undefined; + let authRetries = 0; for (let attempt = 0; attempt <= maxRetries; attempt++) { try { return await fn(); } catch (err: unknown) { const error = err instanceof Error ? err : new Error(String(err)); + const authError = isTransientAuthError(error.message); - // Non-rate-limit errors: re-throw immediately — no retry - if (!isUsageLimitError(error.message)) { + // Non-retryable errors: re-throw immediately — no retry + if (!isUsageLimitError(error.message) && !authError) { throw error; } lastError = error; + if (authError) { + if (authRetries >= AUTH_MAX_RETRIES || signal?.aborted) { + throw lastError; + } + authRetries++; + // Don't consume a rate-limit attempt for an auth retry + attempt--; + + const jitter = AUTH_RETRY_DELAY_MS * 0.1 * (2 * Math.random() - 1); // ±10 % + const delay = Math.max(0, Math.round(AUTH_RETRY_DELAY_MS + jitter)); + + onRetry?.(authRetries, delay, error); + + await sleep(delay, signal); + continue; + } + // All retries exhausted — throw so caller can trigger global pause if (attempt >= maxRetries) { throw lastError; From a486e0b75dbd77c339939d20b81c8e06a9a1e545 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 4 Jul 2026 23:13:19 -0700 Subject: [PATCH 2/2] fix(engine): exclude OAuth scope errors from transient-auth retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback on #1911: - Greptile P1 (blocking): OAuth scope/permission failures are permanent (operator must re-authorize), so they are removed from the transient-auth classifier. A new SCOPE_ERROR_RE exclusion runs BEFORE the transient match, so scope errors wrapped in a generic {"type":"authentication_error"} envelope are also excluded instead of being retried for ~10 s. - CodeRabbit: add a test for abort during the auth-retry sleep, covering the auth-specific short-circuit (the existing abort test only exercised the rate-limit backoff path). - Add a regression test asserting scope errors (plain text, JSON-wrapped, and OAuth error codes insufficient_scope/invalid_scope) are not retried. - Add a changeset (@runfusion/fusion: patch) — engine retry behavior ships in the published CLI bundle. - Add FNXC requirement comments encoding the retry-budget invariants (separate auth budget, flat ~5 s delay, no rate-limit-attempt consumption, abort short-circuit, scope exclusion ordering). Tests: 17/17 (rate-limit-retry). tsc --noEmit clean. eslint --fix clean. --- ...fix-transient-auth-scope-classification.md | 7 +++ .../src/__tests__/rate-limit-retry.test.ts | 43 ++++++++++++++++++- packages/engine/src/rate-limit-retry.ts | 23 +++++++++- 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 .changeset/fix-transient-auth-scope-classification.md diff --git a/.changeset/fix-transient-auth-scope-classification.md b/.changeset/fix-transient-auth-scope-classification.md new file mode 100644 index 0000000000..f25da92d1e --- /dev/null +++ b/.changeset/fix-transient-auth-scope-classification.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Retry transient OAuth token-rotation errors so in-flight agent calls survive rotation without failing the task. +category: fix +dev: withRateLimitRetry now retries transient auth errors (authentication_error, invalid credentials, token_expired) on a separate ~5s flat-delay budget that does not consume rate-limit attempts. OAuth scope/permission failures are explicitly excluded (operator must re-authorize) so they surface immediately instead of retrying pointlessly. diff --git a/packages/engine/src/__tests__/rate-limit-retry.test.ts b/packages/engine/src/__tests__/rate-limit-retry.test.ts index 01f86ec81c..1de497f3e6 100644 --- a/packages/engine/src/__tests__/rate-limit-retry.test.ts +++ b/packages/engine/src/__tests__/rate-limit-retry.test.ts @@ -289,7 +289,6 @@ describe("withRateLimitRetry", () => { "Invalid authentication credentials", "token_expired", "token expired", - "OAuth token does not meet scope requirements", ]; for (const msg of patterns) { @@ -305,4 +304,46 @@ describe("withRateLimitRetry", () => { expect(fn).toHaveBeenCalledTimes(2); } }); + + it("does not retry OAuth scope errors even when wrapped in an authentication_error envelope", async () => { + const scopeErrors = [ + "OAuth token does not meet scope requirements", + "insufficient_scope", + '{"type":"error","error":{"type":"authentication_error","message":"OAuth token does not meet scope requirements"}}', + ]; + + for (const msg of scopeErrors) { + const fn = vi.fn().mockRejectedValue(new Error(msg)); + const onRetry = vi.fn(); + + await expect(withRateLimitRetry(fn, { onRetry })).rejects.toThrow(msg); + + // Permanent scope failures must surface immediately — no retries. + expect(fn).toHaveBeenCalledTimes(1); + expect(onRetry).not.toHaveBeenCalled(); + } + }); + + it("cancels transient-auth retry sleep when abort signal fires", async () => { + const authErr = new Error( + '401 {"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}', + ); + const fn = vi.fn().mockRejectedValue(authErr); + const ac = new AbortController(); + + const promise = withRateLimitRetry(fn, { + maxRetries: 5, + signal: ac.signal, + }); + + // Let the first call fail and enter the auth-retry sleep (~5s). + await vi.advanceTimersByTimeAsync(10); + + // Abort before the 5s auth delay elapses. + ac.abort(new Error("Task paused")); + + await expect(promise).rejects.toThrow("Task paused"); + // Only the initial call — the auth retry never fires. + expect(fn).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/engine/src/rate-limit-retry.ts b/packages/engine/src/rate-limit-retry.ts index 905112f3bd..56e9ac2268 100644 --- a/packages/engine/src/rate-limit-retry.ts +++ b/packages/engine/src/rate-limit-retry.ts @@ -24,6 +24,11 @@ import { isUsageLimitError } from "./usage-limit-detector.js"; +/* +FNXC:EngineAuthRetry 2026-07-05-06:07: +A long-running agent session holds its OAuth access token in memory. Claude Max access tokens rotate mid-run (~8 h lifetime); the in-flight call fails with a 401 authentication_error even though the credentials file has already been refreshed, and the very next call succeeds. Retry these a few times so a token-boundary rotation does not surface as a spurious task-failure alert. This budget is separate from the rate-limit retry budget and must not consume rate-limit attempts. +*/ + /** * Matches transient authentication failures caused by credential rotation — * e.g. a Claude Max OAuth access token expiring mid-run (~8 h lifetime). A @@ -32,10 +37,20 @@ import { isUsageLimitError } from "./usage-limit-detector.js"; * a couple of quick retries before propagating as a task failure. */ const TRANSIENT_AUTH_ERROR_RE = - /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired|oauth token does not meet scope/i; + /"type":\s*"authentication_error"|invalid authentication credentials|token[_\s]?expired/i; + +/* +FNXC:EngineAuthRetry 2026-07-05-06:07: +OAuth scope/permission-grant failures are NOT transient — the token is valid but lacks required grants, so the operator must re-authorize the connection. Retrying would repeat the failing call for ~10 s before surfacing the real (operator-actionable) error. This exclusion runs BEFORE the transient match because providers wrap scope errors inside a generic {"type":"authentication_error"} envelope that would otherwise match TRANSIENT_AUTH_ERROR_RE and retry pointlessly. +*/ +const SCOPE_ERROR_RE = + /oauth token does not meet scope|insufficient[_\s-]?scope|invalid[_\s-]?scope/i; function isTransientAuthError(message: string | undefined): boolean { - return TRANSIENT_AUTH_ERROR_RE.test(message ?? ""); + const msg = message ?? ""; + // Permanent scope failures must surface immediately instead of retrying. + if (SCOPE_ERROR_RE.test(msg)) return false; + return TRANSIENT_AUTH_ERROR_RE.test(msg); } /** Transient-auth retry budget — separate from the rate-limit `maxRetries`. */ @@ -118,6 +133,10 @@ export async function withRateLimitRetry( lastError = error; + /* + FNXC:EngineAuthRetry 2026-07-05-06:07: + Transient-auth retries use a separate, smaller budget (AUTH_MAX_RETRIES) at a flat ~5s delay, and decrement `attempt` so they never burn a rate-limit attempt. Credential rotation completes in seconds, so the rate-limit backoff curve (30s -> 2min) would only prolong the outage. An already-aborted signal short-circuits to throw the original auth error without sleeping, matching the rate-limit path. + */ if (authError) { if (authRetries >= AUTH_MAX_RETRIES || signal?.aborted) { throw lastError;