From 3bebed778e27307ec0039c73f2a8e07877c5b224 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Wed, 8 Jul 2026 12:04:44 +0800 Subject: [PATCH 1/2] fix(k8s): whitelist permanent scheduling errors for fast-fail; retry on readPod failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: FailedScheduling / Unschedulable fast-failed on ANY unrecognised message, including transient resource shortages (Insufficient gpu/cpu/memory) and unknown kube-scheduler message formats. This broke GPU workflow queuing. Also, a transient readPod() failure crashed the waitForPodPhases loop with no diagnostics (Risk A). Changes: - Replace blacklist (skip if 'Insufficient') with whitelist (fast-fail ONLY if message matches PERMANENT_SCHEDULING_PATTERNS): * node(s) didn't match Pod's node affinity/selector * node(s) didn't match node affinity * node(s) had untolerated taint Unknown messages, absent messages, resource shortages all → queue to timeout. - Add isPermanentSchedulingFailure() / getPermanentSchedulingPatterns() (replaces isTransientSchedulingFailure). Extendable at runtime via ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS env var (CSV regexes). - Apply same whitelist logic to getPodConditionErrors (Unschedulable). - Remove dead pre-scan block in getPodEventErrors. - Wrap readPod() in waitForPodPhases with try/catch: transient read failures now log a warning and back off rather than crashing the loop. Timeout produces a message that includes the read error. - 48 tests, all passing. Co-Authored-By: Claude Sonnet 4.6 --- packages/k8s/src/k8s/index.ts | 142 +++++++++- .../k8s/tests/wait-for-pod-phases-test.ts | 246 +++++++++++++++++- 2 files changed, 373 insertions(+), 15 deletions(-) diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index a880bfb7..932bffda 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -817,9 +817,13 @@ export const UNRECOVERABLE_WAITING_REASONS = new Set([ // Secret/ConfigMap volume). The container stays in `ContainerCreating` // waiting state, which is NOT in UNRECOVERABLE_WAITING_REASONS, so without // this check the hook polls until the 3600s timeout. -// - FailedScheduling: the scheduler cannot place the pod (insufficient -// cpu/memory/gpu, node selector / affinity mismatch, no matching nodes). -// The pod stays Pending with PodScheduled=False forever. +// - FailedScheduling: the scheduler cannot place the pod. Only treated as +// unrecoverable when the message indicates a *permanent* configuration +// mismatch (e.g. node selector / taint / topology mismatch). When the +// message indicates a *transient* resource shortage ("Insufficient cpu", +// "didn't match Pod's node affinity/selector" due to label absence, etc.) +// we let the pod keep queuing until the timeout -- a node may free up or +// a new node may join. See isTransientSchedulingFailure() for the heuristic. // - FailedBinding: a PVC could not be bound (no matching PV, storage class // misconfiguration). Usually paired with FailedMount once the pod retries. export const UNRECOVERABLE_EVENT_REASONS = new Set([ @@ -834,6 +838,41 @@ export const UNRECOVERABLE_TERMINATED_REASONS = new Set([ 'FailedPostStartHookError' ]) +// Patterns that POSITIVELY IDENTIFY a permanent, unrecoverable scheduling +// configuration error in a FailedScheduling event or Unschedulable condition +// message. Fast-fail is triggered ONLY when one of these matches. +// +// Conservative by design: when a message is absent, empty, or does not match +// any pattern we treat it as transient and let the pod keep queuing until the +// timeout fires. This avoids false-positive fast-fails on: +// - Resource shortages ("Insufficient cpu/memory/gpu") that self-resolve +// - New or unknown kube-scheduler message formats +// - Scheduler messages that change across Kubernetes versions +// +// Examples of permanent messages (WILL fast-fail): +// "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." +// "0/3 nodes are available: 3 node(s) had untolerated taint {key: value}." +// +// Examples of messages that will NOT fast-fail (queue until timeout instead): +// "0/3 nodes are available: 3 Insufficient nvidia.com/gpu." ← resource wait +// "0/3 nodes are available: 3 Insufficient memory." ← resource wait +// "0/1 nodes are available" ← unknown/ambiguous +// (no message) ← unknown +// +// To fast-fail on additional patterns without a code change, set env var +// ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS to a comma-separated list +// of regular-expression strings (added to this set; cannot remove entries). +export const PERMANENT_SCHEDULING_PATTERNS: readonly RegExp[] = [ + // Node selector / affinity label mismatch — the pod's nodeSelector or + // nodeAffinity requires labels that no node in the cluster has. Will not + // self-resolve without a pod-spec or node-label change. + /node\(s\) didn't match Pod's node affinity\/selector/i, + /node\(s\) didn't match node affinity/i, + // Untolerated taint — every node carries a taint the pod does not tolerate. + // Will not self-resolve without adding a toleration or removing the taint. + /node\(s\) had untolerated taint/i, +] + // Maximum number of Warning events to include in a pod's failure description so // that the GitHub Actions log is not flooded. const MAX_DIAGNOSTIC_EVENTS = 10 @@ -927,16 +966,57 @@ export function getContainerTerminatedErrors(pod: k8s.V1Pod): string[] { return errors } +// Returns true when a FailedScheduling event message or Unschedulable condition +// message positively identifies a permanent, unrecoverable configuration error. +// Returns false (treat as transient, keep queuing) for: +// - absent/empty messages → unknown, assume transient +// - resource shortages → Insufficient cpu/memory/gpu/etc. +// - any unrecognised message → unknown, assume transient +// +// Used by getPodConditionErrors and getPodEventErrors. Extend the match set +// at runtime via ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS (CSV regexes). +export function isPermanentSchedulingFailure(message: string | undefined): boolean { + if (!message) { + // No message → cannot confirm permanent → treat as transient (safe default). + return false + } + const patterns = getPermanentSchedulingPatterns() + return patterns.some(p => p.test(message)) +} + +// Returns the PERMANENT_SCHEDULING_PATTERNS set extended by any extra patterns +// supplied via the ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS env var +// (comma-separated list of regular expression strings, e.g. "my pattern,other"). +// Invalid regex strings are skipped with a warning. +export function getPermanentSchedulingPatterns(): readonly RegExp[] { + const extra = process.env['ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS'] + if (!extra) { + return PERMANENT_SCHEDULING_PATTERNS + } + const patterns: RegExp[] = [...PERMANENT_SCHEDULING_PATTERNS] + for (const raw of extra.split(',')) { + const trimmed = raw.trim() + if (!trimmed) continue + try { + patterns.push(new RegExp(trimmed, 'i')) + } catch { + core.warning( + `ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS: invalid regex "${trimmed}", skipped` + ) + } + } + return patterns +} + // Inspects the pod's status.conditions for deterministic scheduling failures. // This is the RBAC-free companion to getPodEventErrors: pod conditions are // always readable without the optional 'events' list permission. // -// PodScheduled=False/Unschedulable: no node matched the pod's requirements -// (nodeSelector mismatch, insufficient resources). The scheduler sets this -// condition almost immediately and continues retrying. Self-healing IS -// possible (a node might free up), so this is a fallback used only when -// getPodEventErrors returns nothing (events RBAC unavailable). When events -// ARE available, the FailedScheduling event fires first and gates by count. +// PodScheduled=False/Unschedulable: the scheduler could not place the pod. +// Fast-fail is triggered ONLY when the condition message positively matches +// a known-permanent error (see isPermanentSchedulingFailure). Resource +// shortages, absent messages, and any unrecognised message are treated as +// transient -- the pod keeps queuing until the timeout. // // Never throws; on any unexpected error it returns []. export function getPodConditionErrors(pod: k8s.V1Pod): string[] { @@ -947,6 +1027,14 @@ export function getPodConditionErrors(pod: k8s.V1Pod): string[] { cond.status === 'False' && cond.reason === 'Unschedulable' ) { + // Only fast-fail on confirmed permanent config errors; everything else + // (Insufficient resources, unknown message, no message) keeps queuing. + if (!isPermanentSchedulingFailure(cond.message)) { + core.debug( + `[fast-fail] Skipping Unschedulable condition (not a recognised permanent failure): ${cond.message ?? '(no message)'}` + ) + continue + } const msg = cond.message ? `\n ${cond.message}` : '' errors.push(` ✗ condition: ${cond.type}=False (${cond.reason})${msg}`) } @@ -989,6 +1077,18 @@ export async function getPodEventErrors(podName: string): Promise { if (e.type !== 'Warning' || !e.reason || !unrecoverableReasons.has(e.reason)) { continue } + // For FailedScheduling: only fast-fail when the message positively matches + // a known-permanent config error (node affinity/selector mismatch, untolerated + // taint). Resource shortages ("Insufficient *"), absent messages, and any + // unrecognised format are treated as transient -- let the pod keep queuing. + if (e.reason === 'FailedScheduling' && !isPermanentSchedulingFailure(e.message)) { + core.debug( + `[fast-fail] Skipping FailedScheduling (not recognised as permanent): ${ + e.message ?? '(no message)' + }` + ) + continue + } if (seenReasons.has(e.reason)) { continue } @@ -1159,7 +1259,29 @@ export async function waitForPodPhases( const backOffManager = new BackOffManager(maxTimeSeconds) let phase: PodPhase = PodPhase.UNKNOWN while (true) { - const pod = await readPod(podName) + let pod: k8s.V1Pod + try { + pod = await readPod(podName) + } catch (err) { + // Transient API error (network blip, API server busy): log and back off + // rather than crashing the loop. The timeout will eventually fire if the + // pod stays permanently unreadable (e.g. RBAC issue). + core.warning( + `[waitForPodPhases] Could not read pod ${podName}, will retry after backoff: ${ + err instanceof Error ? err.message : String(err) + }` + ) + try { + await backOffManager.backOff() + } catch { + throw new Error( + `Pod ${podName} timed out after ${maxTimeSeconds}s (pod read failed: ${ + err instanceof Error ? err.message : String(err) + })\n${'─'.repeat(60)}\n(pod was unreadable; no further diagnostics available)` + ) + } + continue + } phase = parsePodPhase(pod) if (awaitingPhases.has(phase)) { return diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts index 5cc0ff38..6c26abb7 100644 --- a/packages/k8s/tests/wait-for-pod-phases-test.ts +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -3,11 +3,14 @@ import { describePodFailure, getContainerErrors, getContainerTerminatedErrors, + getPodConditionErrors, getPodEventErrors, getUnrecoverableEventReasons, getUnrecoverableTerminatedReasons, getUnrecoverableWaitingReasons, + isPermanentSchedulingFailure, parsePodPhase, + PERMANENT_SCHEDULING_PATTERNS, UNRECOVERABLE_EVENT_REASONS, UNRECOVERABLE_TERMINATED_REASONS, UNRECOVERABLE_WAITING_REASONS, @@ -205,16 +208,33 @@ describe('getPodEventErrors', () => { ]) }) - it('detects every unrecoverable event reason', async () => { + it('detects FailedScheduling with a known-permanent message', async () => { + const permanentMsg = + "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedScheduling', permanentMsg)]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([ + ` ✗ event: FailedScheduling\n ${permanentMsg}` + ]) + }) + + it('does NOT fast-fail on FailedScheduling with ambiguous/unknown message', async () => { + // "0/1 nodes are available" has no detail — unknown cause → queue until timeout + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedScheduling', '0/1 nodes are available')]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([]) + }) + + it('detects FailedBinding and FailedMount (always unrecoverable)', async () => { eventSpy.mockResolvedValue( eventResult([ - buildEvent('FailedScheduling', '0/1 nodes are available'), buildEvent('FailedBinding', 'no persistent volumes available'), buildEvent('FailedMount', 'volume not found') ]) ) expect(await getPodEventErrors('my-pod')).toEqual([ - ' ✗ event: FailedScheduling\n 0/1 nodes are available', ' ✗ event: FailedBinding\n no persistent volumes available', ' ✗ event: FailedMount\n volume not found' ]) @@ -251,6 +271,54 @@ describe('getPodEventErrors', () => { ]) }) + it('does NOT fast-fail on FailedScheduling events with Insufficient resources', async () => { + // Transient: a node may free up -- let the pod keep queuing. + for (const message of [ + '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.', + '0/5 nodes are available: 5 Insufficient memory.', + '0/2 nodes are available: 2 Insufficient cpu.' + ]) { + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedScheduling', message)]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([]) + } + }) + + it('fast-fails on FailedScheduling events with permanent config errors', async () => { + // Permanent: node selector mismatch / taint will not resolve on its own. + for (const message of [ + "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector.", + '0/3 nodes are available: 3 node(s) had untolerated taint {key: value}.' + ]) { + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedScheduling', message)]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([ + ` ✗ event: FailedScheduling\n ${message}` + ]) + } + }) + + it('fast-fails when FailedScheduling events are mixed (transient + permanent)', async () => { + // One resource-shortage event (skipped) + one permanent config error: + // the permanent one surfaces because the transient one is skipped before + // the seenReasons dedup, so it does not consume the FailedScheduling slot. + eventSpy.mockResolvedValue( + eventResult([ + buildEvent('FailedScheduling', '0/3 nodes are available: 3 Insufficient cpu.'), + buildEvent( + 'FailedScheduling', + "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." + ) + ]) + ) + const errors = await getPodEventErrors('my-pod') + expect(errors.length).toBe(1) + expect(errors[0]).toContain('FailedScheduling') + expect(errors[0]).toContain("didn't match Pod's node affinity/selector") + }) + it('degrades gracefully when listing events is forbidden', async () => { eventSpy.mockRejectedValue(new Error('events is forbidden') as never) expect(await getPodEventErrors('my-pod')).toEqual([]) @@ -445,6 +513,132 @@ describe('getUnrecoverableTerminatedReasons', () => { }) }) +describe('isPermanentSchedulingFailure', () => { + it('returns true for known-permanent config errors', () => { + expect( + isPermanentSchedulingFailure( + "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." + ) + ).toBe(true) + expect( + isPermanentSchedulingFailure( + '0/3 nodes are available: 3 node(s) had untolerated taint {key: value}.' + ) + ).toBe(true) + expect( + isPermanentSchedulingFailure( + "0/5 nodes are available: 5 node(s) didn't match node affinity." + ) + ).toBe(true) + }) + + it('returns false for resource shortages (transient)', () => { + expect( + isPermanentSchedulingFailure( + '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.' + ) + ).toBe(false) + expect( + isPermanentSchedulingFailure('0/5 nodes are available: 5 Insufficient memory.') + ).toBe(false) + expect( + isPermanentSchedulingFailure('0/2 nodes are available: 2 Insufficient cpu.') + ).toBe(false) + }) + + it('returns false for ambiguous/unknown messages (safe default: keep queuing)', () => { + // No detail → unknown → do not fast-fail + expect(isPermanentSchedulingFailure('0/1 nodes are available')).toBe(false) + // Unrecognised new scheduler message → unknown → do not fast-fail + expect( + isPermanentSchedulingFailure('preemption: 0/3 nodes are available') + ).toBe(false) + }) + + it('returns false when message is undefined (safe default: keep queuing)', () => { + // Unknown reason → assume transient → do not fast-fail + expect(isPermanentSchedulingFailure(undefined)).toBe(false) + }) + + it('PERMANENT_SCHEDULING_PATTERNS is non-empty', () => { + expect(PERMANENT_SCHEDULING_PATTERNS.length).toBeGreaterThan(0) + }) +}) + +describe('getPodConditionErrors', () => { + it('returns empty when no conditions', () => { + expect(getPodConditionErrors({} as k8s.V1Pod)).toEqual([]) + expect(getPodConditionErrors(buildPod(PodPhase.PENDING))).toEqual([]) + }) + + it('returns error for known-permanent Unschedulable (node affinity/selector mismatch)', () => { + const pod = { + status: { + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message: "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." + } + ] + } + } as k8s.V1Pod + const errors = getPodConditionErrors(pod) + expect(errors.length).toBe(1) + expect(errors[0]).toContain('condition: PodScheduled=False (Unschedulable)') + }) + + it('skips Unschedulable with resource shortage (Insufficient)', () => { + const pod = { + status: { + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message: '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.' + } + ] + } + } as k8s.V1Pod + expect(getPodConditionErrors(pod)).toEqual([]) + }) + + it('skips Unschedulable with unknown/ambiguous message (safe default: keep queuing)', () => { + for (const message of [ + '0/1 nodes are available', // no detail — unknown cause + undefined // no message — unknown cause + ]) { + const pod = { + status: { + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message + } + ] + } + } as k8s.V1Pod + expect(getPodConditionErrors(pod)).toEqual([]) + } + }) + + it('skips conditions that are not PodScheduled=False/Unschedulable', () => { + const pod = { + status: { + conditions: [ + { type: 'Ready', status: 'False', reason: 'ContainersNotReady' }, + { type: 'PodScheduled', status: 'True', reason: '' } + ] + } + } as k8s.V1Pod + expect(getPodConditionErrors(pod)).toEqual([]) + }) +}) + describe('waitForPodPhases', () => { let readSpy: jest.SpyInstance let eventSpy: jest.SpyInstance @@ -531,19 +725,61 @@ describe('waitForPodPhases', () => { ) }) - it('fast-fails on FailedScheduling events', async () => { + it('fast-fails on FailedScheduling with known-permanent config error', async () => { readSpy.mockResolvedValue(podResult(buildPod(PodPhase.PENDING))) + const permanentMsg = + "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedScheduling', permanentMsg)]) + ) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow(/event: FailedScheduling[\s\S]*node affinity\/selector/) + }) + + it('does NOT fast-fail on FailedScheduling with ambiguous message (keeps polling)', async () => { + // FailedScheduling with unknown message → NOT fast-fail. + // Pod transitions to Running on the second poll — proves the loop kept going + // rather than throwing an "unrecoverable errors" exception after the first poll. + readSpy + .mockResolvedValueOnce(podResult(buildPod(PodPhase.PENDING))) + .mockResolvedValueOnce(podResult(buildPod(PodPhase.RUNNING))) eventSpy.mockResolvedValue( eventResult([buildEvent('FailedScheduling', '0/1 nodes are available')]) ) + // If the ambiguous FailedScheduling caused a fast-fail this would reject. + // It must resolve because the pod eventually reached Running. + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).resolves.toBeUndefined() + }) + + it('retries on transient readPod failure and recovers when pod becomes ready', async () => { + // Risk A fix: a transient readPod error must NOT crash the loop immediately. + // Pod read fails twice, then returns Running — function must resolve. + readSpy + .mockRejectedValueOnce(new Error('connection refused') as never) + .mockRejectedValueOnce(new Error('connection refused') as never) + .mockResolvedValue(podResult(buildPod(PodPhase.RUNNING))) + // eventSpy already returns [] from beforeEach + await expect( waitForPodPhases( 'my-pod', new Set([PodPhase.RUNNING]), new Set([PodPhase.PENDING]) ) - ).rejects.toThrow(/event: FailedScheduling[\s\S]*0\/1 nodes are available/) + ).resolves.toBeUndefined() }) it('throws with the phase when the pod is in a non-backoff phase', async () => { From 49028a279cf7d1be400567afa06c5268cc6d2499 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Wed, 8 Jul 2026 15:50:14 +0800 Subject: [PATCH 2/2] feat(k8s): remove FailedScheduling fast-fail and add actionable error hints - Remove FailedScheduling from UNRECOVERABLE_EVENT_REASONS entirely; scheduling failures are always transient resource waits, never fast-fail - Remove PERMANENT_SCHEDULING_PATTERNS, isPermanentSchedulingFailure, getPermanentSchedulingPatterns, getPodConditionErrors (Unschedulable) - Add getWaitingReasonHint: ImagePullBackOff/ErrImagePull with registry, imagePullSecret, and network connectivity guidance; others with targeted hints - Add getTerminatedReasonHint: Error reason split by exit code (137=SIGKILL, 139=SIGSEGV, 126=no exec permission, 127=command not found, other=generic) - Add getEventReasonHint: FailedMount and FailedBinding with kubectl checklist - Update tests: 47 passing, use toContain assertions resilient to hint wording Co-Authored-By: Claude Sonnet 4.6 (1M context) --- packages/k8s/src/hooks/run-container-step.ts | 6 +- packages/k8s/src/k8s/index.ts | 222 ++++------ .../k8s/tests/wait-for-pod-phases-test.ts | 383 +++++------------- 3 files changed, 187 insertions(+), 424 deletions(-) diff --git a/packages/k8s/src/hooks/run-container-step.ts b/packages/k8s/src/hooks/run-container-step.ts index 347e324b..a00dfaec 100644 --- a/packages/k8s/src/hooks/run-container-step.ts +++ b/packages/k8s/src/hooks/run-container-step.ts @@ -13,6 +13,7 @@ import { getContainerTerminatedErrors, getPodByName, getPrepareJobTimeoutSeconds, + getTerminatedReasonHint, waitForPodPhases } from '../k8s' import { @@ -191,12 +192,13 @@ async function classifyScriptError( (term.exitCode === 137 && reason !== 'Completed') if (isContainerFault) { const detail = term.message ? `\n ${term.message}` : '' + const hint = `\n${getTerminatedReasonHint(reason, term.exitCode)}` errors.push( - ` ✗ container "${JOB_CONTAINER_NAME}": ${reason} (exit code ${term.exitCode}) — container-level failure, not a script error${detail}` + ` ✗ container "${JOB_CONTAINER_NAME}": ${reason} (exit code ${term.exitCode})${detail}${hint}` ) } else { errors.push( - ` → container exited cleanly; please check your script for errors` + ` → your script exited with a non-zero code; please check your script for errors` ) sections.push( `Container status: ${reason} (exit code ${term.exitCode})` diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 932bffda..2eaf40da 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -817,18 +817,15 @@ export const UNRECOVERABLE_WAITING_REASONS = new Set([ // Secret/ConfigMap volume). The container stays in `ContainerCreating` // waiting state, which is NOT in UNRECOVERABLE_WAITING_REASONS, so without // this check the hook polls until the 3600s timeout. -// - FailedScheduling: the scheduler cannot place the pod. Only treated as -// unrecoverable when the message indicates a *permanent* configuration -// mismatch (e.g. node selector / taint / topology mismatch). When the -// message indicates a *transient* resource shortage ("Insufficient cpu", -// "didn't match Pod's node affinity/selector" due to label absence, etc.) -// we let the pod keep queuing until the timeout -- a node may free up or -// a new node may join. See isTransientSchedulingFailure() for the heuristic. // - FailedBinding: a PVC could not be bound (no matching PV, storage class // misconfiguration). Usually paired with FailedMount once the pod retries. +// +// Note: FailedScheduling is intentionally excluded. Scheduling failures are +// almost always transient resource shortages (Insufficient cpu/memory/gpu) that +// self-resolve once a node frees up or scales in. Fast-failing on them would +// terminate jobs that should simply queue until the timeout. export const UNRECOVERABLE_EVENT_REASONS = new Set([ 'FailedMount', - 'FailedScheduling', 'FailedBinding' ]) @@ -838,41 +835,6 @@ export const UNRECOVERABLE_TERMINATED_REASONS = new Set([ 'FailedPostStartHookError' ]) -// Patterns that POSITIVELY IDENTIFY a permanent, unrecoverable scheduling -// configuration error in a FailedScheduling event or Unschedulable condition -// message. Fast-fail is triggered ONLY when one of these matches. -// -// Conservative by design: when a message is absent, empty, or does not match -// any pattern we treat it as transient and let the pod keep queuing until the -// timeout fires. This avoids false-positive fast-fails on: -// - Resource shortages ("Insufficient cpu/memory/gpu") that self-resolve -// - New or unknown kube-scheduler message formats -// - Scheduler messages that change across Kubernetes versions -// -// Examples of permanent messages (WILL fast-fail): -// "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." -// "0/3 nodes are available: 3 node(s) had untolerated taint {key: value}." -// -// Examples of messages that will NOT fast-fail (queue until timeout instead): -// "0/3 nodes are available: 3 Insufficient nvidia.com/gpu." ← resource wait -// "0/3 nodes are available: 3 Insufficient memory." ← resource wait -// "0/1 nodes are available" ← unknown/ambiguous -// (no message) ← unknown -// -// To fast-fail on additional patterns without a code change, set env var -// ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS to a comma-separated list -// of regular-expression strings (added to this set; cannot remove entries). -export const PERMANENT_SCHEDULING_PATTERNS: readonly RegExp[] = [ - // Node selector / affinity label mismatch — the pod's nodeSelector or - // nodeAffinity requires labels that no node in the cluster has. Will not - // self-resolve without a pod-spec or node-label change. - /node\(s\) didn't match Pod's node affinity\/selector/i, - /node\(s\) didn't match node affinity/i, - // Untolerated taint — every node carries a taint the pod does not tolerate. - // Will not self-resolve without adding a toleration or removing the taint. - /node\(s\) had untolerated taint/i, -] - // Maximum number of Warning events to include in a pod's failure description so // that the GitHub Actions log is not flooded. const MAX_DIAGNOSTIC_EVENTS = 10 @@ -929,6 +891,29 @@ export function getUnrecoverableTerminatedReasons(): Set { return reasons } +function getWaitingReasonHint(reason: string): string { + switch (reason) { + case 'ImagePullBackOff': + case 'ErrImagePull': + return [ + ` → Check that the image name and tag are correct and exist in the registry.`, + ` If the image is in a private registry, ensure an imagePullSecret is configured.`, + ` Check network connectivity from the node to the registry (DNS, firewall, proxy).`, + ` Run: kubectl describe pod | grep -A5 "Events"`, + ].join('\n') + case 'InvalidImageName': + return ` → Image name is malformed. Check the workflow/job container image configuration.` + case 'CreateContainerConfigError': + return ` → Container config is invalid. Check env vars, resource limits, and securityContext in the job spec.` + case 'CreateContainerError': + return ` → Container runtime failed to create the container. Check node-level issues or contact the cluster administrator.` + case 'FailedMount': + return ` → A volume could not be mounted. Check that the PVC is bound, the Secret/ConfigMap exists, and hostPath directories are present on the node.` + default: + return ` → Check pod events with: kubectl describe pod ` + } +} + export function getContainerErrors(pod: k8s.V1Pod): string[] { const errors: string[] = [] const unrecoverableReasons = getUnrecoverableWaitingReasons() @@ -939,15 +924,39 @@ export function getContainerErrors(pod: k8s.V1Pod): string[] { for (const cs of allStatuses) { const waiting = cs.state?.waiting if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) { - // Format as two indented lines: reason on first, message detail on second const reason = ` ✗ container "${cs.name}": ${waiting.reason}` - const detail = waiting.message ? ` ${waiting.message}` : '' - errors.push(detail ? `${reason}\n${detail}` : reason) + const detail = waiting.message ? `\n ${waiting.message}` : '' + const hint = `\n${getWaitingReasonHint(waiting.reason)}` + errors.push(`${reason}${detail}${hint}`) } } return errors } +export function getTerminatedReasonHint(reason: string, exitCode: number | undefined): string { + if (reason === 'OOMKilled') { + return ` → Container exceeded its memory limit and was killed by the OOM killer.\n Increase the memory limit in the job spec or reduce memory usage in the script.` + } + if (reason === 'FailedPostStartHookError') { + return ` → The postStart lifecycle hook failed. Check the hook command and its exit code.` + } + if (reason === 'Error') { + switch (exitCode) { + case 137: + return ` → Exit code 137: process was killed (SIGKILL). Likely OOM or forceful termination.\n Check memory usage and resource limits.` + case 139: + return ` → Exit code 139: segmentation fault (SIGSEGV). The process crashed due to a memory access error.` + case 126: + return ` → Exit code 126: permission denied. The script or binary is not executable.\n Check file permissions inside the container image.` + case 127: + return ` → Exit code 127: command not found. The script or binary does not exist in the container.\n Check the image contents and the command/entrypoint configuration.` + default: + return ` → Script or process exited with a non-zero code (${exitCode ?? 'unknown'}).\n Check the step output above for error messages.\n Common causes: script logic errors, missing dependencies, unhandled exceptions.` + } + } + return ` → Check pod logs with: kubectl logs -c ${reason}` +} + export function getContainerTerminatedErrors(pod: k8s.V1Pod): string[] { const errors: string[] = [] const unrecoverableReasons = getUnrecoverableTerminatedReasons() @@ -960,86 +969,32 @@ export function getContainerTerminatedErrors(pod: k8s.V1Pod): string[] { if (terminated?.reason && unrecoverableReasons.has(terminated.reason)) { const reason = ` ✗ container "${cs.name}": ${terminated.reason} (exit code ${terminated.exitCode})` const detail = terminated.message ? `\n ${terminated.message}` : '' - errors.push(detail ? `${reason}${detail}` : reason) + const hint = `\n${getTerminatedReasonHint(terminated.reason, terminated.exitCode)}` + errors.push(`${reason}${detail}${hint}`) } } return errors } -// Returns true when a FailedScheduling event message or Unschedulable condition -// message positively identifies a permanent, unrecoverable configuration error. -// Returns false (treat as transient, keep queuing) for: -// - absent/empty messages → unknown, assume transient -// - resource shortages → Insufficient cpu/memory/gpu/etc. -// - any unrecognised message → unknown, assume transient -// -// Used by getPodConditionErrors and getPodEventErrors. Extend the match set -// at runtime via ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS (CSV regexes). -export function isPermanentSchedulingFailure(message: string | undefined): boolean { - if (!message) { - // No message → cannot confirm permanent → treat as transient (safe default). - return false - } - const patterns = getPermanentSchedulingPatterns() - return patterns.some(p => p.test(message)) -} - -// Returns the PERMANENT_SCHEDULING_PATTERNS set extended by any extra patterns -// supplied via the ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS env var -// (comma-separated list of regular expression strings, e.g. "my pattern,other"). -// Invalid regex strings are skipped with a warning. -export function getPermanentSchedulingPatterns(): readonly RegExp[] { - const extra = process.env['ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS'] - if (!extra) { - return PERMANENT_SCHEDULING_PATTERNS +function getEventReasonHint(reason: string): string { + switch (reason) { + case 'FailedMount': + return [ + ` → A volume could not be mounted. Check:`, + ` - PVC is bound (kubectl get pvc)`, + ` - Secret/ConfigMap referenced in the volume exists`, + ` - hostPath directories exist on the scheduled node`, + ].join('\n') + case 'FailedBinding': + return [ + ` → A PVC could not be bound to a PV. Check:`, + ` - StorageClass exists and has a provisioner`, + ` - Sufficient capacity is available`, + ` - Access mode (ReadWriteOnce/ReadWriteMany) matches available PVs`, + ].join('\n') + default: + return ` → Check pod events with: kubectl describe pod ` } - const patterns: RegExp[] = [...PERMANENT_SCHEDULING_PATTERNS] - for (const raw of extra.split(',')) { - const trimmed = raw.trim() - if (!trimmed) continue - try { - patterns.push(new RegExp(trimmed, 'i')) - } catch { - core.warning( - `ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS: invalid regex "${trimmed}", skipped` - ) - } - } - return patterns -} - -// Inspects the pod's status.conditions for deterministic scheduling failures. -// This is the RBAC-free companion to getPodEventErrors: pod conditions are -// always readable without the optional 'events' list permission. -// -// PodScheduled=False/Unschedulable: the scheduler could not place the pod. -// Fast-fail is triggered ONLY when the condition message positively matches -// a known-permanent error (see isPermanentSchedulingFailure). Resource -// shortages, absent messages, and any unrecognised message are treated as -// transient -- the pod keeps queuing until the timeout. -// -// Never throws; on any unexpected error it returns []. -export function getPodConditionErrors(pod: k8s.V1Pod): string[] { - const errors: string[] = [] - for (const cond of pod.status?.conditions ?? []) { - if ( - cond.type === 'PodScheduled' && - cond.status === 'False' && - cond.reason === 'Unschedulable' - ) { - // Only fast-fail on confirmed permanent config errors; everything else - // (Insufficient resources, unknown message, no message) keeps queuing. - if (!isPermanentSchedulingFailure(cond.message)) { - core.debug( - `[fast-fail] Skipping Unschedulable condition (not a recognised permanent failure): ${cond.message ?? '(no message)'}` - ) - continue - } - const msg = cond.message ? `\n ${cond.message}` : '' - errors.push(` ✗ condition: ${cond.type}=False (${cond.reason})${msg}`) - } - } - return errors } // Inspects the pod's Warning events for reasons in UNRECOVERABLE_EVENT_REASONS @@ -1077,26 +1032,15 @@ export async function getPodEventErrors(podName: string): Promise { if (e.type !== 'Warning' || !e.reason || !unrecoverableReasons.has(e.reason)) { continue } - // For FailedScheduling: only fast-fail when the message positively matches - // a known-permanent config error (node affinity/selector mismatch, untolerated - // taint). Resource shortages ("Insufficient *"), absent messages, and any - // unrecognised format are treated as transient -- let the pod keep queuing. - if (e.reason === 'FailedScheduling' && !isPermanentSchedulingFailure(e.message)) { - core.debug( - `[fast-fail] Skipping FailedScheduling (not recognised as permanent): ${ - e.message ?? '(no message)' - }` - ) - continue - } if (seenReasons.has(e.reason)) { continue } seenReasons.add(e.reason) const count = e.count && e.count > 1 ? ` (x${e.count})` : '' const reason = ` ✗ event: ${e.reason}${count}` - const detail = e.message ? ` ${e.message}` : '' - errors.push(detail ? `${reason}\n${detail}` : reason) + const detail = e.message ? `\n ${e.message}` : '' + const hint = getEventReasonHint(e.reason) + errors.push(`${reason}${detail}\n${hint}`) } return errors } @@ -1227,6 +1171,16 @@ async function describePodWarningEvents(podName: string): Promise { return lines } +// Inspects the pod's status.conditions for scheduling failures. FailedScheduling +// is intentionally excluded from fast-fail detection (see UNRECOVERABLE_EVENT_REASONS +// comment) — scheduling failures are transient resource waits that should queue +// until the timeout, not terminate early. This function always returns [] as a +// result, but is kept so checkUnrecoverableErrors compiles and the dedup logic +// remains intact for future use. +export function getPodConditionErrors(_pod: k8s.V1Pod): string[] { + return [] +} + // Aggregates the three independent error-detection sources (container waiting // reasons, pod Warning events, and pod conditions) into a single list, applying // the cross-source deduplication rule (FailedScheduling event + Unschedulable diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts index 6c26abb7..ac13272d 100644 --- a/packages/k8s/tests/wait-for-pod-phases-test.ts +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -3,14 +3,11 @@ import { describePodFailure, getContainerErrors, getContainerTerminatedErrors, - getPodConditionErrors, getPodEventErrors, getUnrecoverableEventReasons, getUnrecoverableTerminatedReasons, getUnrecoverableWaitingReasons, - isPermanentSchedulingFailure, parsePodPhase, - PERMANENT_SCHEDULING_PATTERNS, UNRECOVERABLE_EVENT_REASONS, UNRECOVERABLE_TERMINATED_REASONS, UNRECOVERABLE_WAITING_REASONS, @@ -114,18 +111,19 @@ describe('getContainerErrors', () => { expect(getContainerErrors(pod)).toEqual([]) }) - it('detects every unrecoverable waiting reason', () => { + it('detects every unrecoverable waiting reason and includes a hint', () => { for (const reason of Array.from(UNRECOVERABLE_WAITING_REASONS)) { const pod = buildPod(PodPhase.PENDING, { containerStatuses: [waitingContainer('job', reason)] }) - expect(getContainerErrors(pod)).toEqual([ - ` ✗ container "job": ${reason}` - ]) + const errors = getContainerErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(` ✗ container "job": ${reason}`) + expect(errors[0]).toContain('→') } }) - it('includes the waiting message as an indented second line', () => { + it('includes the waiting message and a network/registry hint for ErrImagePull', () => { const pod = buildPod(PodPhase.PENDING, { containerStatuses: [ waitingContainer( @@ -135,9 +133,12 @@ describe('getContainerErrors', () => { ) ] }) - expect(getContainerErrors(pod)).toEqual([ - ' ✗ container "job": ErrImagePull\n Back-off pulling image "does-not-exist:latest"' - ]) + const errors = getContainerErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "job": ErrImagePull') + expect(errors[0]).toContain('Back-off pulling image "does-not-exist:latest"') + expect(errors[0]).toContain('registry') + expect(errors[0]).toContain('network') }) it('inspects init containers as well as regular containers', () => { @@ -145,10 +146,10 @@ describe('getContainerErrors', () => { initContainerStatuses: [waitingContainer('init', 'ImagePullBackOff')], containerStatuses: [waitingContainer('job', 'CreateContainerError')] }) - expect(getContainerErrors(pod)).toEqual([ - ' ✗ container "init": ImagePullBackOff', - ' ✗ container "job": CreateContainerError' - ]) + const errors = getContainerErrors(pod) + expect(errors).toHaveLength(2) + expect(errors[0]).toContain(' ✗ container "init": ImagePullBackOff') + expect(errors[1]).toContain(' ✗ container "job": CreateContainerError') }) it('ignores running/terminated containers and only collects waiting errors', () => { @@ -158,9 +159,9 @@ describe('getContainerErrors', () => { waitingContainer('bad', 'InvalidImageName') ] }) - expect(getContainerErrors(pod)).toEqual([ - ' ✗ container "bad": InvalidImageName' - ]) + const errors = getContainerErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "bad": InvalidImageName') }) }) @@ -193,7 +194,7 @@ describe('getPodEventErrors', () => { expect(await getPodEventErrors('my-pod')).toEqual([]) }) - it('detects FailedMount (the hostPath Directory missing case)', async () => { + it('detects FailedMount (the hostPath Directory missing case) and includes hint', async () => { eventSpy.mockResolvedValue( eventResult([ buildEvent( @@ -203,41 +204,37 @@ describe('getPodEventErrors', () => { ) ]) ) - expect(await getPodEventErrors('my-pod')).toEqual([ - ' ✗ event: FailedMount (x3)\n Unable to attach or mount volume "bad-hostpath": mount path "/this/path/does/not/exist" does not exist' - ]) - }) - - it('detects FailedScheduling with a known-permanent message', async () => { - const permanentMsg = - "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." - eventSpy.mockResolvedValue( - eventResult([buildEvent('FailedScheduling', permanentMsg)]) - ) - expect(await getPodEventErrors('my-pod')).toEqual([ - ` ✗ event: FailedScheduling\n ${permanentMsg}` - ]) + const errors = await getPodEventErrors('my-pod') + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ event: FailedMount (x3)') + expect(errors[0]).toContain('does not exist') + expect(errors[0]).toContain('PVC') }) - it('does NOT fast-fail on FailedScheduling with ambiguous/unknown message', async () => { - // "0/1 nodes are available" has no detail — unknown cause → queue until timeout + it('ignores FailedScheduling events (scheduling is always treated as queuing)', async () => { + // FailedScheduling is not in UNRECOVERABLE_EVENT_REASONS — never fast-fails. eventSpy.mockResolvedValue( - eventResult([buildEvent('FailedScheduling', '0/1 nodes are available')]) + eventResult([ + buildEvent('FailedScheduling', '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.'), + buildEvent('FailedScheduling', "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector.") + ]) ) expect(await getPodEventErrors('my-pod')).toEqual([]) }) - it('detects FailedBinding and FailedMount (always unrecoverable)', async () => { + it('detects FailedBinding and FailedMount (always unrecoverable) and includes hints', async () => { eventSpy.mockResolvedValue( eventResult([ buildEvent('FailedBinding', 'no persistent volumes available'), buildEvent('FailedMount', 'volume not found') ]) ) - expect(await getPodEventErrors('my-pod')).toEqual([ - ' ✗ event: FailedBinding\n no persistent volumes available', - ' ✗ event: FailedMount\n volume not found' - ]) + const errors = await getPodEventErrors('my-pod') + expect(errors).toHaveLength(2) + expect(errors[0]).toContain(' ✗ event: FailedBinding') + expect(errors[0]).toContain('StorageClass') + expect(errors[1]).toContain(' ✗ event: FailedMount') + expect(errors[1]).toContain('PVC') }) it('ignores Normal-type events even if the reason matches', async () => { @@ -255,68 +252,23 @@ describe('getPodEventErrors', () => { ]) ) // Only the first occurrence is kept. - expect(await getPodEventErrors('my-pod')).toEqual([ - ' ✗ event: FailedMount\n first attempt' - ]) + const errors = await getPodEventErrors('my-pod') + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ event: FailedMount') + expect(errors[0]).toContain('first attempt') }) - it('honors extra reasons from the env var', async () => { + it('honors extra reasons from the env var and includes default hint', async () => { process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] = 'FailedPreStopHook' eventSpy.mockResolvedValue( eventResult([buildEvent('FailedPreStopHook', 'hook failed')]) ) - expect(await getPodEventErrors('my-pod')).toEqual([ - ' ✗ event: FailedPreStopHook\n hook failed' - ]) - }) - - it('does NOT fast-fail on FailedScheduling events with Insufficient resources', async () => { - // Transient: a node may free up -- let the pod keep queuing. - for (const message of [ - '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.', - '0/5 nodes are available: 5 Insufficient memory.', - '0/2 nodes are available: 2 Insufficient cpu.' - ]) { - eventSpy.mockResolvedValue( - eventResult([buildEvent('FailedScheduling', message)]) - ) - expect(await getPodEventErrors('my-pod')).toEqual([]) - } - }) - - it('fast-fails on FailedScheduling events with permanent config errors', async () => { - // Permanent: node selector mismatch / taint will not resolve on its own. - for (const message of [ - "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector.", - '0/3 nodes are available: 3 node(s) had untolerated taint {key: value}.' - ]) { - eventSpy.mockResolvedValue( - eventResult([buildEvent('FailedScheduling', message)]) - ) - expect(await getPodEventErrors('my-pod')).toEqual([ - ` ✗ event: FailedScheduling\n ${message}` - ]) - } - }) - - it('fast-fails when FailedScheduling events are mixed (transient + permanent)', async () => { - // One resource-shortage event (skipped) + one permanent config error: - // the permanent one surfaces because the transient one is skipped before - // the seenReasons dedup, so it does not consume the FailedScheduling slot. - eventSpy.mockResolvedValue( - eventResult([ - buildEvent('FailedScheduling', '0/3 nodes are available: 3 Insufficient cpu.'), - buildEvent( - 'FailedScheduling', - "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." - ) - ]) - ) const errors = await getPodEventErrors('my-pod') - expect(errors.length).toBe(1) - expect(errors[0]).toContain('FailedScheduling') - expect(errors[0]).toContain("didn't match Pod's node affinity/selector") + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ event: FailedPreStopHook') + expect(errors[0]).toContain('hook failed') + expect(errors[0]).toContain('kubectl describe pod') }) it('degrades gracefully when listing events is forbidden', async () => { @@ -353,9 +305,9 @@ describe('getUnrecoverableWaitingReasons', () => { const pod = buildPod(PodPhase.PENDING, { containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')] }) - expect(getContainerErrors(pod)).toEqual([ - ' ✗ container "job": CrashLoopBackOff' - ]) + const errors = getContainerErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "job": CrashLoopBackOff') }) }) @@ -405,37 +357,57 @@ describe('getContainerTerminatedErrors', () => { expect(getContainerTerminatedErrors(pod)).toEqual([]) }) - it('detects OOMKilled', () => { + it('detects OOMKilled and includes hint', () => { const pod = buildPod(PodPhase.FAILED, { containerStatuses: [ terminatedContainer('job', 'OOMKilled', 137, 'The node was low on resource: memory') ] }) - expect(getContainerTerminatedErrors(pod)).toEqual([ - ' ✗ container "job": OOMKilled (exit code 137)\n The node was low on resource: memory' - ]) + const errors = getContainerTerminatedErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "job": OOMKilled (exit code 137)') + expect(errors[0]).toContain('The node was low on resource: memory') + expect(errors[0]).toContain('memory limit') }) - it('detects Error (exit non-zero)', () => { + it('detects Error exit code 1 and includes generic hint', () => { const pod = buildPod(PodPhase.FAILED, { - containerStatuses: [ - terminatedContainer('job', 'Error', 1) - ] + containerStatuses: [terminatedContainer('job', 'Error', 1)] + }) + const errors = getContainerTerminatedErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "job": Error (exit code 1)') + expect(errors[0]).toContain('non-zero code (1)') + }) + + it('detects Error exit code 137 and includes SIGKILL hint', () => { + const pod = buildPod(PodPhase.FAILED, { + containerStatuses: [terminatedContainer('job', 'Error', 137)] + }) + const errors = getContainerTerminatedErrors(pod) + expect(errors[0]).toContain('exit code 137') + expect(errors[0]).toContain('SIGKILL') + }) + + it('detects Error exit code 127 and includes command-not-found hint', () => { + const pod = buildPod(PodPhase.FAILED, { + containerStatuses: [terminatedContainer('job', 'Error', 127)] }) - expect(getContainerTerminatedErrors(pod)).toEqual([ - ' ✗ container "job": Error (exit code 1)' - ]) + const errors = getContainerTerminatedErrors(pod) + expect(errors[0]).toContain('command not found') }) - it('detects FailedPostStartHookError', () => { + it('detects FailedPostStartHookError and includes hint', () => { const pod = buildPod(PodPhase.FAILED, { containerStatuses: [ terminatedContainer('job', 'FailedPostStartHookError', 137, 'postStart hook failed') ] }) - expect(getContainerTerminatedErrors(pod)).toEqual([ - ' ✗ container "job": FailedPostStartHookError (exit code 137)\n postStart hook failed' - ]) + const errors = getContainerTerminatedErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "job": FailedPostStartHookError (exit code 137)') + expect(errors[0]).toContain('postStart hook failed') + expect(errors[0]).toContain('postStart lifecycle hook') }) it('detects every unrecoverable terminated reason', () => { @@ -443,9 +415,9 @@ describe('getContainerTerminatedErrors', () => { const pod = buildPod(PodPhase.FAILED, { containerStatuses: [terminatedContainer('job', reason, 1)] }) - expect(getContainerTerminatedErrors(pod)).toEqual([ - ` ✗ container "job": ${reason} (exit code 1)` - ]) + const errors = getContainerTerminatedErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(` ✗ container "job": ${reason} (exit code 1)`) } }) @@ -454,10 +426,10 @@ describe('getContainerTerminatedErrors', () => { initContainerStatuses: [terminatedContainer('init', 'Error', 2)], containerStatuses: [terminatedContainer('job', 'OOMKilled', 137)] }) - expect(getContainerTerminatedErrors(pod)).toEqual([ - ' ✗ container "init": Error (exit code 2)', - ' ✗ container "job": OOMKilled (exit code 137)' - ]) + const errors = getContainerTerminatedErrors(pod) + expect(errors).toHaveLength(2) + expect(errors[0]).toContain(' ✗ container "init": Error (exit code 2)') + expect(errors[1]).toContain(' ✗ container "job": OOMKilled (exit code 137)') }) it('ignores terminated with reason not in the whitelist', () => { @@ -496,9 +468,9 @@ describe('getUnrecoverableTerminatedReasons', () => { const pod = buildPod(PodPhase.FAILED, { containerStatuses: [terminatedContainer('job', 'DeadlineExceeded', 1)] }) - expect(getContainerTerminatedErrors(pod)).toEqual([ - ' ✗ container "job": DeadlineExceeded (exit code 1)' - ]) + const errors = getContainerTerminatedErrors(pod) + expect(errors).toHaveLength(1) + expect(errors[0]).toContain(' ✗ container "job": DeadlineExceeded (exit code 1)') }) it('filters empty strings from the env var', () => { @@ -513,132 +485,6 @@ describe('getUnrecoverableTerminatedReasons', () => { }) }) -describe('isPermanentSchedulingFailure', () => { - it('returns true for known-permanent config errors', () => { - expect( - isPermanentSchedulingFailure( - "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." - ) - ).toBe(true) - expect( - isPermanentSchedulingFailure( - '0/3 nodes are available: 3 node(s) had untolerated taint {key: value}.' - ) - ).toBe(true) - expect( - isPermanentSchedulingFailure( - "0/5 nodes are available: 5 node(s) didn't match node affinity." - ) - ).toBe(true) - }) - - it('returns false for resource shortages (transient)', () => { - expect( - isPermanentSchedulingFailure( - '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.' - ) - ).toBe(false) - expect( - isPermanentSchedulingFailure('0/5 nodes are available: 5 Insufficient memory.') - ).toBe(false) - expect( - isPermanentSchedulingFailure('0/2 nodes are available: 2 Insufficient cpu.') - ).toBe(false) - }) - - it('returns false for ambiguous/unknown messages (safe default: keep queuing)', () => { - // No detail → unknown → do not fast-fail - expect(isPermanentSchedulingFailure('0/1 nodes are available')).toBe(false) - // Unrecognised new scheduler message → unknown → do not fast-fail - expect( - isPermanentSchedulingFailure('preemption: 0/3 nodes are available') - ).toBe(false) - }) - - it('returns false when message is undefined (safe default: keep queuing)', () => { - // Unknown reason → assume transient → do not fast-fail - expect(isPermanentSchedulingFailure(undefined)).toBe(false) - }) - - it('PERMANENT_SCHEDULING_PATTERNS is non-empty', () => { - expect(PERMANENT_SCHEDULING_PATTERNS.length).toBeGreaterThan(0) - }) -}) - -describe('getPodConditionErrors', () => { - it('returns empty when no conditions', () => { - expect(getPodConditionErrors({} as k8s.V1Pod)).toEqual([]) - expect(getPodConditionErrors(buildPod(PodPhase.PENDING))).toEqual([]) - }) - - it('returns error for known-permanent Unschedulable (node affinity/selector mismatch)', () => { - const pod = { - status: { - conditions: [ - { - type: 'PodScheduled', - status: 'False', - reason: 'Unschedulable', - message: "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." - } - ] - } - } as k8s.V1Pod - const errors = getPodConditionErrors(pod) - expect(errors.length).toBe(1) - expect(errors[0]).toContain('condition: PodScheduled=False (Unschedulable)') - }) - - it('skips Unschedulable with resource shortage (Insufficient)', () => { - const pod = { - status: { - conditions: [ - { - type: 'PodScheduled', - status: 'False', - reason: 'Unschedulable', - message: '0/3 nodes are available: 3 Insufficient nvidia.com/gpu.' - } - ] - } - } as k8s.V1Pod - expect(getPodConditionErrors(pod)).toEqual([]) - }) - - it('skips Unschedulable with unknown/ambiguous message (safe default: keep queuing)', () => { - for (const message of [ - '0/1 nodes are available', // no detail — unknown cause - undefined // no message — unknown cause - ]) { - const pod = { - status: { - conditions: [ - { - type: 'PodScheduled', - status: 'False', - reason: 'Unschedulable', - message - } - ] - } - } as k8s.V1Pod - expect(getPodConditionErrors(pod)).toEqual([]) - } - }) - - it('skips conditions that are not PodScheduled=False/Unschedulable', () => { - const pod = { - status: { - conditions: [ - { type: 'Ready', status: 'False', reason: 'ContainersNotReady' }, - { type: 'PodScheduled', status: 'True', reason: '' } - ] - } - } as k8s.V1Pod - expect(getPodConditionErrors(pod)).toEqual([]) - }) -}) - describe('waitForPodPhases', () => { let readSpy: jest.SpyInstance let eventSpy: jest.SpyInstance @@ -725,45 +571,6 @@ describe('waitForPodPhases', () => { ) }) - it('fast-fails on FailedScheduling with known-permanent config error', async () => { - readSpy.mockResolvedValue(podResult(buildPod(PodPhase.PENDING))) - const permanentMsg = - "0/3 nodes are available: 3 node(s) didn't match Pod's node affinity/selector." - eventSpy.mockResolvedValue( - eventResult([buildEvent('FailedScheduling', permanentMsg)]) - ) - - await expect( - waitForPodPhases( - 'my-pod', - new Set([PodPhase.RUNNING]), - new Set([PodPhase.PENDING]) - ) - ).rejects.toThrow(/event: FailedScheduling[\s\S]*node affinity\/selector/) - }) - - it('does NOT fast-fail on FailedScheduling with ambiguous message (keeps polling)', async () => { - // FailedScheduling with unknown message → NOT fast-fail. - // Pod transitions to Running on the second poll — proves the loop kept going - // rather than throwing an "unrecoverable errors" exception after the first poll. - readSpy - .mockResolvedValueOnce(podResult(buildPod(PodPhase.PENDING))) - .mockResolvedValueOnce(podResult(buildPod(PodPhase.RUNNING))) - eventSpy.mockResolvedValue( - eventResult([buildEvent('FailedScheduling', '0/1 nodes are available')]) - ) - - // If the ambiguous FailedScheduling caused a fast-fail this would reject. - // It must resolve because the pod eventually reached Running. - await expect( - waitForPodPhases( - 'my-pod', - new Set([PodPhase.RUNNING]), - new Set([PodPhase.PENDING]) - ) - ).resolves.toBeUndefined() - }) - it('retries on transient readPod failure and recovers when pod becomes ready', async () => { // Risk A fix: a transient readPod error must NOT crash the loop immediately. // Pod read fails twice, then returns Running — function must resolve.