diff --git a/packages/k8s/README.md b/packages/k8s/README.md index ecc893b8..81881e87 100644 --- a/packages/k8s/README.md +++ b/packages/k8s/README.md @@ -26,6 +26,16 @@ rules: resources: ["secrets"] verbs: ["get", "list", "create", "delete"] ``` +- (Optional, recommended) Granting `events` read access lets the hooks attach + recent `Warning` events (e.g. `FailedScheduling`, `FailedMount`) to the error + message when a pod fails to come online. This permission is **not required** — + if it is missing the hooks still work, they just omit the event section from + diagnostics. +``` +- apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] +``` - The `ACTIONS_RUNNER_POD_NAME` env should be set to the name of the pod - The `ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER` env should be set to true to prevent the runner from running any jobs outside of a container - The runner pod should map a persistent volume claim into the `_work` directory @@ -33,6 +43,14 @@ rules: - Some actions runner env's are expected to be set. These are set automatically by the runner. - `RUNNER_WORKSPACE` is expected to be set to the workspace of the runner - `GITHUB_WORKSPACE` is expected to be set to the workspace of the job +- Optional tuning env's + - `ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS` — comma-separated list of + extra container `waiting` reasons that should be treated as deterministic + terminal failures (fail fast instead of waiting for the timeout). These are + *added* to the built-ins (`ImagePullBackOff`, `ErrImagePull`, + `InvalidImageName`, `CreateContainerConfigError`, `CreateContainerError`); + the built-ins can never be removed. Use with care — only list reasons that + are truly unrecoverable for your workloads (e.g. `CrashLoopBackOff`). ## Limitations diff --git a/packages/k8s/jest.setup.js b/packages/k8s/jest.setup.js index a7551362..fca39576 100644 --- a/packages/k8s/jest.setup.js +++ b/packages/k8s/jest.setup.js @@ -1,2 +1 @@ -// eslint-disable-next-line filenames/match-regex, no-undef jest.setTimeout(500000) diff --git a/packages/k8s/package.json b/packages/k8s/package.json index 6821b7c1..1a33c6aa 100644 --- a/packages/k8s/package.json +++ b/packages/k8s/package.json @@ -18,10 +18,10 @@ "@actions/io": "^1.1.3", "@kubernetes/client-node": "^1.3.0", "hooklib": "file:../hooklib", - "js-yaml": "^4.1.0", + "js-yaml": "^4.2.0", "shlex": "^3.0.0", "tar-fs": "^3.1.0", - "uuid": "^11.1.0" + "uuid": "^11.1.1" }, "devDependencies": { "@babel/core": "^7.28.3", diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts index 28453c17..ccacd851 100644 --- a/packages/k8s/src/hooks/prepare-job.ts +++ b/packages/k8s/src/hooks/prepare-job.ts @@ -85,8 +85,43 @@ export async function prepareJob( } catch (err) { await prunePods() core.debug(`createPod failed: ${JSON.stringify(err)}`) - const message = (err as any)?.response?.body?.message || err - throw new Error(`failed to create job pod: ${message}`) + // The k8s client throws HttpException whose message is a multi-line string + // containing the raw HTTP dump. Extract the human-readable "message" field + // from the embedded JSON body so the log shows something like: + // failed to create job pod: + // spec.volumes[5].name: Duplicate value: "bad-hostpath" + // instead of the full HTTP dump. + const raw = err instanceof Error ? err.message : String(err) + let detail = raw + // The k8s HttpException message is a multi-line dump: + // HTTP-Code: 422 + // Body: "{\"kind\":\"Status\",\"message\":\"...\", ...}" + // Headers: {...} + // Extract the Body JSON string, unescape it, and pull out "message". + try { + const bodyStart = raw.indexOf('Body: "') + // The boundary may be '"\nHeaders:' (real newline) or the literal + // string ends before Headers — use the last '"' before 'Headers:' as fallback + const headersIdx = raw.indexOf('Headers:') + const bodyEnd = headersIdx !== -1 + ? raw.lastIndexOf('"', headersIdx) - 0 // last " before Headers: + : raw.indexOf('"\nHeaders:') + if (bodyStart !== -1 && bodyEnd !== -1 && bodyEnd > bodyStart) { + const escaped = raw.substring(bodyStart + 7, bodyEnd) + // Body uses JSON string escaping: \" → " and \\ → \ + const unescaped = escaped + .replace(/\\\\/g, '\x00') // protect \\ first + .replace(/\\"/g, '"') // unescape \" + .replace(/\x00/g, '\\') // restore \\ + const parsed = JSON.parse(unescaped) + if (typeof parsed?.message === 'string') { + detail = parsed.message + } + } + } catch { + // Parsing failed — fall through and show the raw string + } + throw new Error(`failed to create job pod:\n ${detail}`) } if (!createdPod?.metadata?.name) { @@ -112,7 +147,11 @@ export async function prepareJob( ) } catch (err) { await prunePods() - throw new Error(`pod failed to come online with error: ${err}`) + // Unwrap nested "Error: " prefix so the message renders as: + // pod failed to come online: + // + const detail = err instanceof Error ? err.message : String(err) + throw new Error(`pod failed to come online:\n${detail}`) } await execCpToPod(createdPod.metadata.name, runnerWorkspace, '/__w') diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 36696a73..4339d3c2 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -1,4 +1,5 @@ import * as core from '@actions/core' +import * as fs from 'fs' import * as path from 'path' import { spawn } from 'child_process' import * as k8s from '@kubernetes/client-node' @@ -673,6 +674,296 @@ export async function pruneSecrets(): Promise { ) } +export const UNRECOVERABLE_WAITING_REASONS = new Set([ + 'ImagePullBackOff', + 'ErrImagePull', + 'InvalidImageName', + 'CreateContainerConfigError', + 'CreateContainerError' +]) + +// Pod *event* reasons (from the event stream, not container status) that +// indicate a permanent failure the pod will never recover from on its own. +// These surface as Warning events on the pod (visible via `kubectl describe`) +// rather than in container.status.state.waiting.reason, so they need a separate +// list + a separate (async) detection path. +// +// - FailedMount: a volume cannot be mounted (e.g. hostPath `type: Directory` +// pointing at a path that does not exist on the node, missing PVC, missing +// Secret/ConfigMap volume). The container stays in `ContainerCreating` +// waiting state, which is NOT in UNRECOVERABLE_WAITING_REASONS, so without +// this check the hook polls until the 3600s timeout. +// - FailedScheduling: the scheduler cannot place the pod (insufficient +// cpu/memory/gpu, node selector / affinity mismatch, no matching nodes). +// The pod stays Pending with PodScheduled=False forever. +// - FailedBinding: a PVC could not be bound (no matching PV, storage class +// misconfiguration). Usually paired with FailedMount once the pod retries. +export const UNRECOVERABLE_EVENT_REASONS = new Set([ + 'FailedMount', + 'FailedScheduling', + 'FailedBinding' +]) + +// Maximum number of Warning events to include in a pod's failure description so +// that the GitHub Actions log is not flooded. +const MAX_DIAGNOSTIC_EVENTS = 10 + +// The fast-fail whitelist can be extended (not narrowed) from the environment so +// operators can add deterministic terminal reasons without a code change. This +// reuses the same env-var configuration pattern as the prepare-job timeout. When +// the variable is unset, behaviour is identical to the built-in defaults. +export function getUnrecoverableWaitingReasons(): Set { + const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + if (!extra) { + return UNRECOVERABLE_WAITING_REASONS + } + const reasons = new Set(UNRECOVERABLE_WAITING_REASONS) + for (const reason of extra.split(',')) { + const trimmed = reason.trim() + if (trimmed) { + reasons.add(trimmed) + } + } + return reasons +} + +// Mirrors getUnrecoverableWaitingReasons() but for pod *event* reasons. The +// env var ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS adds to (never removes +// from) the built-in UNRECOVERABLE_EVENT_REASONS set. +export function getUnrecoverableEventReasons(): Set { + const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] + if (!extra) { + return UNRECOVERABLE_EVENT_REASONS + } + const reasons = new Set(UNRECOVERABLE_EVENT_REASONS) + for (const reason of extra.split(',')) { + const trimmed = reason.trim() + if (trimmed) { + reasons.add(trimmed) + } + } + return reasons +} + +export function getContainerErrors(pod: k8s.V1Pod): string[] { + const errors: string[] = [] + const unrecoverableReasons = getUnrecoverableWaitingReasons() + const allStatuses = [ + ...(pod.status?.initContainerStatuses ?? []), + ...(pod.status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) { + // Format as two indented lines: reason on first, message detail on second + const reason = ` ✗ container "${cs.name}": ${waiting.reason}` + const detail = waiting.message ? ` ${waiting.message}` : '' + errors.push(detail ? `${reason}\n${detail}` : reason) + } + } + return errors +} + +// Inspects the pod's status.conditions for deterministic scheduling failures. +// This is the RBAC-free companion to getPodEventErrors: pod conditions are +// always readable without the optional 'events' list permission. +// +// PodScheduled=False/Unschedulable: no node matched the pod's requirements +// (nodeSelector mismatch, insufficient resources). The scheduler sets this +// condition almost immediately and continues retrying. Self-healing IS +// possible (a node might free up), so this is a fallback used only when +// getPodEventErrors returns nothing (events RBAC unavailable). When events +// ARE available, the FailedScheduling event fires first and gates by count. +// +// Never throws; on any unexpected error it returns []. +export function getPodConditionErrors(pod: k8s.V1Pod): string[] { + const errors: string[] = [] + for (const cond of pod.status?.conditions ?? []) { + if ( + cond.type === 'PodScheduled' && + cond.status === 'False' && + cond.reason === 'Unschedulable' + ) { + const msg = cond.message ? `\n ${cond.message}` : '' + errors.push(` ✗ condition: ${cond.type}=False (${cond.reason})${msg}`) + } + } + return errors +} + +// Inspects the pod's Warning events for reasons in UNRECOVERABLE_EVENT_REASONS +// (e.g. FailedMount when a hostPath directory does not exist). Unlike container +// waiting reasons, these appear only in the event stream, so a separate API +// call is required. Best-effort: if events cannot be listed (e.g. the optional +// 'events' RBAC permission is missing) it returns an empty list instead of +// blocking fast-fail detection of container-level errors. Deduplicates by +// reason so a repeatedly retried FailedMount is reported once. +export async function getPodEventErrors(podName: string): Promise { + const unrecoverableReasons = getUnrecoverableEventReasons() + if (unrecoverableReasons.size === 0) { + return [] + } + + let items: k8s.CoreV1Event[] + try { + const result = await k8sApi.listNamespacedEvent({ + namespace: namespace(), + fieldSelector: `involvedObject.name=${podName}` + }) + items = result.items + } catch (err) { + core.debug( + `Could not list events for pod ${podName} during fast-fail check: ${ + err instanceof Error ? err.message : String(err) + }` + ) + return [] + } + + const errors: string[] = [] + const seenReasons = new Set() + for (const e of items) { + if (e.type !== 'Warning' || !e.reason || !unrecoverableReasons.has(e.reason)) { + continue + } + if (seenReasons.has(e.reason)) { + continue + } + seenReasons.add(e.reason) + const count = e.count && e.count > 1 ? ` (x${e.count})` : '' + const reason = ` ✗ event: ${e.reason}${count}` + const detail = e.message ? ` ${e.message}` : '' + errors.push(detail ? `${reason}\n${detail}` : reason) + } + return errors +} + +// describePodFailure aggregates everything that might explain why a pod is not +// healthy into a single human-readable string. It makes NO success/failure +// judgement of its own (a terminated exitCode=0 container is just reported as +// such) -- callers decide what is "bad". It never throws: if it cannot read the +// pod or list events it returns/embeds a best-effort note instead, so it is safe +// to call from any error path. +export async function describePodFailure(podName: string): Promise { + let pod: k8s.V1Pod + try { + pod = await readPod(podName) + } catch (err) { + return `Could not read pod ${podName} for diagnostics: ${ + err instanceof Error ? err.message : String(err) + }` + } + + const sections: string[] = [] + const status = pod.status + + // --- Pod-level status --- + const podLines: string[] = [] + const phaseLabel = status?.phase + ? `${status.phase}${status.reason ? ` (${status.reason})` : ''}` + : 'Unknown' + podLines.push(`Pod status: ${phaseLabel}`) + if (status?.message) { + podLines.push(` ${status.message}`) + } + + // Surface conditions that are False (e.g. PodScheduled=False from FailedScheduling) + for (const cond of status?.conditions ?? []) { + if (cond.status === 'False') { + const reason = cond.reason ? ` (${cond.reason})` : '' + const msg = cond.message ? `: ${cond.message}` : '' + podLines.push(` ✗ ${cond.type}=False${reason}${msg}`) + } + } + sections.push(podLines.join('\n')) + + // --- Container-level status (non-zero exits and non-unrecoverable waits) --- + const unrecoverableReasons = getUnrecoverableWaitingReasons() + const allStatuses = [ + ...(status?.initContainerStatuses ?? []), + ...(status?.containerStatuses ?? []) + ] + const containerLines: string[] = [] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason) { + // Skip reasons already surfaced by getContainerErrors() in the caller's + // first line to avoid printing the same error twice. + if (!unrecoverableReasons.has(waiting.reason)) { + const msg = waiting.message ? `\n ${waiting.message}` : '' + containerLines.push(` ✗ container "${cs.name}" waiting: ${waiting.reason}${msg}`) + } + } + const terminated = cs.state?.terminated + // Only surface non-zero exits; exit 0 (e.g. fs-init Completed) is noise. + if (terminated && terminated.exitCode !== 0) { + const reason = terminated.reason ?? 'Unknown' + const msg = terminated.message ? `\n ${terminated.message}` : '' + containerLines.push( + ` ✗ container "${cs.name}" terminated: ${reason} (exit code ${terminated.exitCode})${msg}` + ) + } + } + if (containerLines.length) { + sections.push(`Container details:\n${containerLines.join('\n')}`) + } + + // --- Warning events --- + const eventLines = await describePodWarningEvents(podName) + if (eventLines.length) { + sections.push(`Recent warning events:\n${eventLines.join('\n')}`) + } + + if (!sections.length) { + return `No additional diagnostic information available for pod ${podName}` + } + return sections.join('\n\n') +} + +// Reads the most recent Warning events for a pod. Best-effort: listing events +// requires the optional "events" get/list permission, so any error (including +// RBAC Forbidden) is swallowed and an empty list is returned. +async function describePodWarningEvents(podName: string): Promise { + let items: k8s.CoreV1Event[] + try { + const result = await k8sApi.listNamespacedEvent({ + namespace: namespace(), + fieldSelector: `involvedObject.name=${podName}` + }) + items = result.items + } catch (err) { + core.debug( + `Could not list events for pod ${podName} (the 'events' permission may be missing): ${ + err instanceof Error ? err.message : String(err) + }` + ) + return [] + } + + const warnings = items.filter(e => e.type === 'Warning') + if (!warnings.length) { + return [] + } + + const eventTime = (e: k8s.CoreV1Event): number => { + const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp + return t ? new Date(t).getTime() : 0 + } + warnings.sort((a, b) => eventTime(a) - eventTime(b)) + + const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS) + const lines = recent.map(e => { + const count = e.count && e.count > 1 ? ` (x${e.count})` : '' + return ` [${e.reason ?? 'Warning'}]${count} ${e.message ?? ''}`.trimEnd() + }) + if (warnings.length > recent.length) { + lines.unshift( + ` (showing ${recent.length} of ${warnings.length} warning events)` + ) + } + return lines +} + export async function waitForPodPhases( podName: string, awaitingPhases: Set, @@ -681,24 +972,58 @@ export async function waitForPodPhases( ): Promise { const backOffManager = new BackOffManager(maxTimeSeconds) let phase: PodPhase = PodPhase.UNKNOWN - try { - while (true) { - phase = await getPodPhase(podName) - if (awaitingPhases.has(phase)) { - return - } + while (true) { + const pod = await readPod(podName) + phase = parsePodPhase(pod) + if (awaitingPhases.has(phase)) { + return + } - if (!backOffPhases.has(phase)) { - throw new Error( - `Pod ${podName} is unhealthy with phase status ${phase}` - ) - } - await backOffManager.backOff() + // The pod reached a phase we are not willing to keep waiting on + // (a terminal/unhealthy phase). Attach full diagnostics and stop. + if (!backOffPhases.has(phase)) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy (phase: ${phase})\n${'─'.repeat(60)}\n${details}` + ) } - } catch (error) { - throw new Error( - `Pod ${podName} is unhealthy with phase status ${phase}: ${JSON.stringify(error)}` + + // Still in a back-off phase, but a container hit a deterministic terminal + // error (e.g. ImagePullBackOff) -- fail fast with diagnostics instead of + // waiting out the full timeout. + const containerErrors = getContainerErrors(pod) + // Check pod Warning events (FailedMount, FailedScheduling, FailedBinding). + // Best-effort: returns [] when the optional events RBAC permission is absent. + const eventErrors = await getPodEventErrors(podName) + // ALWAYS run condition check (not just as fallback when events are empty): + // conditions are set near-instantly by the scheduler while events may take + // a few extra seconds to propagate. Running both ensures we catch scheduling + // failures even if events appear slightly later. Deduplicate the output so + // the same failure isn't printed twice (event and condition both fire for + // FailedScheduling when RBAC works). + const rawConditionErrors = getPodConditionErrors(pod) + const conditionErrors = rawConditionErrors.filter( + c => !eventErrors.some(e => e.includes('FailedScheduling') && c.includes('Unschedulable')) ) + if (containerErrors.length > 0 || eventErrors.length > 0 || conditionErrors.length > 0) { + const allErrors = [...containerErrors, ...eventErrors, ...conditionErrors] + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} has unrecoverable errors:\n${allErrors.join('\n')}\n${'─'.repeat(60)}\n${details}` + ) + } + + try { + await backOffManager.backOff() + } catch (error) { + // BackOffManager throws "backoff timeout" when maxTimeSeconds is exceeded. + // Don't surface that bare message: collect diagnostics first so the user + // can see WHY the pod never became ready. + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} timed out after ${maxTimeSeconds}s (phase: ${phase})\n${'─'.repeat(60)}\n${details}` + ) + } } } @@ -721,23 +1046,26 @@ export function getPrepareJobTimeoutSeconds(): number { return timeoutSeconds } -async function getPodPhase(name: string): Promise { - const podPhaseLookup = new Set([ - PodPhase.PENDING, - PodPhase.RUNNING, - PodPhase.SUCCEEDED, - PodPhase.FAILED, - PodPhase.UNKNOWN - ]) - const pod = await k8sApi.readNamespacedPod({ - name, +async function readPod(podName: string): Promise { + return await k8sApi.readNamespacedPod({ + name: podName, namespace: namespace() }) +} +const podPhaseLookup = new Set([ + PodPhase.PENDING, + PodPhase.RUNNING, + PodPhase.SUCCEEDED, + PodPhase.FAILED, + PodPhase.UNKNOWN +]) + +export function parsePodPhase(pod: k8s.V1Pod): PodPhase { if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) { return PodPhase.UNKNOWN } - return pod.status?.phase as PodPhase + return pod.status.phase as PodPhase } async function isJobSucceeded(name: string): Promise { @@ -849,12 +1177,26 @@ export function namespace(): string { } const context = kc.getContexts().find(ctx => ctx.namespace) - if (!context?.namespace) { - throw new Error( - 'Failed to determine namespace, falling back to `default`. Namespace should be set in context, or in env variable "ACTIONS_RUNNER_KUBERNETES_NAMESPACE"' - ) + if (context?.namespace) { + return context.namespace + } + + // When running in-cluster the kubeconfig context has no namespace field; + // read it from the mounted ServiceAccount file instead. + const saNamespaceFile = + '/var/run/secrets/kubernetes.io/serviceaccount/namespace' + try { + const ns = fs.readFileSync(saNamespaceFile, 'utf8').trim() + if (ns) { + return ns + } + } catch { + // not running in-cluster, fall through to error } - return context.namespace + + throw new Error( + 'Failed to determine namespace. Set the ACTIONS_RUNNER_KUBERNETES_NAMESPACE environment variable or ensure the kubeconfig context includes a namespace.' + ) } class BackOffManager { diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts new file mode 100644 index 00000000..60758615 --- /dev/null +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -0,0 +1,516 @@ +import * as k8s from '@kubernetes/client-node' +import { + describePodFailure, + getContainerErrors, + getPodEventErrors, + getUnrecoverableEventReasons, + getUnrecoverableWaitingReasons, + parsePodPhase, + UNRECOVERABLE_EVENT_REASONS, + UNRECOVERABLE_WAITING_REASONS, + waitForPodPhases +} from '../src/k8s' +import { PodPhase } from '../src/k8s/utils' + +// Build a minimal V1Pod with the given phase and container statuses so we can +// exercise the container-error detection logic in isolation. +function buildPod( + phase?: string, + opts: { + containerStatuses?: k8s.V1ContainerStatus[] + initContainerStatuses?: k8s.V1ContainerStatus[] + } = {} +): k8s.V1Pod { + return { + status: { + phase, + containerStatuses: opts.containerStatuses, + initContainerStatuses: opts.initContainerStatuses + } + } as k8s.V1Pod +} + +function waitingContainer( + name: string, + reason?: string, + message?: string +): k8s.V1ContainerStatus { + return { + name, + state: { waiting: { reason, message } } + } as k8s.V1ContainerStatus +} + +// Build a Warning event with the fields getPodEventErrors() inspects. +function buildEvent( + reason: string, + message: string, + opts: { type?: string; count?: number } = {} +): k8s.CoreV1Event { + return { + type: opts.type ?? 'Warning', + reason, + message, + count: opts.count, + lastTimestamp: new Date('2026-01-01T00:00:00Z') + } as k8s.CoreV1Event +} + +// @kubernetes/client-node 1.x object-style API returns the object directly +// (not wrapped in { body }), so mocks must do the same. +function podResult(pod: k8s.V1Pod): never { + return pod as never +} +function eventResult(items: k8s.CoreV1Event[]): never { + return { items } as never +} + +describe('parsePodPhase', () => { + it('returns the phase when it is a known value', () => { + expect(parsePodPhase(buildPod(PodPhase.RUNNING))).toBe(PodPhase.RUNNING) + expect(parsePodPhase(buildPod(PodPhase.PENDING))).toBe(PodPhase.PENDING) + expect(parsePodPhase(buildPod(PodPhase.SUCCEEDED))).toBe(PodPhase.SUCCEEDED) + expect(parsePodPhase(buildPod(PodPhase.FAILED))).toBe(PodPhase.FAILED) + }) + + it('returns UNKNOWN when phase is missing', () => { + expect(parsePodPhase(buildPod(undefined))).toBe(PodPhase.UNKNOWN) + expect(parsePodPhase({} as k8s.V1Pod)).toBe(PodPhase.UNKNOWN) + }) + + it('returns UNKNOWN when phase is not a recognized value', () => { + expect(parsePodPhase(buildPod('SomethingElse'))).toBe(PodPhase.UNKNOWN) + }) +}) + +describe('getContainerErrors', () => { + it('returns no errors when there are no container statuses', () => { + expect(getContainerErrors(buildPod(PodPhase.PENDING))).toEqual([]) + expect(getContainerErrors({} as k8s.V1Pod)).toEqual([]) + }) + + it('returns no errors when containers are waiting for a recoverable reason', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'ContainerCreating')] + }) + expect(getContainerErrors(pod)).toEqual([]) + }) + + it('detects every unrecoverable waiting reason', () => { + for (const reason of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', reason)] + }) + expect(getContainerErrors(pod)).toEqual([ + ` ✗ container "job": ${reason}` + ]) + } + }) + + it('includes the waiting message as an indented second line', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ErrImagePull', + 'Back-off pulling image "does-not-exist:latest"' + ) + ] + }) + expect(getContainerErrors(pod)).toEqual([ + ' ✗ container "job": ErrImagePull\n Back-off pulling image "does-not-exist:latest"' + ]) + }) + + it('inspects init containers as well as regular containers', () => { + const pod = buildPod(PodPhase.PENDING, { + initContainerStatuses: [waitingContainer('init', 'ImagePullBackOff')], + containerStatuses: [waitingContainer('job', 'CreateContainerError')] + }) + expect(getContainerErrors(pod)).toEqual([ + ' ✗ container "init": ImagePullBackOff', + ' ✗ container "job": CreateContainerError' + ]) + }) + + it('ignores running/terminated containers and only collects waiting errors', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + { name: 'running', state: { running: {} } } as k8s.V1ContainerStatus, + waitingContainer('bad', 'InvalidImageName') + ] + }) + expect(getContainerErrors(pod)).toEqual([ + ' ✗ container "bad": InvalidImageName' + ]) + }) +}) + +describe('getPodEventErrors', () => { + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] + }) + + it('returns no errors when there are no warning events', async () => { + eventSpy.mockResolvedValue(eventResult([])) + expect(await getPodEventErrors('my-pod')).toEqual([]) + }) + + it('returns no errors for warning events whose reason is not unrecoverable', async () => { + eventSpy.mockResolvedValue( + eventResult([ + buildEvent('Unhealthy', 'Liveness probe failed'), + buildEvent('BackOff', 'container restart back-off') + ]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([]) + }) + + it('detects FailedMount (the hostPath Directory missing case)', async () => { + eventSpy.mockResolvedValue( + eventResult([ + buildEvent( + 'FailedMount', + 'Unable to attach or mount volume "bad-hostpath": mount path "/this/path/does/not/exist" does not exist', + { count: 3 } + ) + ]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([ + ' ✗ event: FailedMount (x3)\n Unable to attach or mount volume "bad-hostpath": mount path "/this/path/does/not/exist" does not exist' + ]) + }) + + it('detects every unrecoverable event reason', async () => { + eventSpy.mockResolvedValue( + eventResult([ + buildEvent('FailedScheduling', '0/1 nodes are available'), + buildEvent('FailedBinding', 'no persistent volumes available'), + buildEvent('FailedMount', 'volume not found') + ]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([ + ' ✗ event: FailedScheduling\n 0/1 nodes are available', + ' ✗ event: FailedBinding\n no persistent volumes available', + ' ✗ event: FailedMount\n volume not found' + ]) + }) + + it('ignores Normal-type events even if the reason matches', async () => { + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedMount', 'volume not found', { type: 'Normal' })]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([]) + }) + + it('deduplicates events that share the same reason', async () => { + eventSpy.mockResolvedValue( + eventResult([ + buildEvent('FailedMount', 'first attempt', { count: 1 }), + buildEvent('FailedMount', 'second attempt', { count: 5 }) + ]) + ) + // Only the first occurrence is kept. + expect(await getPodEventErrors('my-pod')).toEqual([ + ' ✗ event: FailedMount\n first attempt' + ]) + }) + + it('honors extra reasons from the env var', async () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] = + 'FailedPreStopHook' + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedPreStopHook', 'hook failed')]) + ) + expect(await getPodEventErrors('my-pod')).toEqual([ + ' ✗ event: FailedPreStopHook\n hook failed' + ]) + }) + + it('degrades gracefully when listing events is forbidden', async () => { + eventSpy.mockRejectedValue(new Error('events is forbidden') as never) + expect(await getPodEventErrors('my-pod')).toEqual([]) + }) +}) + +describe('getUnrecoverableWaitingReasons', () => { + afterEach(() => { + delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + }) + + it('returns the built-in defaults when the env var is unset', () => { + expect(getUnrecoverableWaitingReasons()).toEqual( + UNRECOVERABLE_WAITING_REASONS + ) + }) + + it('adds extra reasons from the env var without dropping the defaults', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff, RunContainerError' + const reasons = getUnrecoverableWaitingReasons() + for (const builtin of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + expect(reasons.has(builtin)).toBe(true) + } + expect(reasons.has('CrashLoopBackOff')).toBe(true) + expect(reasons.has('RunContainerError')).toBe(true) + }) + + it('makes getContainerErrors honor the extended whitelist', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff' + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')] + }) + expect(getContainerErrors(pod)).toEqual([ + ' ✗ container "job": CrashLoopBackOff' + ]) + }) +}) + +describe('getUnrecoverableEventReasons', () => { + afterEach(() => { + delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] + }) + + it('returns the built-in defaults when the env var is unset', () => { + expect(getUnrecoverableEventReasons()).toEqual(UNRECOVERABLE_EVENT_REASONS) + }) + + it('adds extra reasons from the env var without dropping the defaults', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] = + 'FailedPreStopHook, FailedPostStartHook' + const reasons = getUnrecoverableEventReasons() + for (const builtin of Array.from(UNRECOVERABLE_EVENT_REASONS)) { + expect(reasons.has(builtin)).toBe(true) + } + expect(reasons.has('FailedPreStopHook')).toBe(true) + expect(reasons.has('FailedPostStartHook')).toBe(true) + }) +}) + +describe('waitForPodPhases', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + // waitForPodPhases now calls getPodEventErrors (lists events) on every poll + // for fast-fail detection, and describePodFailure on failure paths (also + // lists events). Stub it out so the tests never hit a real cluster. + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + eventSpy.mockResolvedValue(eventResult([])) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('returns once the pod reaches an awaited phase', async () => { + readSpy.mockResolvedValue(podResult(buildPod(PodPhase.RUNNING))) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).resolves.toBeUndefined() + }) + + it('surfaces unrecoverable container errors in the thrown message', async () => { + readSpy.mockResolvedValue( + podResult( + buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ImagePullBackOff', + 'Back-off pulling image "nope:latest"' + ) + ] + }) + ) + ) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow('Pod my-pod has unrecoverable errors') + }) + + it('fast-fails on FailedMount events instead of polling to timeout', async () => { + // Container is in ContainerCreating (recoverable waiting reason), but the + // pod has a FailedMount Warning event -- this is the hostPath-missing case. + readSpy.mockResolvedValue( + podResult( + buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'ContainerCreating')] + }) + ) + ) + eventSpy.mockResolvedValue( + eventResult([ + buildEvent( + 'FailedMount', + 'mount path "/this/path/does/not/exist" does not exist', + { count: 4 } + ) + ]) + ) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow( + /has unrecoverable errors:[\s\S]*event: FailedMount \(x4\)/ + ) + }) + + it('fast-fails on FailedScheduling events', async () => { + readSpy.mockResolvedValue(podResult(buildPod(PodPhase.PENDING))) + eventSpy.mockResolvedValue( + eventResult([buildEvent('FailedScheduling', '0/1 nodes are available')]) + ) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow(/event: FailedScheduling[\s\S]*0\/1 nodes are available/) + }) + + it('throws with the phase when the pod is in a non-backoff phase', async () => { + readSpy.mockResolvedValue(podResult(buildPod(PodPhase.FAILED))) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow('Pod my-pod is unhealthy (phase: Failed)') + }) +}) + +describe('describePodFailure', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + // Default: no events. Individual tests override this. + eventSpy.mockResolvedValue(eventResult([])) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('reports phase, failing conditions and terminated containers', async () => { + readSpy.mockResolvedValue( + podResult({ + status: { + phase: PodPhase.FAILED, + reason: 'Evicted', + message: 'The node was low on resource: memory', + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message: 'no nodes available' + } + ], + containerStatuses: [ + { + name: 'job', + state: { + terminated: { exitCode: 137, reason: 'OOMKilled' } + } + } + ] + } + } as k8s.V1Pod) + ) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Pod status: Failed (Evicted)') + expect(description).toContain('The node was low on resource: memory') + expect(description).toContain( + '✗ PodScheduled=False (Unschedulable): no nodes available' + ) + expect(description).toContain( + '✗ container "job" terminated: OOMKilled (exit code 137)' + ) + }) + + it('includes recent Warning events and skips Normal ones', async () => { + readSpy.mockResolvedValue( + podResult({ status: { phase: PodPhase.PENDING } } as k8s.V1Pod) + ) + eventSpy.mockResolvedValue( + eventResult([ + { + type: 'Normal', + reason: 'Scheduled', + message: 'assigned', + lastTimestamp: new Date('2026-01-01T00:00:00Z') + } as k8s.CoreV1Event, + { + type: 'Warning', + reason: 'FailedScheduling', + message: '0/1 nodes are available', + count: 3, + lastTimestamp: new Date('2026-01-01T00:01:00Z') + } as k8s.CoreV1Event + ]) + ) + + const description = await describePodFailure('my-pod') + expect(description).toContain( + '[FailedScheduling] (x3) 0/1 nodes are available' + ) + expect(description).not.toContain('[Scheduled]') + }) + + it('degrades gracefully when listing events is forbidden', async () => { + readSpy.mockResolvedValue( + podResult({ status: { phase: PodPhase.PENDING } } as k8s.V1Pod) + ) + eventSpy.mockRejectedValue(new Error('events is forbidden') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Pod status: Pending') + // No throw, and no event lines. + expect(description).not.toContain('[Warning]') + }) + + it('never throws when the pod cannot be read', async () => { + readSpy.mockRejectedValue(new Error('pod not found') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Could not read pod my-pod for diagnostics') + }) +})