From 5d7cfb07d426a7be5a3f7ca85dcc298c9d7a4ed9 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Thu, 25 Jun 2026 06:57:43 +0000 Subject: [PATCH] feat: surface unrecoverable container errors during pod wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a CI job references a non-existent image or one it lacks permission to pull, the pod stays in Pending and waitForPodPhases previously timed out with only a generic phase-status message. GitHub Actions users had no indication of the real cause. Detect unrecoverable container waiting reasons (ImagePullBackOff, ErrImagePull, InvalidImageName, CreateContainerConfigError, CreateContainerError) on both init and regular containers, and fail fast with the container name, reason, and Kubernetes message so the error is visible in the Actions log. Refactor getPodPhase into readPod + parsePodPhase so the pod object can be inspected for container errors, and add describePodFailure to aggregate pod phase, conditions, container statuses and Warning events into a single diagnostic string. describePodWarningEvents provides best-effort retrieval of recent Warning events (requires optional events RBAC permission, degrades gracefully when missing). getUnrecoverableWaitingReasons allows operators to extend the built-in fast-fail whitelist via the ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS env var. All three failure paths in waitForPodPhases now attach full diagnostics: 1. Non-backoff phase (e.g. Failed) — includes pod details + events 2. Unrecoverable container error (e.g. ImagePullBackOff) — fail-fast 3. Timeout — includes pod details so the user can see WHY the pod never became ready README: document the optional events permission and the new env var. Tests: 19 new test cases covering parsePodPhase, getContainerErrors, waitForPodPhases, getUnrecoverableWaitingReasons, describePodFailure, describePodWarningEvents, and edge cases such as forbidden events API and unreadable pods. --- packages/k8s/README.md | 18 + packages/k8s/src/k8s/index.ts | 223 ++++++++++-- .../k8s/tests/wait-for-pod-phases-test.ts | 323 ++++++++++++++++++ 3 files changed, 541 insertions(+), 23 deletions(-) create mode 100644 packages/k8s/tests/wait-for-pod-phases-test.ts diff --git a/packages/k8s/README.md b/packages/k8s/README.md index dc35c347..c2b30487 100644 --- a/packages/k8s/README.md +++ b/packages/k8s/README.md @@ -29,6 +29,16 @@ rules: resources: ["secrets"] verbs: ["get", "list", "create", "delete"] ``` +- (Optional, recommended) Granting `events` read access lets the hooks attach + recent `Warning` events (e.g. `FailedScheduling`, `FailedMount`) to the error + message when a pod fails to come online. This permission is **not required** — + if it is missing the hooks still work, they just omit the event section from + diagnostics. +``` +- apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] +``` - The `ACTIONS_RUNNER_POD_NAME` env should be set to the name of the pod - The `ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER` env should be set to true to prevent the runner from running any jobs outside of a container - The runner pod should map a persistent volume claim into the `_work` directory @@ -36,6 +46,14 @@ rules: - Some actions runner env's are expected to be set. These are set automatically by the runner. - `RUNNER_WORKSPACE` is expected to be set to the workspace of the runner - `GITHUB_WORKSPACE` is expected to be set to the workspace of the job +- Optional tuning env's + - `ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS` — comma-separated list of + extra container `waiting` reasons that should be treated as deterministic + terminal failures (fail fast instead of waiting for the timeout). These are + *added* to the built-ins (`ImagePullBackOff`, `ErrImagePull`, + `InvalidImageName`, `CreateContainerConfigError`, `CreateContainerError`); + the built-ins can never be removed. Use with care — only list reasons that + are truly unrecoverable for your workloads (e.g. `CrashLoopBackOff`). ## Limitations diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 249a0605..1ee93164 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -368,6 +368,164 @@ export async function pruneSecrets(): Promise { ) } +export const UNRECOVERABLE_WAITING_REASONS = new Set([ + 'ImagePullBackOff', + 'ErrImagePull', + 'InvalidImageName', + 'CreateContainerConfigError', + 'CreateContainerError' +]) + +const MAX_DIAGNOSTIC_EVENTS = 10 + +export function getUnrecoverableWaitingReasons(): Set { + const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + if (!extra) { + return UNRECOVERABLE_WAITING_REASONS + } + const reasons = new Set(UNRECOVERABLE_WAITING_REASONS) + for (const reason of extra.split(',')) { + const trimmed = reason.trim() + if (trimmed) { + reasons.add(trimmed) + } + } + return reasons +} + +export function getContainerErrors(pod: k8s.V1Pod): string[] { + const errors: string[] = [] + const unrecoverableReasons = getUnrecoverableWaitingReasons() + const allStatuses = [ + ...(pod.status?.initContainerStatuses ?? []), + ...(pod.status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) { + errors.push( + `container "${cs.name}": ${waiting.reason}${ + waiting.message ? ` - ${waiting.message}` : '' + }` + ) + } + } + return errors +} + +export async function describePodFailure(podName: string): Promise { + let pod: k8s.V1Pod + try { + pod = await readPod(podName) + } catch (err) { + return `Could not read pod ${podName} for diagnostics: ${ + err instanceof Error ? err.message : String(err) + }` + } + + const lines: string[] = [] + const status = pod.status + + if (status?.phase) { + lines.push( + `Phase: ${status.phase}${ + status.reason ? ` (reason: ${status.reason})` : '' + }` + ) + } + if (status?.message) { + lines.push(`Message: ${status.message}`) + } + + for (const cond of status?.conditions ?? []) { + if (cond.status === 'False') { + lines.push( + `Condition ${cond.type}=False${ + cond.reason ? ` (reason: ${cond.reason})` : '' + }${cond.message ? `: ${cond.message}` : ''}` + ) + } + } + + const allStatuses = [ + ...(status?.initContainerStatuses ?? []), + ...(status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason) { + lines.push( + `Container "${cs.name}" waiting: ${waiting.reason}${ + waiting.message ? ` - ${waiting.message}` : '' + }` + ) + } + const terminated = cs.state?.terminated + if (terminated) { + lines.push( + `Container "${cs.name}" terminated: ${ + terminated.reason ?? 'Unknown' + } (exit code ${terminated.exitCode})${ + terminated.message ? ` - ${terminated.message}` : '' + }` + ) + } + } + + lines.push(...(await describePodWarningEvents(podName))) + + if (!lines.length) { + return `No additional diagnostic information available for pod ${podName}` + } + return lines.join('\n') +} + +async function describePodWarningEvents(podName: string): Promise { + let items: k8s.CoreV1Event[] + try { + const { body } = await k8sApi.listNamespacedEvent( + namespace(), + undefined, + undefined, + undefined, + `involvedObject.name=${podName}` + ) + items = body.items + } catch (err) { + core.debug( + `Could not list events for pod ${podName} (the 'events' permission may be missing): ${ + err instanceof Error ? err.message : String(err) + }` + ) + return [] + } + + const warnings = items.filter(e => e.type === 'Warning') + if (!warnings.length) { + return [] + } + + const eventTime = (e: k8s.CoreV1Event): number => { + const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp + return t ? new Date(t).getTime() : 0 + } + warnings.sort((a, b) => eventTime(a) - eventTime(b)) + + const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS) + const lines = recent.map(e => { + const count = e.count && e.count > 1 ? ` (x${e.count})` : '' + return `Event [Warning] ${e.reason ?? ''}${count}: ${ + e.message ?? '' + }`.trim() + }) + if (warnings.length > recent.length) { + lines.unshift( + `(showing ${recent.length} of ${warnings.length} warning events)` + ) + } + return lines +} + export async function waitForPodPhases( podName: string, awaitingPhases: Set, @@ -376,22 +534,38 @@ export async function waitForPodPhases( ): Promise { const backOffManager = new BackOffManager(maxTimeSeconds) let phase: PodPhase = PodPhase.UNKNOWN - try { - while (true) { - phase = await getPodPhase(podName) - if (awaitingPhases.has(phase)) { - return - } + while (true) { + const pod = await readPod(podName) + phase = parsePodPhase(pod) + if (awaitingPhases.has(phase)) { + return + } - if (!backOffPhases.has(phase)) { - throw new Error( - `Pod ${podName} is unhealthy with phase status ${phase}` - ) - } + if (!backOffPhases.has(phase)) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy with phase status ${phase}\n${details}` + ) + } + + const containerErrors = getContainerErrors(pod) + if (containerErrors.length > 0) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} has unrecoverable container errors: ${containerErrors.join( + '; ' + )}\n${details}` + ) + } + + try { await backOffManager.backOff() + } catch (error) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy: timed out after ${maxTimeSeconds}s in phase ${phase}\n${details}` + ) } - } catch (error) { - throw new Error(`Pod ${podName} is unhealthy with phase status ${phase}`) } } @@ -414,21 +588,24 @@ export function getPrepareJobTimeoutSeconds(): number { return timeoutSeconds } -async function getPodPhase(podName: string): Promise { - const podPhaseLookup = new Set([ - PodPhase.PENDING, - PodPhase.RUNNING, - PodPhase.SUCCEEDED, - PodPhase.FAILED, - PodPhase.UNKNOWN - ]) +async function readPod(podName: string): Promise { const { body } = await k8sApi.readNamespacedPod(podName, namespace()) - const pod = body + return body +} + +const podPhaseLookup = new Set([ + PodPhase.PENDING, + PodPhase.RUNNING, + PodPhase.SUCCEEDED, + PodPhase.FAILED, + PodPhase.UNKNOWN +]) +export function parsePodPhase(pod: k8s.V1Pod): PodPhase { if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) { return PodPhase.UNKNOWN } - return pod.status?.phase as PodPhase + return pod.status.phase as PodPhase } async function isJobSucceeded(jobName: string): Promise { diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts new file mode 100644 index 00000000..30e56aa0 --- /dev/null +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -0,0 +1,323 @@ +import * as k8s from '@kubernetes/client-node' +import { + describePodFailure, + getContainerErrors, + getUnrecoverableWaitingReasons, + parsePodPhase, + UNRECOVERABLE_WAITING_REASONS, + waitForPodPhases +} from '../src/k8s' +import { PodPhase } from '../src/k8s/utils' + +function buildPod( + phase?: string, + opts: { + containerStatuses?: k8s.V1ContainerStatus[] + initContainerStatuses?: k8s.V1ContainerStatus[] + } = {} +): k8s.V1Pod { + return { + status: { + phase, + containerStatuses: opts.containerStatuses, + initContainerStatuses: opts.initContainerStatuses + } + } as k8s.V1Pod +} + +function waitingContainer( + name: string, + reason?: string, + message?: string +): k8s.V1ContainerStatus { + return { + name, + state: { waiting: { reason, message } } + } as k8s.V1ContainerStatus +} + +describe('parsePodPhase', () => { + it('returns the phase when it is a known value', () => { + expect(parsePodPhase(buildPod(PodPhase.RUNNING))).toBe(PodPhase.RUNNING) + expect(parsePodPhase(buildPod(PodPhase.PENDING))).toBe(PodPhase.PENDING) + expect(parsePodPhase(buildPod(PodPhase.SUCCEEDED))).toBe(PodPhase.SUCCEEDED) + expect(parsePodPhase(buildPod(PodPhase.FAILED))).toBe(PodPhase.FAILED) + }) + + it('returns UNKNOWN when phase is missing', () => { + expect(parsePodPhase(buildPod(undefined))).toBe(PodPhase.UNKNOWN) + expect(parsePodPhase({} as k8s.V1Pod)).toBe(PodPhase.UNKNOWN) + }) + + it('returns UNKNOWN when phase is not a recognized value', () => { + expect(parsePodPhase(buildPod('SomethingElse'))).toBe(PodPhase.UNKNOWN) + }) +}) + +describe('getContainerErrors', () => { + it('returns no errors when there are no container statuses', () => { + expect(getContainerErrors(buildPod(PodPhase.PENDING))).toEqual([]) + expect(getContainerErrors({} as k8s.V1Pod)).toEqual([]) + }) + + it('returns no errors when containers are waiting for a recoverable reason', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'ContainerCreating')] + }) + expect(getContainerErrors(pod)).toEqual([]) + }) + + it('detects every unrecoverable waiting reason', () => { + for (const reason of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', reason)] + }) + expect(getContainerErrors(pod)).toEqual([`container "job": ${reason}`]) + } + }) + + it('includes the waiting message when present', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ErrImagePull', + 'Back-off pulling image "does-not-exist:latest"' + ) + ] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "job": ErrImagePull - Back-off pulling image "does-not-exist:latest"' + ]) + }) + + it('inspects init containers as well as regular containers', () => { + const pod = buildPod(PodPhase.PENDING, { + initContainerStatuses: [waitingContainer('init', 'ImagePullBackOff')], + containerStatuses: [waitingContainer('job', 'CreateContainerError')] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "init": ImagePullBackOff', + 'container "job": CreateContainerError' + ]) + }) + + it('ignores running/terminated containers and only collects waiting errors', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + { name: 'running', state: { running: {} } } as k8s.V1ContainerStatus, + waitingContainer('bad', 'InvalidImageName') + ] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "bad": InvalidImageName' + ]) + }) +}) + +describe('waitForPodPhases', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + eventSpy.mockResolvedValue({ body: { items: [] } } as never) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('returns once the pod reaches an awaited phase', async () => { + readSpy.mockResolvedValue({ body: buildPod(PodPhase.RUNNING) } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).resolves.toBeUndefined() + }) + + it('surfaces unrecoverable container errors in the thrown message', async () => { + readSpy.mockResolvedValue({ + body: buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ImagePullBackOff', + 'Back-off pulling image "nope:latest"' + ) + ] + }) + } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow( + 'container "job": ImagePullBackOff - Back-off pulling image "nope:latest"' + ) + }) + + it('throws with the phase when the pod is in a non-backoff phase', async () => { + readSpy.mockResolvedValue({ + body: buildPod(PodPhase.FAILED) + } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow('Pod my-pod is unhealthy with phase status Failed') + }) +}) + +describe('getUnrecoverableWaitingReasons', () => { + afterEach(() => { + delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + }) + + it('returns the built-in defaults when the env var is unset', () => { + expect(getUnrecoverableWaitingReasons()).toEqual( + UNRECOVERABLE_WAITING_REASONS + ) + }) + + it('adds extra reasons from the env var without dropping the defaults', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff, RunContainerError' + const reasons = getUnrecoverableWaitingReasons() + for (const builtin of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + expect(reasons.has(builtin)).toBe(true) + } + expect(reasons.has('CrashLoopBackOff')).toBe(true) + expect(reasons.has('RunContainerError')).toBe(true) + }) + + it('makes getContainerErrors honor the extended whitelist', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff' + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "job": CrashLoopBackOff' + ]) + }) +}) + +describe('describePodFailure', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + eventSpy.mockResolvedValue({ body: { items: [] } } as never) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('reports phase, failing conditions and terminated containers', async () => { + readSpy.mockResolvedValue({ + body: { + status: { + phase: PodPhase.FAILED, + reason: 'Evicted', + message: 'The node was low on resource: memory', + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message: 'no nodes available' + } + ], + containerStatuses: [ + { + name: 'job', + state: { + terminated: { exitCode: 137, reason: 'OOMKilled' } + } + } + ] + } + } + } as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Phase: Failed (reason: Evicted)') + expect(description).toContain( + 'Message: The node was low on resource: memory' + ) + expect(description).toContain( + 'Condition PodScheduled=False (reason: Unschedulable): no nodes available' + ) + expect(description).toContain( + 'Container "job" terminated: OOMKilled (exit code 137)' + ) + }) + + it('includes recent Warning events and skips Normal ones', async () => { + readSpy.mockResolvedValue({ + body: { status: { phase: PodPhase.PENDING } } + } as never) + eventSpy.mockResolvedValue({ + body: { + items: [ + { + type: 'Normal', + reason: 'Scheduled', + message: 'assigned', + lastTimestamp: new Date('2026-01-01T00:00:00Z') + }, + { + type: 'Warning', + reason: 'FailedScheduling', + message: '0/1 nodes are available', + count: 3, + lastTimestamp: new Date('2026-01-01T00:01:00Z') + } + ] + } + } as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain( + 'Event [Warning] FailedScheduling (x3): 0/1 nodes are available' + ) + expect(description).not.toContain('Scheduled') + }) + + it('degrades gracefully when listing events is forbidden', async () => { + readSpy.mockResolvedValue({ + body: { status: { phase: PodPhase.PENDING } } + } as never) + eventSpy.mockRejectedValue(new Error('events is forbidden') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Phase: Pending') + expect(description).not.toContain('Event [Warning]') + }) + + it('never throws when the pod cannot be read', async () => { + readSpy.mockRejectedValue(new Error('pod not found') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Could not read pod my-pod for diagnostics') + }) +})