diff --git a/packages/k8s/README.md b/packages/k8s/README.md index dc35c347..c2b30487 100644 --- a/packages/k8s/README.md +++ b/packages/k8s/README.md @@ -29,6 +29,16 @@ rules: resources: ["secrets"] verbs: ["get", "list", "create", "delete"] ``` +- (Optional, recommended) Granting `events` read access lets the hooks attach + recent `Warning` events (e.g. `FailedScheduling`, `FailedMount`) to the error + message when a pod fails to come online. This permission is **not required** — + if it is missing the hooks still work, they just omit the event section from + diagnostics. +``` +- apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] +``` - The `ACTIONS_RUNNER_POD_NAME` env should be set to the name of the pod - The `ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER` env should be set to true to prevent the runner from running any jobs outside of a container - The runner pod should map a persistent volume claim into the `_work` directory @@ -36,6 +46,14 @@ rules: - Some actions runner env's are expected to be set. These are set automatically by the runner. - `RUNNER_WORKSPACE` is expected to be set to the workspace of the runner - `GITHUB_WORKSPACE` is expected to be set to the workspace of the job +- Optional tuning env's + - `ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS` — comma-separated list of + extra container `waiting` reasons that should be treated as deterministic + terminal failures (fail fast instead of waiting for the timeout). These are + *added* to the built-ins (`ImagePullBackOff`, `ErrImagePull`, + `InvalidImageName`, `CreateContainerConfigError`, `CreateContainerError`); + the built-ins can never be removed. Use with care — only list reasons that + are truly unrecoverable for your workloads (e.g. `CrashLoopBackOff`). ## Limitations diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 249a0605..1ee93164 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -368,6 +368,164 @@ export async function pruneSecrets(): Promise { ) } +export const UNRECOVERABLE_WAITING_REASONS = new Set([ + 'ImagePullBackOff', + 'ErrImagePull', + 'InvalidImageName', + 'CreateContainerConfigError', + 'CreateContainerError' +]) + +const MAX_DIAGNOSTIC_EVENTS = 10 + +export function getUnrecoverableWaitingReasons(): Set { + const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + if (!extra) { + return UNRECOVERABLE_WAITING_REASONS + } + const reasons = new Set(UNRECOVERABLE_WAITING_REASONS) + for (const reason of extra.split(',')) { + const trimmed = reason.trim() + if (trimmed) { + reasons.add(trimmed) + } + } + return reasons +} + +export function getContainerErrors(pod: k8s.V1Pod): string[] { + const errors: string[] = [] + const unrecoverableReasons = getUnrecoverableWaitingReasons() + const allStatuses = [ + ...(pod.status?.initContainerStatuses ?? []), + ...(pod.status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) { + errors.push( + `container "${cs.name}": ${waiting.reason}${ + waiting.message ? ` - ${waiting.message}` : '' + }` + ) + } + } + return errors +} + +export async function describePodFailure(podName: string): Promise { + let pod: k8s.V1Pod + try { + pod = await readPod(podName) + } catch (err) { + return `Could not read pod ${podName} for diagnostics: ${ + err instanceof Error ? err.message : String(err) + }` + } + + const lines: string[] = [] + const status = pod.status + + if (status?.phase) { + lines.push( + `Phase: ${status.phase}${ + status.reason ? ` (reason: ${status.reason})` : '' + }` + ) + } + if (status?.message) { + lines.push(`Message: ${status.message}`) + } + + for (const cond of status?.conditions ?? []) { + if (cond.status === 'False') { + lines.push( + `Condition ${cond.type}=False${ + cond.reason ? ` (reason: ${cond.reason})` : '' + }${cond.message ? `: ${cond.message}` : ''}` + ) + } + } + + const allStatuses = [ + ...(status?.initContainerStatuses ?? []), + ...(status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason) { + lines.push( + `Container "${cs.name}" waiting: ${waiting.reason}${ + waiting.message ? ` - ${waiting.message}` : '' + }` + ) + } + const terminated = cs.state?.terminated + if (terminated) { + lines.push( + `Container "${cs.name}" terminated: ${ + terminated.reason ?? 'Unknown' + } (exit code ${terminated.exitCode})${ + terminated.message ? ` - ${terminated.message}` : '' + }` + ) + } + } + + lines.push(...(await describePodWarningEvents(podName))) + + if (!lines.length) { + return `No additional diagnostic information available for pod ${podName}` + } + return lines.join('\n') +} + +async function describePodWarningEvents(podName: string): Promise { + let items: k8s.CoreV1Event[] + try { + const { body } = await k8sApi.listNamespacedEvent( + namespace(), + undefined, + undefined, + undefined, + `involvedObject.name=${podName}` + ) + items = body.items + } catch (err) { + core.debug( + `Could not list events for pod ${podName} (the 'events' permission may be missing): ${ + err instanceof Error ? err.message : String(err) + }` + ) + return [] + } + + const warnings = items.filter(e => e.type === 'Warning') + if (!warnings.length) { + return [] + } + + const eventTime = (e: k8s.CoreV1Event): number => { + const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp + return t ? new Date(t).getTime() : 0 + } + warnings.sort((a, b) => eventTime(a) - eventTime(b)) + + const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS) + const lines = recent.map(e => { + const count = e.count && e.count > 1 ? ` (x${e.count})` : '' + return `Event [Warning] ${e.reason ?? ''}${count}: ${ + e.message ?? '' + }`.trim() + }) + if (warnings.length > recent.length) { + lines.unshift( + `(showing ${recent.length} of ${warnings.length} warning events)` + ) + } + return lines +} + export async function waitForPodPhases( podName: string, awaitingPhases: Set, @@ -376,22 +534,38 @@ export async function waitForPodPhases( ): Promise { const backOffManager = new BackOffManager(maxTimeSeconds) let phase: PodPhase = PodPhase.UNKNOWN - try { - while (true) { - phase = await getPodPhase(podName) - if (awaitingPhases.has(phase)) { - return - } + while (true) { + const pod = await readPod(podName) + phase = parsePodPhase(pod) + if (awaitingPhases.has(phase)) { + return + } - if (!backOffPhases.has(phase)) { - throw new Error( - `Pod ${podName} is unhealthy with phase status ${phase}` - ) - } + if (!backOffPhases.has(phase)) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy with phase status ${phase}\n${details}` + ) + } + + const containerErrors = getContainerErrors(pod) + if (containerErrors.length > 0) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} has unrecoverable container errors: ${containerErrors.join( + '; ' + )}\n${details}` + ) + } + + try { await backOffManager.backOff() + } catch (error) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy: timed out after ${maxTimeSeconds}s in phase ${phase}\n${details}` + ) } - } catch (error) { - throw new Error(`Pod ${podName} is unhealthy with phase status ${phase}`) } } @@ -414,21 +588,24 @@ export function getPrepareJobTimeoutSeconds(): number { return timeoutSeconds } -async function getPodPhase(podName: string): Promise { - const podPhaseLookup = new Set([ - PodPhase.PENDING, - PodPhase.RUNNING, - PodPhase.SUCCEEDED, - PodPhase.FAILED, - PodPhase.UNKNOWN - ]) +async function readPod(podName: string): Promise { const { body } = await k8sApi.readNamespacedPod(podName, namespace()) - const pod = body + return body +} + +const podPhaseLookup = new Set([ + PodPhase.PENDING, + PodPhase.RUNNING, + PodPhase.SUCCEEDED, + PodPhase.FAILED, + PodPhase.UNKNOWN +]) +export function parsePodPhase(pod: k8s.V1Pod): PodPhase { if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) { return PodPhase.UNKNOWN } - return pod.status?.phase as PodPhase + return pod.status.phase as PodPhase } async function isJobSucceeded(jobName: string): Promise { diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts new file mode 100644 index 00000000..30e56aa0 --- /dev/null +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -0,0 +1,323 @@ +import * as k8s from '@kubernetes/client-node' +import { + describePodFailure, + getContainerErrors, + getUnrecoverableWaitingReasons, + parsePodPhase, + UNRECOVERABLE_WAITING_REASONS, + waitForPodPhases +} from '../src/k8s' +import { PodPhase } from '../src/k8s/utils' + +function buildPod( + phase?: string, + opts: { + containerStatuses?: k8s.V1ContainerStatus[] + initContainerStatuses?: k8s.V1ContainerStatus[] + } = {} +): k8s.V1Pod { + return { + status: { + phase, + containerStatuses: opts.containerStatuses, + initContainerStatuses: opts.initContainerStatuses + } + } as k8s.V1Pod +} + +function waitingContainer( + name: string, + reason?: string, + message?: string +): k8s.V1ContainerStatus { + return { + name, + state: { waiting: { reason, message } } + } as k8s.V1ContainerStatus +} + +describe('parsePodPhase', () => { + it('returns the phase when it is a known value', () => { + expect(parsePodPhase(buildPod(PodPhase.RUNNING))).toBe(PodPhase.RUNNING) + expect(parsePodPhase(buildPod(PodPhase.PENDING))).toBe(PodPhase.PENDING) + expect(parsePodPhase(buildPod(PodPhase.SUCCEEDED))).toBe(PodPhase.SUCCEEDED) + expect(parsePodPhase(buildPod(PodPhase.FAILED))).toBe(PodPhase.FAILED) + }) + + it('returns UNKNOWN when phase is missing', () => { + expect(parsePodPhase(buildPod(undefined))).toBe(PodPhase.UNKNOWN) + expect(parsePodPhase({} as k8s.V1Pod)).toBe(PodPhase.UNKNOWN) + }) + + it('returns UNKNOWN when phase is not a recognized value', () => { + expect(parsePodPhase(buildPod('SomethingElse'))).toBe(PodPhase.UNKNOWN) + }) +}) + +describe('getContainerErrors', () => { + it('returns no errors when there are no container statuses', () => { + expect(getContainerErrors(buildPod(PodPhase.PENDING))).toEqual([]) + expect(getContainerErrors({} as k8s.V1Pod)).toEqual([]) + }) + + it('returns no errors when containers are waiting for a recoverable reason', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'ContainerCreating')] + }) + expect(getContainerErrors(pod)).toEqual([]) + }) + + it('detects every unrecoverable waiting reason', () => { + for (const reason of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', reason)] + }) + expect(getContainerErrors(pod)).toEqual([`container "job": ${reason}`]) + } + }) + + it('includes the waiting message when present', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ErrImagePull', + 'Back-off pulling image "does-not-exist:latest"' + ) + ] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "job": ErrImagePull - Back-off pulling image "does-not-exist:latest"' + ]) + }) + + it('inspects init containers as well as regular containers', () => { + const pod = buildPod(PodPhase.PENDING, { + initContainerStatuses: [waitingContainer('init', 'ImagePullBackOff')], + containerStatuses: [waitingContainer('job', 'CreateContainerError')] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "init": ImagePullBackOff', + 'container "job": CreateContainerError' + ]) + }) + + it('ignores running/terminated containers and only collects waiting errors', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + { name: 'running', state: { running: {} } } as k8s.V1ContainerStatus, + waitingContainer('bad', 'InvalidImageName') + ] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "bad": InvalidImageName' + ]) + }) +}) + +describe('waitForPodPhases', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + eventSpy.mockResolvedValue({ body: { items: [] } } as never) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('returns once the pod reaches an awaited phase', async () => { + readSpy.mockResolvedValue({ body: buildPod(PodPhase.RUNNING) } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).resolves.toBeUndefined() + }) + + it('surfaces unrecoverable container errors in the thrown message', async () => { + readSpy.mockResolvedValue({ + body: buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ImagePullBackOff', + 'Back-off pulling image "nope:latest"' + ) + ] + }) + } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow( + 'container "job": ImagePullBackOff - Back-off pulling image "nope:latest"' + ) + }) + + it('throws with the phase when the pod is in a non-backoff phase', async () => { + readSpy.mockResolvedValue({ + body: buildPod(PodPhase.FAILED) + } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow('Pod my-pod is unhealthy with phase status Failed') + }) +}) + +describe('getUnrecoverableWaitingReasons', () => { + afterEach(() => { + delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + }) + + it('returns the built-in defaults when the env var is unset', () => { + expect(getUnrecoverableWaitingReasons()).toEqual( + UNRECOVERABLE_WAITING_REASONS + ) + }) + + it('adds extra reasons from the env var without dropping the defaults', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff, RunContainerError' + const reasons = getUnrecoverableWaitingReasons() + for (const builtin of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + expect(reasons.has(builtin)).toBe(true) + } + expect(reasons.has('CrashLoopBackOff')).toBe(true) + expect(reasons.has('RunContainerError')).toBe(true) + }) + + it('makes getContainerErrors honor the extended whitelist', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff' + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "job": CrashLoopBackOff' + ]) + }) +}) + +describe('describePodFailure', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + eventSpy.mockResolvedValue({ body: { items: [] } } as never) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('reports phase, failing conditions and terminated containers', async () => { + readSpy.mockResolvedValue({ + body: { + status: { + phase: PodPhase.FAILED, + reason: 'Evicted', + message: 'The node was low on resource: memory', + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message: 'no nodes available' + } + ], + containerStatuses: [ + { + name: 'job', + state: { + terminated: { exitCode: 137, reason: 'OOMKilled' } + } + } + ] + } + } + } as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Phase: Failed (reason: Evicted)') + expect(description).toContain( + 'Message: The node was low on resource: memory' + ) + expect(description).toContain( + 'Condition PodScheduled=False (reason: Unschedulable): no nodes available' + ) + expect(description).toContain( + 'Container "job" terminated: OOMKilled (exit code 137)' + ) + }) + + it('includes recent Warning events and skips Normal ones', async () => { + readSpy.mockResolvedValue({ + body: { status: { phase: PodPhase.PENDING } } + } as never) + eventSpy.mockResolvedValue({ + body: { + items: [ + { + type: 'Normal', + reason: 'Scheduled', + message: 'assigned', + lastTimestamp: new Date('2026-01-01T00:00:00Z') + }, + { + type: 'Warning', + reason: 'FailedScheduling', + message: '0/1 nodes are available', + count: 3, + lastTimestamp: new Date('2026-01-01T00:01:00Z') + } + ] + } + } as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain( + 'Event [Warning] FailedScheduling (x3): 0/1 nodes are available' + ) + expect(description).not.toContain('Scheduled') + }) + + it('degrades gracefully when listing events is forbidden', async () => { + readSpy.mockResolvedValue({ + body: { status: { phase: PodPhase.PENDING } } + } as never) + eventSpy.mockRejectedValue(new Error('events is forbidden') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Phase: Pending') + expect(description).not.toContain('Event [Warning]') + }) + + it('never throws when the pod cannot be read', async () => { + readSpy.mockRejectedValue(new Error('pod not found') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Could not read pod my-pod for diagnostics') + }) +})