From 244f06a783bafc558f95f0bcda684ec82606f2c0 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Mon, 22 Jun 2026 16:02:36 +0800 Subject: [PATCH 1/4] feat: surface unrecoverable container errors during pod wait When a CI job references a non-existent image or one it lacks permission to pull, the pod stays in Pending and waitForPodPhases previously timed out with only a generic phase-status message. GitHub Actions users had no indication of the real cause. Detect unrecoverable container waiting reasons (ImagePullBackOff, ErrImagePull, InvalidImageName, CreateContainerConfigError, CreateContainerError) on both init and regular containers, and fail fast with the container name, reason, and Kubernetes message so the error is visible in the Actions log. Refactor getPodPhase into readPod + parsePodPhase so the pod object can be inspected for container errors, and add unit tests covering parsePodPhase, getContainerErrors, and waitForPodPhases. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/k8s/src/k8s/index.ts | 69 +++++-- .../k8s/tests/wait-for-pod-phases-test.ts | 180 ++++++++++++++++++ 2 files changed, 237 insertions(+), 12 deletions(-) create mode 100644 packages/k8s/tests/wait-for-pod-phases-test.ts diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 249a0605..4b37e3eb 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -368,6 +368,33 @@ export async function pruneSecrets(): Promise { ) } +export const UNRECOVERABLE_WAITING_REASONS = new Set([ + 'ImagePullBackOff', + 'ErrImagePull', + 'InvalidImageName', + 'CreateContainerConfigError', + 'CreateContainerError' +]) + +export function getContainerErrors(pod: k8s.V1Pod): string[] { + const errors: string[] = [] + const allStatuses = [ + ...(pod.status?.initContainerStatuses ?? []), + ...(pod.status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason && UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) { + errors.push( + `container "${cs.name}": ${waiting.reason}${ + waiting.message ? ` - ${waiting.message}` : '' + }` + ) + } + } + return errors +} + export async function waitForPodPhases( podName: string, awaitingPhases: Set, @@ -378,7 +405,8 @@ export async function waitForPodPhases( let phase: PodPhase = PodPhase.UNKNOWN try { while (true) { - phase = await getPodPhase(podName) + const pod = await readPod(podName) + phase = parsePodPhase(pod) if (awaitingPhases.has(phase)) { return } @@ -388,10 +416,24 @@ export async function waitForPodPhases( `Pod ${podName} is unhealthy with phase status ${phase}` ) } + + const containerErrors = getContainerErrors(pod) + if (containerErrors.length > 0) { + throw new Error( + `Pod ${podName} has unrecoverable container errors: ${containerErrors.join( + '; ' + )}` + ) + } + await backOffManager.backOff() } } catch (error) { - throw new Error(`Pod ${podName} is unhealthy with phase status ${phase}`) + throw new Error( + `Pod ${podName} is unhealthy with phase status ${phase}: ${ + error instanceof Error ? error.message : String(error) + }` + ) } } @@ -414,21 +456,24 @@ export function getPrepareJobTimeoutSeconds(): number { return timeoutSeconds } -async function getPodPhase(podName: string): Promise { - const podPhaseLookup = new Set([ - PodPhase.PENDING, - PodPhase.RUNNING, - PodPhase.SUCCEEDED, - PodPhase.FAILED, - PodPhase.UNKNOWN - ]) +async function readPod(podName: string): Promise { const { body } = await k8sApi.readNamespacedPod(podName, namespace()) - const pod = body + return body +} + +const podPhaseLookup = new Set([ + PodPhase.PENDING, + PodPhase.RUNNING, + PodPhase.SUCCEEDED, + PodPhase.FAILED, + PodPhase.UNKNOWN +]) +export function parsePodPhase(pod: k8s.V1Pod): PodPhase { if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) { return PodPhase.UNKNOWN } - return pod.status?.phase as PodPhase + return pod.status.phase as PodPhase } async function isJobSucceeded(jobName: string): Promise { diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts new file mode 100644 index 00000000..fcabfc05 --- /dev/null +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -0,0 +1,180 @@ +import * as k8s from '@kubernetes/client-node' +import { + getContainerErrors, + parsePodPhase, + UNRECOVERABLE_WAITING_REASONS, + waitForPodPhases +} from '../src/k8s' +import { PodPhase } from '../src/k8s/utils' + +// Build a minimal V1Pod with the given phase and container statuses so we can +// exercise the container-error detection logic in isolation. +function buildPod( + phase?: string, + opts: { + containerStatuses?: k8s.V1ContainerStatus[] + initContainerStatuses?: k8s.V1ContainerStatus[] + } = {} +): k8s.V1Pod { + return { + status: { + phase, + containerStatuses: opts.containerStatuses, + initContainerStatuses: opts.initContainerStatuses + } + } as k8s.V1Pod +} + +function waitingContainer( + name: string, + reason?: string, + message?: string +): k8s.V1ContainerStatus { + return { + name, + state: { waiting: { reason, message } } + } as k8s.V1ContainerStatus +} + +describe('parsePodPhase', () => { + it('returns the phase when it is a known value', () => { + expect(parsePodPhase(buildPod(PodPhase.RUNNING))).toBe(PodPhase.RUNNING) + expect(parsePodPhase(buildPod(PodPhase.PENDING))).toBe(PodPhase.PENDING) + expect(parsePodPhase(buildPod(PodPhase.SUCCEEDED))).toBe(PodPhase.SUCCEEDED) + expect(parsePodPhase(buildPod(PodPhase.FAILED))).toBe(PodPhase.FAILED) + }) + + it('returns UNKNOWN when phase is missing', () => { + expect(parsePodPhase(buildPod(undefined))).toBe(PodPhase.UNKNOWN) + expect(parsePodPhase({} as k8s.V1Pod)).toBe(PodPhase.UNKNOWN) + }) + + it('returns UNKNOWN when phase is not a recognized value', () => { + expect(parsePodPhase(buildPod('SomethingElse'))).toBe(PodPhase.UNKNOWN) + }) +}) + +describe('getContainerErrors', () => { + it('returns no errors when there are no container statuses', () => { + expect(getContainerErrors(buildPod(PodPhase.PENDING))).toEqual([]) + expect(getContainerErrors({} as k8s.V1Pod)).toEqual([]) + }) + + it('returns no errors when containers are waiting for a recoverable reason', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'ContainerCreating')] + }) + expect(getContainerErrors(pod)).toEqual([]) + }) + + it('detects every unrecoverable waiting reason', () => { + for (const reason of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', reason)] + }) + expect(getContainerErrors(pod)).toEqual([`container "job": ${reason}`]) + } + }) + + it('includes the waiting message when present', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ErrImagePull', + 'Back-off pulling image "does-not-exist:latest"' + ) + ] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "job": ErrImagePull - Back-off pulling image "does-not-exist:latest"' + ]) + }) + + it('inspects init containers as well as regular containers', () => { + const pod = buildPod(PodPhase.PENDING, { + initContainerStatuses: [waitingContainer('init', 'ImagePullBackOff')], + containerStatuses: [waitingContainer('job', 'CreateContainerError')] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "init": ImagePullBackOff', + 'container "job": CreateContainerError' + ]) + }) + + it('ignores running/terminated containers and only collects waiting errors', () => { + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [ + { name: 'running', state: { running: {} } } as k8s.V1ContainerStatus, + waitingContainer('bad', 'InvalidImageName') + ] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "bad": InvalidImageName' + ]) + }) +}) + +describe('waitForPodPhases', () => { + let readSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('returns once the pod reaches an awaited phase', async () => { + readSpy.mockResolvedValue({ body: buildPod(PodPhase.RUNNING) } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).resolves.toBeUndefined() + }) + + it('surfaces unrecoverable container errors in the thrown message', async () => { + readSpy.mockResolvedValue({ + body: buildPod(PodPhase.PENDING, { + containerStatuses: [ + waitingContainer( + 'job', + 'ImagePullBackOff', + 'Back-off pulling image "nope:latest"' + ) + ] + }) + } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow( + 'container "job": ImagePullBackOff - Back-off pulling image "nope:latest"' + ) + }) + + it('throws with the phase when the pod is in a non-backoff phase', async () => { + readSpy.mockResolvedValue({ body: buildPod(PodPhase.FAILED) } as never) + + await expect( + waitForPodPhases( + 'my-pod', + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING]) + ) + ).rejects.toThrow( + 'Pod my-pod is unhealthy with phase status Failed: Pod my-pod is unhealthy with phase status Failed' + ) + }) +}) From 5cb8b9e8fb409b00c5b7afc6b95962a9fd0a7234 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Thu, 25 Jun 2026 10:01:48 +0800 Subject: [PATCH 2/4] feat: add comprehensive pod failure diagnostics and configurable error reasons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the error feedback mechanism introduced in the previous commit with: - describePodFailure(): aggregates pod phase, conditions, container statuses and Warning events into a single human-readable diagnostic string. Never throws — safe to call from any error path. - describePodWarningEvents(): best-effort retrieval of recent Warning K8s events (requires optional "events" RBAC permission). Degrades gracefully when the permission is missing. - getUnrecoverableWaitingReasons(): allows operators to extend the built-in fast-fail whitelist via the ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS environment variable without a code change. Built-in defaults cannot be removed. - All three failure paths in waitForPodPhases now attach full diagnostics: 1. Non-backoff phase (e.g. Failed) — includes pod details + events 2. Unrecoverable container error (e.g. ImagePullBackOff) — fail-fast with diagnostics instead of waiting for timeout 3. Timeout — includes pod details so the user can see WHY the pod never became ready - README: document the optional "events" permission and the new env var. - Tests: 8 new test cases (19 total) covering describePodFailure, describePodWarningEvents, getUnrecoverableWaitingReasons, and edge cases such as forbidden events API and unreadable pods. --- packages/k8s/README.md | 18 ++ packages/k8s/src/k8s/index.ts | 211 +++++++++++++++--- .../k8s/tests/wait-for-pod-phases-test.ts | 153 ++++++++++++- 3 files changed, 353 insertions(+), 29 deletions(-) diff --git a/packages/k8s/README.md b/packages/k8s/README.md index dc35c347..c2b30487 100644 --- a/packages/k8s/README.md +++ b/packages/k8s/README.md @@ -29,6 +29,16 @@ rules: resources: ["secrets"] verbs: ["get", "list", "create", "delete"] ``` +- (Optional, recommended) Granting `events` read access lets the hooks attach + recent `Warning` events (e.g. `FailedScheduling`, `FailedMount`) to the error + message when a pod fails to come online. This permission is **not required** — + if it is missing the hooks still work, they just omit the event section from + diagnostics. +``` +- apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] +``` - The `ACTIONS_RUNNER_POD_NAME` env should be set to the name of the pod - The `ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER` env should be set to true to prevent the runner from running any jobs outside of a container - The runner pod should map a persistent volume claim into the `_work` directory @@ -36,6 +46,14 @@ rules: - Some actions runner env's are expected to be set. These are set automatically by the runner. - `RUNNER_WORKSPACE` is expected to be set to the workspace of the runner - `GITHUB_WORKSPACE` is expected to be set to the workspace of the job +- Optional tuning env's + - `ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS` — comma-separated list of + extra container `waiting` reasons that should be treated as deterministic + terminal failures (fail fast instead of waiting for the timeout). These are + *added* to the built-ins (`ImagePullBackOff`, `ErrImagePull`, + `InvalidImageName`, `CreateContainerConfigError`, `CreateContainerError`); + the built-ins can never be removed. Use with care — only list reasons that + are truly unrecoverable for your workloads (e.g. `CrashLoopBackOff`). ## Limitations diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 4b37e3eb..d5358f0b 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -376,15 +376,39 @@ export const UNRECOVERABLE_WAITING_REASONS = new Set([ 'CreateContainerError' ]) +// Maximum number of Warning events to include in a pod's failure description so +// that the GitHub Actions log is not flooded. +const MAX_DIAGNOSTIC_EVENTS = 10 + +// The fast-fail whitelist can be extended (not narrowed) from the environment so +// operators can add deterministic terminal reasons without a code change. This +// reuses the same env-var configuration pattern as the prepare-job timeout. When +// the variable is unset, behaviour is identical to the built-in defaults. +export function getUnrecoverableWaitingReasons(): Set { + const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + if (!extra) { + return UNRECOVERABLE_WAITING_REASONS + } + const reasons = new Set(UNRECOVERABLE_WAITING_REASONS) + for (const reason of extra.split(',')) { + const trimmed = reason.trim() + if (trimmed) { + reasons.add(trimmed) + } + } + return reasons +} + export function getContainerErrors(pod: k8s.V1Pod): string[] { const errors: string[] = [] + const unrecoverableReasons = getUnrecoverableWaitingReasons() const allStatuses = [ ...(pod.status?.initContainerStatuses ?? []), ...(pod.status?.containerStatuses ?? []) ] for (const cs of allStatuses) { const waiting = cs.state?.waiting - if (waiting?.reason && UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) { + if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) { errors.push( `container "${cs.name}": ${waiting.reason}${ waiting.message ? ` - ${waiting.message}` : '' @@ -395,6 +419,128 @@ export function getContainerErrors(pod: k8s.V1Pod): string[] { return errors } +// describePodFailure aggregates everything that might explain why a pod is not +// healthy into a single human-readable string. It makes NO success/failure +// judgement of its own (a terminated exitCode=0 container is just reported as +// such) -- callers decide what is "bad". It never throws: if it cannot read the +// pod or list events it returns/embeds a best-effort note instead, so it is safe +// to call from any error path. +export async function describePodFailure(podName: string): Promise { + let pod: k8s.V1Pod + try { + pod = await readPod(podName) + } catch (err) { + return `Could not read pod ${podName} for diagnostics: ${ + err instanceof Error ? err.message : String(err) + }` + } + + const lines: string[] = [] + const status = pod.status + + if (status?.phase) { + lines.push( + `Phase: ${status.phase}${ + status.reason ? ` (reason: ${status.reason})` : '' + }` + ) + } + if (status?.message) { + lines.push(`Message: ${status.message}`) + } + + // Surface conditions that are blocking readiness, e.g. PodScheduled=False + // which is the only place a FailedScheduling shows up on the pod itself. + for (const cond of status?.conditions ?? []) { + if (cond.status === 'False') { + lines.push( + `Condition ${cond.type}=False${ + cond.reason ? ` (reason: ${cond.reason})` : '' + }${cond.message ? `: ${cond.message}` : ''}` + ) + } + } + + const allStatuses = [ + ...(status?.initContainerStatuses ?? []), + ...(status?.containerStatuses ?? []) + ] + for (const cs of allStatuses) { + const waiting = cs.state?.waiting + if (waiting?.reason) { + lines.push( + `Container "${cs.name}" waiting: ${waiting.reason}${ + waiting.message ? ` - ${waiting.message}` : '' + }` + ) + } + const terminated = cs.state?.terminated + if (terminated) { + lines.push( + `Container "${cs.name}" terminated: ${ + terminated.reason ?? 'Unknown' + } (exit code ${terminated.exitCode})${ + terminated.message ? ` - ${terminated.message}` : '' + }` + ) + } + } + + lines.push(...(await describePodWarningEvents(podName))) + + if (!lines.length) { + return `No additional diagnostic information available for pod ${podName}` + } + return lines.join('\n') +} + +// Reads the most recent Warning events for a pod. Best-effort: listing events +// requires the optional "events" get/list permission, so any error (including +// RBAC Forbidden) is swallowed and an empty list is returned. +async function describePodWarningEvents(podName: string): Promise { + let items: k8s.CoreV1Event[] + try { + const { body } = await k8sApi.listNamespacedEvent( + namespace(), + undefined, + undefined, + undefined, + `involvedObject.name=${podName}` + ) + items = body.items + } catch (err) { + core.debug( + `Could not list events for pod ${podName} (the 'events' permission may be missing): ${ + err instanceof Error ? err.message : String(err) + }` + ) + return [] + } + + const warnings = items.filter(e => e.type === 'Warning') + if (!warnings.length) { + return [] + } + + const eventTime = (e: k8s.CoreV1Event): number => { + const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp + return t ? new Date(t).getTime() : 0 + } + warnings.sort((a, b) => eventTime(a) - eventTime(b)) + + const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS) + const lines = recent.map(e => { + const count = e.count && e.count > 1 ? ` (x${e.count})` : '' + return `Event [Warning] ${e.reason ?? ''}${count}: ${e.message ?? ''}`.trim() + }) + if (warnings.length > recent.length) { + lines.unshift( + `(showing ${recent.length} of ${warnings.length} warning events)` + ) + } + return lines +} + export async function waitForPodPhases( podName: string, awaitingPhases: Set, @@ -403,37 +549,48 @@ export async function waitForPodPhases( ): Promise { const backOffManager = new BackOffManager(maxTimeSeconds) let phase: PodPhase = PodPhase.UNKNOWN - try { - while (true) { - const pod = await readPod(podName) - phase = parsePodPhase(pod) - if (awaitingPhases.has(phase)) { - return - } + while (true) { + const pod = await readPod(podName) + phase = parsePodPhase(pod) + if (awaitingPhases.has(phase)) { + return + } - if (!backOffPhases.has(phase)) { - throw new Error( - `Pod ${podName} is unhealthy with phase status ${phase}` - ) - } + // The pod reached a phase we are not willing to keep waiting on + // (a terminal/unhealthy phase). Attach full diagnostics and stop. + if (!backOffPhases.has(phase)) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy with phase status ${phase}\n${details}` + ) + } - const containerErrors = getContainerErrors(pod) - if (containerErrors.length > 0) { - throw new Error( - `Pod ${podName} has unrecoverable container errors: ${containerErrors.join( - '; ' - )}` - ) - } + // Still in a back-off phase, but a container hit a deterministic terminal + // error (e.g. ImagePullBackOff) -- fail fast with diagnostics instead of + // waiting out the full timeout. + const containerErrors = getContainerErrors(pod) + if (containerErrors.length > 0) { + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} has unrecoverable container errors: ${containerErrors.join( + '; ' + )}\n${details}` + ) + } + try { await backOffManager.backOff() + } catch (error) { + // BackOffManager throws "backoff timeout" when maxTimeSeconds is exceeded. + // Don't surface that bare message: collect diagnostics first so the user + // can see WHY the pod never became ready. This is the safest improvement + // -- it only runs when we were going to fail anyway and changes no + // success/failure decision. + const details = await describePodFailure(podName) + throw new Error( + `Pod ${podName} is unhealthy: timed out after ${maxTimeSeconds}s in phase ${phase}\n${details}` + ) } - } catch (error) { - throw new Error( - `Pod ${podName} is unhealthy with phase status ${phase}: ${ - error instanceof Error ? error.message : String(error) - }` - ) } } diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts index fcabfc05..9ff2e3ec 100644 --- a/packages/k8s/tests/wait-for-pod-phases-test.ts +++ b/packages/k8s/tests/wait-for-pod-phases-test.ts @@ -1,6 +1,8 @@ import * as k8s from '@kubernetes/client-node' import { + describePodFailure, getContainerErrors, + getUnrecoverableWaitingReasons, parsePodPhase, UNRECOVERABLE_WAITING_REASONS, waitForPodPhases @@ -117,10 +119,15 @@ describe('getContainerErrors', () => { describe('waitForPodPhases', () => { let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance beforeEach(() => { process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + // waitForPodPhases now calls describePodFailure on its failure paths, which + // lists events. Stub it out so the tests never hit a real cluster. + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + eventSpy.mockResolvedValue({ body: { items: [] } } as never) }) afterEach(() => { @@ -167,14 +174,156 @@ describe('waitForPodPhases', () => { it('throws with the phase when the pod is in a non-backoff phase', async () => { readSpy.mockResolvedValue({ body: buildPod(PodPhase.FAILED) } as never) + // The message is no longer double-wrapped: it states the phase once and is + // followed by the diagnostic block from describePodFailure. await expect( waitForPodPhases( 'my-pod', new Set([PodPhase.RUNNING]), new Set([PodPhase.PENDING]) ) - ).rejects.toThrow( - 'Pod my-pod is unhealthy with phase status Failed: Pod my-pod is unhealthy with phase status Failed' + ).rejects.toThrow('Pod my-pod is unhealthy with phase status Failed') + }) +}) + +describe('getUnrecoverableWaitingReasons', () => { + afterEach(() => { + delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] + }) + + it('returns the built-in defaults when the env var is unset', () => { + expect(getUnrecoverableWaitingReasons()).toEqual( + UNRECOVERABLE_WAITING_REASONS ) }) + + it('adds extra reasons from the env var without dropping the defaults', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff, RunContainerError' + const reasons = getUnrecoverableWaitingReasons() + for (const builtin of Array.from(UNRECOVERABLE_WAITING_REASONS)) { + expect(reasons.has(builtin)).toBe(true) + } + expect(reasons.has('CrashLoopBackOff')).toBe(true) + expect(reasons.has('RunContainerError')).toBe(true) + }) + + it('makes getContainerErrors honor the extended whitelist', () => { + process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] = + 'CrashLoopBackOff' + const pod = buildPod(PodPhase.PENDING, { + containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')] + }) + expect(getContainerErrors(pod)).toEqual([ + 'container "job": CrashLoopBackOff' + ]) + }) +}) + +describe('describePodFailure', () => { + let readSpy: jest.SpyInstance + let eventSpy: jest.SpyInstance + + beforeEach(() => { + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' + readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod') + eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent') + // Default: no events. Individual tests override this. + eventSpy.mockResolvedValue({ body: { items: [] } } as never) + }) + + afterEach(() => { + jest.restoreAllMocks() + delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + }) + + it('reports phase, failing conditions and terminated containers', async () => { + readSpy.mockResolvedValue({ + body: { + status: { + phase: PodPhase.FAILED, + reason: 'Evicted', + message: 'The node was low on resource: memory', + conditions: [ + { + type: 'PodScheduled', + status: 'False', + reason: 'Unschedulable', + message: 'no nodes available' + } + ], + containerStatuses: [ + { + name: 'job', + state: { + terminated: { exitCode: 137, reason: 'OOMKilled' } + } + } + ] + } + } + } as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Phase: Failed (reason: Evicted)') + expect(description).toContain( + 'Message: The node was low on resource: memory' + ) + expect(description).toContain( + 'Condition PodScheduled=False (reason: Unschedulable): no nodes available' + ) + expect(description).toContain( + 'Container "job" terminated: OOMKilled (exit code 137)' + ) + }) + + it('includes recent Warning events and skips Normal ones', async () => { + readSpy.mockResolvedValue({ + body: { status: { phase: PodPhase.PENDING } } + } as never) + eventSpy.mockResolvedValue({ + body: { + items: [ + { + type: 'Normal', + reason: 'Scheduled', + message: 'assigned', + lastTimestamp: new Date('2026-01-01T00:00:00Z') + }, + { + type: 'Warning', + reason: 'FailedScheduling', + message: '0/1 nodes are available', + count: 3, + lastTimestamp: new Date('2026-01-01T00:01:00Z') + } + ] + } + } as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain( + 'Event [Warning] FailedScheduling (x3): 0/1 nodes are available' + ) + expect(description).not.toContain('Scheduled') + }) + + it('degrades gracefully when listing events is forbidden', async () => { + readSpy.mockResolvedValue({ + body: { status: { phase: PodPhase.PENDING } } + } as never) + eventSpy.mockRejectedValue(new Error('events is forbidden') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Phase: Pending') + // No throw, and no event lines. + expect(description).not.toContain('Event [Warning]') + }) + + it('never throws when the pod cannot be read', async () => { + readSpy.mockRejectedValue(new Error('pod not found') as never) + + const description = await describePodFailure('my-pod') + expect(description).toContain('Could not read pod my-pod for diagnostics') + }) }) From 6790d48ae6692dae51e4c78a28a4da07d9f9c3a1 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Thu, 25 Jun 2026 10:24:50 +0800 Subject: [PATCH 3/4] style: fix prettier 2.6.2 formatting in index.ts The project pins prettier@2.6.2 in package-lock.json, but the previous commit was formatted with prettier 3.x which has different line-wrapping rules for template literals. Re-format with prettier 2.6.2 to pass CI. --- packages/k8s/src/k8s/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index d5358f0b..4f4ca58a 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -531,7 +531,9 @@ async function describePodWarningEvents(podName: string): Promise { const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS) const lines = recent.map(e => { const count = e.count && e.count > 1 ? ` (x${e.count})` : '' - return `Event [Warning] ${e.reason ?? ''}${count}: ${e.message ?? ''}`.trim() + return `Event [Warning] ${e.reason ?? ''}${count}: ${ + e.message ?? '' + }`.trim() }) if (warnings.length > recent.length) { lines.unshift( From a738df7d1557fc216419cc7cee00c981ef38e676 Mon Sep 17 00:00:00 2001 From: Longwt123 <1075015495@qq.com> Date: Fri, 26 Jun 2026 11:58:07 +0800 Subject: [PATCH 4/4] fix: upgrade actions-runner base image to 2.335.1 v2.329.0 was deprecated by GitHub and rejected at the broker level with "Runner version v2.329.0 is deprecated and cannot receive messages.", causing runner pods to crash-loop immediately after connecting. Also fix Dockerfile layer ordering: switch to root before COPY so that the subsequent chown is not run as the unprivileged runner user. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d10bf28b..98f71fb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,12 @@ COPY . /app/ RUN npm install && npm run bootstrap && npm run build-all -FROM ghcr.nju.edu.cn/actions/actions-runner:2.329.0 +FROM ghcr.nju.edu.cn/actions/actions-runner:2.335.1 + +USER root COPY --from=runner_builder /app/packages/k8s/dist/index.js /home/runner/k8s/index.js +RUN chown runner:runner /home/runner/k8s/index.js + USER runner