From da1044a67343c0ea6bf12fb501d271e60883f4e5 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Mon, 22 Jun 2026 16:02:36 +0800
Subject: [PATCH 01/18] feat: surface unrecoverable container errors during pod
 wait

When a CI job references a non-existent image or one it lacks permission
to pull, the pod stays in Pending and waitForPodPhases previously timed
out with only a generic phase-status message. GitHub Actions users had
no indication of the real cause.

Detect unrecoverable container waiting reasons (ImagePullBackOff,
ErrImagePull, InvalidImageName, CreateContainerConfigError,
CreateContainerError) on both init and regular containers, and fail fast
with the container name, reason, and Kubernetes message so the error is
visible in the Actions log.

Refactor getPodPhase into readPod + parsePodPhase so the pod object can
be inspected for container errors, and add unit tests covering
parsePodPhase, getContainerErrors, and waitForPodPhases.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts                 |  69 +++++--
 .../k8s/tests/wait-for-pod-phases-test.ts     | 180 ++++++++++++++++++
 2 files changed, 236 insertions(+), 13 deletions(-)
 create mode 100644 packages/k8s/tests/wait-for-pod-phases-test.ts

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 36696a73..5f219b8a 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -673,6 +673,33 @@ export async function pruneSecrets(): Promise<void> {
   )
 }
 
+export const UNRECOVERABLE_WAITING_REASONS = new Set([
+  'ImagePullBackOff',
+  'ErrImagePull',
+  'InvalidImageName',
+  'CreateContainerConfigError',
+  'CreateContainerError'
+])
+
+export function getContainerErrors(pod: k8s.V1Pod): string[] {
+  const errors: string[] = []
+  const allStatuses = [
+    ...(pod.status?.initContainerStatuses ?? []),
+    ...(pod.status?.containerStatuses ?? [])
+  ]
+  for (const cs of allStatuses) {
+    const waiting = cs.state?.waiting
+    if (waiting?.reason && UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) {
+      errors.push(
+        `container "${cs.name}": ${waiting.reason}${
+          waiting.message ? ` - ${waiting.message}` : ''
+        }`
+      )
+    }
+  }
+  return errors
+}
+
 export async function waitForPodPhases(
   podName: string,
   awaitingPhases: Set<PodPhase>,
@@ -683,7 +710,8 @@ export async function waitForPodPhases(
   let phase: PodPhase = PodPhase.UNKNOWN
   try {
     while (true) {
-      phase = await getPodPhase(podName)
+      const pod = await readPod(podName)
+      phase = parsePodPhase(pod)
       if (awaitingPhases.has(phase)) {
         return
       }
@@ -693,11 +721,23 @@ export async function waitForPodPhases(
           `Pod ${podName} is unhealthy with phase status ${phase}`
         )
       }
+
+      const containerErrors = getContainerErrors(pod)
+      if (containerErrors.length > 0) {
+        throw new Error(
+          `Pod ${podName} has unrecoverable container errors: ${containerErrors.join(
+            '; '
+          )}`
+        )
+      }
+
       await backOffManager.backOff()
     }
   } catch (error) {
     throw new Error(
-      `Pod ${podName} is unhealthy with phase status ${phase}: ${JSON.stringify(error)}`
+      `Pod ${podName} is unhealthy with phase status ${phase}: ${
+        error instanceof Error ? error.message : String(error)
+      }`
     )
   }
 }
@@ -721,23 +761,26 @@ export function getPrepareJobTimeoutSeconds(): number {
   return timeoutSeconds
 }
 
-async function getPodPhase(name: string): Promise<PodPhase> {
-  const podPhaseLookup = new Set<string>([
-    PodPhase.PENDING,
-    PodPhase.RUNNING,
-    PodPhase.SUCCEEDED,
-    PodPhase.FAILED,
-    PodPhase.UNKNOWN
-  ])
-  const pod = await k8sApi.readNamespacedPod({
-    name,
+async function readPod(podName: string): Promise<k8s.V1Pod> {
+  return await k8sApi.readNamespacedPod({
+    name: podName,
     namespace: namespace()
   })
+}
+
+const podPhaseLookup = new Set<string>([
+  PodPhase.PENDING,
+  PodPhase.RUNNING,
+  PodPhase.SUCCEEDED,
+  PodPhase.FAILED,
+  PodPhase.UNKNOWN
+])
 
+export function parsePodPhase(pod: k8s.V1Pod): PodPhase {
   if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) {
     return PodPhase.UNKNOWN
   }
-  return pod.status?.phase as PodPhase
+  return pod.status.phase as PodPhase
 }
 
 async function isJobSucceeded(name: string): Promise<boolean> {
diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts
new file mode 100644
index 00000000..fcabfc05
--- /dev/null
+++ b/packages/k8s/tests/wait-for-pod-phases-test.ts
@@ -0,0 +1,180 @@
+import * as k8s from '@kubernetes/client-node'
+import {
+  getContainerErrors,
+  parsePodPhase,
+  UNRECOVERABLE_WAITING_REASONS,
+  waitForPodPhases
+} from '../src/k8s'
+import { PodPhase } from '../src/k8s/utils'
+
+// Build a minimal V1Pod with the given phase and container statuses so we can
+// exercise the container-error detection logic in isolation.
+function buildPod(
+  phase?: string,
+  opts: {
+    containerStatuses?: k8s.V1ContainerStatus[]
+    initContainerStatuses?: k8s.V1ContainerStatus[]
+  } = {}
+): k8s.V1Pod {
+  return {
+    status: {
+      phase,
+      containerStatuses: opts.containerStatuses,
+      initContainerStatuses: opts.initContainerStatuses
+    }
+  } as k8s.V1Pod
+}
+
+function waitingContainer(
+  name: string,
+  reason?: string,
+  message?: string
+): k8s.V1ContainerStatus {
+  return {
+    name,
+    state: { waiting: { reason, message } }
+  } as k8s.V1ContainerStatus
+}
+
+describe('parsePodPhase', () => {
+  it('returns the phase when it is a known value', () => {
+    expect(parsePodPhase(buildPod(PodPhase.RUNNING))).toBe(PodPhase.RUNNING)
+    expect(parsePodPhase(buildPod(PodPhase.PENDING))).toBe(PodPhase.PENDING)
+    expect(parsePodPhase(buildPod(PodPhase.SUCCEEDED))).toBe(PodPhase.SUCCEEDED)
+    expect(parsePodPhase(buildPod(PodPhase.FAILED))).toBe(PodPhase.FAILED)
+  })
+
+  it('returns UNKNOWN when phase is missing', () => {
+    expect(parsePodPhase(buildPod(undefined))).toBe(PodPhase.UNKNOWN)
+    expect(parsePodPhase({} as k8s.V1Pod)).toBe(PodPhase.UNKNOWN)
+  })
+
+  it('returns UNKNOWN when phase is not a recognized value', () => {
+    expect(parsePodPhase(buildPod('SomethingElse'))).toBe(PodPhase.UNKNOWN)
+  })
+})
+
+describe('getContainerErrors', () => {
+  it('returns no errors when there are no container statuses', () => {
+    expect(getContainerErrors(buildPod(PodPhase.PENDING))).toEqual([])
+    expect(getContainerErrors({} as k8s.V1Pod)).toEqual([])
+  })
+
+  it('returns no errors when containers are waiting for a recoverable reason', () => {
+    const pod = buildPod(PodPhase.PENDING, {
+      containerStatuses: [waitingContainer('job', 'ContainerCreating')]
+    })
+    expect(getContainerErrors(pod)).toEqual([])
+  })
+
+  it('detects every unrecoverable waiting reason', () => {
+    for (const reason of Array.from(UNRECOVERABLE_WAITING_REASONS)) {
+      const pod = buildPod(PodPhase.PENDING, {
+        containerStatuses: [waitingContainer('job', reason)]
+      })
+      expect(getContainerErrors(pod)).toEqual([`container "job": ${reason}`])
+    }
+  })
+
+  it('includes the waiting message when present', () => {
+    const pod = buildPod(PodPhase.PENDING, {
+      containerStatuses: [
+        waitingContainer(
+          'job',
+          'ErrImagePull',
+          'Back-off pulling image "does-not-exist:latest"'
+        )
+      ]
+    })
+    expect(getContainerErrors(pod)).toEqual([
+      'container "job": ErrImagePull - Back-off pulling image "does-not-exist:latest"'
+    ])
+  })
+
+  it('inspects init containers as well as regular containers', () => {
+    const pod = buildPod(PodPhase.PENDING, {
+      initContainerStatuses: [waitingContainer('init', 'ImagePullBackOff')],
+      containerStatuses: [waitingContainer('job', 'CreateContainerError')]
+    })
+    expect(getContainerErrors(pod)).toEqual([
+      'container "init": ImagePullBackOff',
+      'container "job": CreateContainerError'
+    ])
+  })
+
+  it('ignores running/terminated containers and only collects waiting errors', () => {
+    const pod = buildPod(PodPhase.PENDING, {
+      containerStatuses: [
+        { name: 'running', state: { running: {} } } as k8s.V1ContainerStatus,
+        waitingContainer('bad', 'InvalidImageName')
+      ]
+    })
+    expect(getContainerErrors(pod)).toEqual([
+      'container "bad": InvalidImageName'
+    ])
+  })
+})
+
+describe('waitForPodPhases', () => {
+  let readSpy: jest.SpyInstance
+
+  beforeEach(() => {
+    process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default'
+    readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod')
+  })
+
+  afterEach(() => {
+    jest.restoreAllMocks()
+    delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+  })
+
+  it('returns once the pod reaches an awaited phase', async () => {
+    readSpy.mockResolvedValue({ body: buildPod(PodPhase.RUNNING) } as never)
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).resolves.toBeUndefined()
+  })
+
+  it('surfaces unrecoverable container errors in the thrown message', async () => {
+    readSpy.mockResolvedValue({
+      body: buildPod(PodPhase.PENDING, {
+        containerStatuses: [
+          waitingContainer(
+            'job',
+            'ImagePullBackOff',
+            'Back-off pulling image "nope:latest"'
+          )
+        ]
+      })
+    } as never)
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).rejects.toThrow(
+      'container "job": ImagePullBackOff - Back-off pulling image "nope:latest"'
+    )
+  })
+
+  it('throws with the phase when the pod is in a non-backoff phase', async () => {
+    readSpy.mockResolvedValue({ body: buildPod(PodPhase.FAILED) } as never)
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).rejects.toThrow(
+      'Pod my-pod is unhealthy with phase status Failed: Pod my-pod is unhealthy with phase status Failed'
+    )
+  })
+})

From 58f99ddbb391538da29a91f759772fe2473eac12 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Thu, 25 Jun 2026 10:01:48 +0800
Subject: [PATCH 02/18] feat: add comprehensive pod failure diagnostics and
 configurable error reasons
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Extend the error feedback mechanism introduced in the previous commit with:

- describePodFailure(): aggregates pod phase, conditions, container statuses
  and Warning events into a single human-readable diagnostic string.
  Never throws — safe to call from any error path.

- describePodWarningEvents(): best-effort retrieval of recent Warning K8s
  events (requires optional "events" RBAC permission). Degrades gracefully
  when the permission is missing.

- getUnrecoverableWaitingReasons(): allows operators to extend the built-in
  fast-fail whitelist via the ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS
  environment variable without a code change. Built-in defaults cannot be
  removed.

- All three failure paths in waitForPodPhases now attach full diagnostics:
  1. Non-backoff phase (e.g. Failed) — includes pod details + events
  2. Unrecoverable container error (e.g. ImagePullBackOff) — fail-fast with
     diagnostics instead of waiting for timeout
  3. Timeout — includes pod details so the user can see WHY the pod never
     became ready

- README: document the optional "events" permission and the new env var.

- Tests: 8 new test cases (19 total) covering describePodFailure,
  describePodWarningEvents, getUnrecoverableWaitingReasons, and edge cases
  such as forbidden events API and unreadable pods.
---
 packages/k8s/README.md                        |  18 ++
 packages/k8s/src/k8s/index.ts                 | 211 +++++++++++++++---
 .../k8s/tests/wait-for-pod-phases-test.ts     | 153 ++++++++++++-
 3 files changed, 353 insertions(+), 29 deletions(-)

diff --git a/packages/k8s/README.md b/packages/k8s/README.md
index ecc893b8..81881e87 100644
--- a/packages/k8s/README.md
+++ b/packages/k8s/README.md
@@ -26,6 +26,16 @@ rules:
   resources: ["secrets"]
   verbs: ["get", "list", "create", "delete"]
 ```
+- (Optional, recommended) Granting `events` read access lets the hooks attach
+  recent `Warning` events (e.g. `FailedScheduling`, `FailedMount`) to the error
+  message when a pod fails to come online. This permission is **not required** —
+  if it is missing the hooks still work, they just omit the event section from
+  diagnostics.
+```
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["get", "list"]
+```
 - The `ACTIONS_RUNNER_POD_NAME` env should be set to the name of the pod
 - The `ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER` env should be set to true to prevent the runner from running any jobs outside of a container
 - The runner pod should map a persistent volume claim into the `_work` directory
@@ -33,6 +43,14 @@ rules:
 - Some actions runner env's are expected to be set. These are set automatically by the runner.
     - `RUNNER_WORKSPACE` is expected to be set to the workspace of the runner
     - `GITHUB_WORKSPACE` is expected to be set to the workspace of the job
+- Optional tuning env's
+    - `ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS` — comma-separated list of
+      extra container `waiting` reasons that should be treated as deterministic
+      terminal failures (fail fast instead of waiting for the timeout). These are
+      *added* to the built-ins (`ImagePullBackOff`, `ErrImagePull`,
+      `InvalidImageName`, `CreateContainerConfigError`, `CreateContainerError`);
+      the built-ins can never be removed. Use with care — only list reasons that
+      are truly unrecoverable for your workloads (e.g. `CrashLoopBackOff`).
 
 
 ## Limitations
diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 5f219b8a..1307a48e 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -681,15 +681,39 @@ export const UNRECOVERABLE_WAITING_REASONS = new Set([
   'CreateContainerError'
 ])
 
+// Maximum number of Warning events to include in a pod's failure description so
+// that the GitHub Actions log is not flooded.
+const MAX_DIAGNOSTIC_EVENTS = 10
+
+// The fast-fail whitelist can be extended (not narrowed) from the environment so
+// operators can add deterministic terminal reasons without a code change. This
+// reuses the same env-var configuration pattern as the prepare-job timeout. When
+// the variable is unset, behaviour is identical to the built-in defaults.
+export function getUnrecoverableWaitingReasons(): Set<string> {
+  const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS']
+  if (!extra) {
+    return UNRECOVERABLE_WAITING_REASONS
+  }
+  const reasons = new Set(UNRECOVERABLE_WAITING_REASONS)
+  for (const reason of extra.split(',')) {
+    const trimmed = reason.trim()
+    if (trimmed) {
+      reasons.add(trimmed)
+    }
+  }
+  return reasons
+}
+
 export function getContainerErrors(pod: k8s.V1Pod): string[] {
   const errors: string[] = []
+  const unrecoverableReasons = getUnrecoverableWaitingReasons()
   const allStatuses = [
     ...(pod.status?.initContainerStatuses ?? []),
     ...(pod.status?.containerStatuses ?? [])
   ]
   for (const cs of allStatuses) {
     const waiting = cs.state?.waiting
-    if (waiting?.reason && UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) {
+    if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) {
       errors.push(
         `container "${cs.name}": ${waiting.reason}${
           waiting.message ? ` - ${waiting.message}` : ''
@@ -700,6 +724,128 @@ export function getContainerErrors(pod: k8s.V1Pod): string[] {
   return errors
 }
 
+// describePodFailure aggregates everything that might explain why a pod is not
+// healthy into a single human-readable string. It makes NO success/failure
+// judgement of its own (a terminated exitCode=0 container is just reported as
+// such) -- callers decide what is "bad". It never throws: if it cannot read the
+// pod or list events it returns/embeds a best-effort note instead, so it is safe
+// to call from any error path.
+export async function describePodFailure(podName: string): Promise<string> {
+  let pod: k8s.V1Pod
+  try {
+    pod = await readPod(podName)
+  } catch (err) {
+    return `Could not read pod ${podName} for diagnostics: ${
+      err instanceof Error ? err.message : String(err)
+    }`
+  }
+
+  const lines: string[] = []
+  const status = pod.status
+
+  if (status?.phase) {
+    lines.push(
+      `Phase: ${status.phase}${
+        status.reason ? ` (reason: ${status.reason})` : ''
+      }`
+    )
+  }
+  if (status?.message) {
+    lines.push(`Message: ${status.message}`)
+  }
+
+  // Surface conditions that are blocking readiness, e.g. PodScheduled=False
+  // which is the only place a FailedScheduling shows up on the pod itself.
+  for (const cond of status?.conditions ?? []) {
+    if (cond.status === 'False') {
+      lines.push(
+        `Condition ${cond.type}=False${
+          cond.reason ? ` (reason: ${cond.reason})` : ''
+        }${cond.message ? `: ${cond.message}` : ''}`
+      )
+    }
+  }
+
+  const allStatuses = [
+    ...(status?.initContainerStatuses ?? []),
+    ...(status?.containerStatuses ?? [])
+  ]
+  for (const cs of allStatuses) {
+    const waiting = cs.state?.waiting
+    if (waiting?.reason) {
+      lines.push(
+        `Container "${cs.name}" waiting: ${waiting.reason}${
+          waiting.message ? ` - ${waiting.message}` : ''
+        }`
+      )
+    }
+    const terminated = cs.state?.terminated
+    if (terminated) {
+      lines.push(
+        `Container "${cs.name}" terminated: ${
+          terminated.reason ?? 'Unknown'
+        } (exit code ${terminated.exitCode})${
+          terminated.message ? ` - ${terminated.message}` : ''
+        }`
+      )
+    }
+  }
+
+  lines.push(...(await describePodWarningEvents(podName)))
+
+  if (!lines.length) {
+    return `No additional diagnostic information available for pod ${podName}`
+  }
+  return lines.join('\n')
+}
+
+// Reads the most recent Warning events for a pod. Best-effort: listing events
+// requires the optional "events" get/list permission, so any error (including
+// RBAC Forbidden) is swallowed and an empty list is returned.
+async function describePodWarningEvents(podName: string): Promise<string[]> {
+  let items: k8s.CoreV1Event[]
+  try {
+    const { body } = await k8sApi.listNamespacedEvent(
+      namespace(),
+      undefined,
+      undefined,
+      undefined,
+      `involvedObject.name=${podName}`
+    )
+    items = body.items
+  } catch (err) {
+    core.debug(
+      `Could not list events for pod ${podName} (the 'events' permission may be missing): ${
+        err instanceof Error ? err.message : String(err)
+      }`
+    )
+    return []
+  }
+
+  const warnings = items.filter(e => e.type === 'Warning')
+  if (!warnings.length) {
+    return []
+  }
+
+  const eventTime = (e: k8s.CoreV1Event): number => {
+    const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp
+    return t ? new Date(t).getTime() : 0
+  }
+  warnings.sort((a, b) => eventTime(a) - eventTime(b))
+
+  const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS)
+  const lines = recent.map(e => {
+    const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
+    return `Event [Warning] ${e.reason ?? ''}${count}: ${e.message ?? ''}`.trim()
+  })
+  if (warnings.length > recent.length) {
+    lines.unshift(
+      `(showing ${recent.length} of ${warnings.length} warning events)`
+    )
+  }
+  return lines
+}
+
 export async function waitForPodPhases(
   podName: string,
   awaitingPhases: Set<PodPhase>,
@@ -708,37 +854,48 @@ export async function waitForPodPhases(
 ): Promise<void> {
   const backOffManager = new BackOffManager(maxTimeSeconds)
   let phase: PodPhase = PodPhase.UNKNOWN
-  try {
-    while (true) {
-      const pod = await readPod(podName)
-      phase = parsePodPhase(pod)
-      if (awaitingPhases.has(phase)) {
-        return
-      }
+  while (true) {
+    const pod = await readPod(podName)
+    phase = parsePodPhase(pod)
+    if (awaitingPhases.has(phase)) {
+      return
+    }
 
-      if (!backOffPhases.has(phase)) {
-        throw new Error(
-          `Pod ${podName} is unhealthy with phase status ${phase}`
-        )
-      }
+    // The pod reached a phase we are not willing to keep waiting on
+    // (a terminal/unhealthy phase). Attach full diagnostics and stop.
+    if (!backOffPhases.has(phase)) {
+      const details = await describePodFailure(podName)
+      throw new Error(
+        `Pod ${podName} is unhealthy with phase status ${phase}\n${details}`
+      )
+    }
 
-      const containerErrors = getContainerErrors(pod)
-      if (containerErrors.length > 0) {
-        throw new Error(
-          `Pod ${podName} has unrecoverable container errors: ${containerErrors.join(
-            '; '
-          )}`
-        )
-      }
+    // Still in a back-off phase, but a container hit a deterministic terminal
+    // error (e.g. ImagePullBackOff) -- fail fast with diagnostics instead of
+    // waiting out the full timeout.
+    const containerErrors = getContainerErrors(pod)
+    if (containerErrors.length > 0) {
+      const details = await describePodFailure(podName)
+      throw new Error(
+        `Pod ${podName} has unrecoverable container errors: ${containerErrors.join(
+          '; '
+        )}\n${details}`
+      )
+    }
 
+    try {
       await backOffManager.backOff()
+    } catch (error) {
+      // BackOffManager throws "backoff timeout" when maxTimeSeconds is exceeded.
+      // Don't surface that bare message: collect diagnostics first so the user
+      // can see WHY the pod never became ready. This is the safest improvement
+      // -- it only runs when we were going to fail anyway and changes no
+      // success/failure decision.
+      const details = await describePodFailure(podName)
+      throw new Error(
+        `Pod ${podName} is unhealthy: timed out after ${maxTimeSeconds}s in phase ${phase}\n${details}`
+      )
     }
-  } catch (error) {
-    throw new Error(
-      `Pod ${podName} is unhealthy with phase status ${phase}: ${
-        error instanceof Error ? error.message : String(error)
-      }`
-    )
   }
 }
 
diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts
index fcabfc05..9ff2e3ec 100644
--- a/packages/k8s/tests/wait-for-pod-phases-test.ts
+++ b/packages/k8s/tests/wait-for-pod-phases-test.ts
@@ -1,6 +1,8 @@
 import * as k8s from '@kubernetes/client-node'
 import {
+  describePodFailure,
   getContainerErrors,
+  getUnrecoverableWaitingReasons,
   parsePodPhase,
   UNRECOVERABLE_WAITING_REASONS,
   waitForPodPhases
@@ -117,10 +119,15 @@ describe('getContainerErrors', () => {
 
 describe('waitForPodPhases', () => {
   let readSpy: jest.SpyInstance
+  let eventSpy: jest.SpyInstance
 
   beforeEach(() => {
     process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default'
     readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod')
+    // waitForPodPhases now calls describePodFailure on its failure paths, which
+    // lists events. Stub it out so the tests never hit a real cluster.
+    eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent')
+    eventSpy.mockResolvedValue({ body: { items: [] } } as never)
   })
 
   afterEach(() => {
@@ -167,14 +174,156 @@ describe('waitForPodPhases', () => {
   it('throws with the phase when the pod is in a non-backoff phase', async () => {
     readSpy.mockResolvedValue({ body: buildPod(PodPhase.FAILED) } as never)
 
+    // The message is no longer double-wrapped: it states the phase once and is
+    // followed by the diagnostic block from describePodFailure.
     await expect(
       waitForPodPhases(
         'my-pod',
         new Set([PodPhase.RUNNING]),
         new Set([PodPhase.PENDING])
       )
-    ).rejects.toThrow(
-      'Pod my-pod is unhealthy with phase status Failed: Pod my-pod is unhealthy with phase status Failed'
+    ).rejects.toThrow('Pod my-pod is unhealthy with phase status Failed')
+  })
+})
+
+describe('getUnrecoverableWaitingReasons', () => {
+  afterEach(() => {
+    delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS']
+  })
+
+  it('returns the built-in defaults when the env var is unset', () => {
+    expect(getUnrecoverableWaitingReasons()).toEqual(
+      UNRECOVERABLE_WAITING_REASONS
     )
   })
+
+  it('adds extra reasons from the env var without dropping the defaults', () => {
+    process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] =
+      'CrashLoopBackOff, RunContainerError'
+    const reasons = getUnrecoverableWaitingReasons()
+    for (const builtin of Array.from(UNRECOVERABLE_WAITING_REASONS)) {
+      expect(reasons.has(builtin)).toBe(true)
+    }
+    expect(reasons.has('CrashLoopBackOff')).toBe(true)
+    expect(reasons.has('RunContainerError')).toBe(true)
+  })
+
+  it('makes getContainerErrors honor the extended whitelist', () => {
+    process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS'] =
+      'CrashLoopBackOff'
+    const pod = buildPod(PodPhase.PENDING, {
+      containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')]
+    })
+    expect(getContainerErrors(pod)).toEqual([
+      'container "job": CrashLoopBackOff'
+    ])
+  })
+})
+
+describe('describePodFailure', () => {
+  let readSpy: jest.SpyInstance
+  let eventSpy: jest.SpyInstance
+
+  beforeEach(() => {
+    process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default'
+    readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod')
+    eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent')
+    // Default: no events. Individual tests override this.
+    eventSpy.mockResolvedValue({ body: { items: [] } } as never)
+  })
+
+  afterEach(() => {
+    jest.restoreAllMocks()
+    delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+  })
+
+  it('reports phase, failing conditions and terminated containers', async () => {
+    readSpy.mockResolvedValue({
+      body: {
+        status: {
+          phase: PodPhase.FAILED,
+          reason: 'Evicted',
+          message: 'The node was low on resource: memory',
+          conditions: [
+            {
+              type: 'PodScheduled',
+              status: 'False',
+              reason: 'Unschedulable',
+              message: 'no nodes available'
+            }
+          ],
+          containerStatuses: [
+            {
+              name: 'job',
+              state: {
+                terminated: { exitCode: 137, reason: 'OOMKilled' }
+              }
+            }
+          ]
+        }
+      }
+    } as never)
+
+    const description = await describePodFailure('my-pod')
+    expect(description).toContain('Phase: Failed (reason: Evicted)')
+    expect(description).toContain(
+      'Message: The node was low on resource: memory'
+    )
+    expect(description).toContain(
+      'Condition PodScheduled=False (reason: Unschedulable): no nodes available'
+    )
+    expect(description).toContain(
+      'Container "job" terminated: OOMKilled (exit code 137)'
+    )
+  })
+
+  it('includes recent Warning events and skips Normal ones', async () => {
+    readSpy.mockResolvedValue({
+      body: { status: { phase: PodPhase.PENDING } }
+    } as never)
+    eventSpy.mockResolvedValue({
+      body: {
+        items: [
+          {
+            type: 'Normal',
+            reason: 'Scheduled',
+            message: 'assigned',
+            lastTimestamp: new Date('2026-01-01T00:00:00Z')
+          },
+          {
+            type: 'Warning',
+            reason: 'FailedScheduling',
+            message: '0/1 nodes are available',
+            count: 3,
+            lastTimestamp: new Date('2026-01-01T00:01:00Z')
+          }
+        ]
+      }
+    } as never)
+
+    const description = await describePodFailure('my-pod')
+    expect(description).toContain(
+      'Event [Warning] FailedScheduling (x3): 0/1 nodes are available'
+    )
+    expect(description).not.toContain('Scheduled')
+  })
+
+  it('degrades gracefully when listing events is forbidden', async () => {
+    readSpy.mockResolvedValue({
+      body: { status: { phase: PodPhase.PENDING } }
+    } as never)
+    eventSpy.mockRejectedValue(new Error('events is forbidden') as never)
+
+    const description = await describePodFailure('my-pod')
+    expect(description).toContain('Phase: Pending')
+    // No throw, and no event lines.
+    expect(description).not.toContain('Event [Warning]')
+  })
+
+  it('never throws when the pod cannot be read', async () => {
+    readSpy.mockRejectedValue(new Error('pod not found') as never)
+
+    const description = await describePodFailure('my-pod')
+    expect(description).toContain('Could not read pod my-pod for diagnostics')
+  })
 })

From 79aeb1934c6f204177eca0c0507bf4217528f3b6 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Thu, 25 Jun 2026 10:24:50 +0800
Subject: [PATCH 03/18] style: fix prettier 2.6.2 formatting in index.ts

The project pins prettier@2.6.2 in package-lock.json, but the previous
commit was formatted with prettier 3.x which has different line-wrapping
rules for template literals. Re-format with prettier 2.6.2 to pass CI.
---
 packages/k8s/src/k8s/index.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 1307a48e..23366deb 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -836,7 +836,9 @@ async function describePodWarningEvents(podName: string): Promise<string[]> {
   const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS)
   const lines = recent.map(e => {
     const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
-    return `Event [Warning] ${e.reason ?? ''}${count}: ${e.message ?? ''}`.trim()
+    return `Event [Warning] ${e.reason ?? ''}${count}: ${
+      e.message ?? ''
+    }`.trim()
   })
   if (warnings.length > recent.length) {
     lines.unshift(

From 893606ec5c14b832bb0d3a3bf00789d3ce4095e8 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Fri, 26 Jun 2026 11:58:07 +0800
Subject: [PATCH 04/18] fix: upgrade actions-runner base image to 2.335.1

v2.329.0 was deprecated by GitHub and rejected at the broker level with
"Runner version v2.329.0 is deprecated and cannot receive messages.",
causing runner pods to crash-loop immediately after connecting.

Also fix Dockerfile layer ordering: switch to root before COPY so that
the subsequent chown is not run as the unprivileged runner user.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 Dockerfile | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6f758a7b..04974605 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,14 +1,17 @@
-FROM swr.cn-north-4.myhuaweicloud.com/opensourceway/node:latest AS runner_builder
-
-WORKDIR /app
-
-COPY . /app/
-
-RUN npm install && npm run bootstrap && npm run build-all
-
-FROM ghcr.io/actions/actions-runner:2.334.0
-
-COPY --from=runner_builder /app/packages/k8s/dist/index.js /home/runner/k8s/index.js
-
-USER runner
-
+FROM swr.cn-north-4.myhuaweicloud.com/opensourceway/node:latest AS runner_builder
+
+WORKDIR /app
+
+COPY . /app/
+
+RUN npm install && npm run bootstrap && npm run build-all
+
+FROM --platform=linux/amd64 ghcr.nju.edu.cn/actions/actions-runner:2.335.1
+
+USER root
+
+COPY --from=runner_builder /app/packages/k8s/dist/index.js /home/runner/k8s/index.js
+
+RUN chown runner:runner /home/runner/k8s/index.js
+
+USER runner

From 820f0974201f2cc7b43ec8692f108db22edeed97 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Fri, 26 Jun 2026 17:10:16 +0800
Subject: [PATCH 05/18] fix: add namespace() in-cluster fallback and amd64
 platform flag

- namespace() now reads /var/run/secrets/kubernetes.io/serviceaccount/namespace
  as fallback when kubeconfig context has no namespace (in-cluster ARC setup)
- Dockerfile: add --platform=linux/amd64 to runner base image stage

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 23366deb..4295ef8b 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -1,4 +1,5 @@
 import * as core from '@actions/core'
+import * as fs from 'fs'
 import * as path from 'path'
 import { spawn } from 'child_process'
 import * as k8s from '@kubernetes/client-node'
@@ -1051,12 +1052,26 @@ export function namespace(): string {
   }
 
   const context = kc.getContexts().find(ctx => ctx.namespace)
-  if (!context?.namespace) {
-    throw new Error(
-      'Failed to determine namespace, falling back to `default`. Namespace should be set in context, or in env variable "ACTIONS_RUNNER_KUBERNETES_NAMESPACE"'
-    )
+  if (context?.namespace) {
+    return context.namespace
+  }
+
+  // When running in-cluster the kubeconfig context has no namespace field;
+  // read it from the mounted ServiceAccount file instead.
+  const saNamespaceFile =
+    '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
+  try {
+    const ns = fs.readFileSync(saNamespaceFile, 'utf8').trim()
+    if (ns) {
+      return ns
+    }
+  } catch {
+    // not running in-cluster, fall through to error
   }
-  return context.namespace
+
+  throw new Error(
+    'Failed to determine namespace. Set the ACTIONS_RUNNER_KUBERNETES_NAMESPACE environment variable or ensure the kubeconfig context includes a namespace.'
+  )
 }
 
 class BackOffManager {

From a56d0b66fabfc82a3c5f8e652710e17b17ad22a7 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Fri, 26 Jun 2026 17:22:36 +0800
Subject: [PATCH 06/18] fix: update listNamespacedEvent to new k8s client API
 style

The release/no_volumes base uses @kubernetes/client-node v0.22+ where
API methods take an options object and return the resource directly
(no .body wrapper). Fix describePodWarningEvents() accordingly.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 4295ef8b..05b4e9fe 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -806,14 +806,11 @@ export async function describePodFailure(podName: string): Promise<string> {
 async function describePodWarningEvents(podName: string): Promise<string[]> {
   let items: k8s.CoreV1Event[]
   try {
-    const { body } = await k8sApi.listNamespacedEvent(
-      namespace(),
-      undefined,
-      undefined,
-      undefined,
-      `involvedObject.name=${podName}`
-    )
-    items = body.items
+    const result = await k8sApi.listNamespacedEvent({
+      namespace: namespace(),
+      fieldSelector: `involvedObject.name=${podName}`
+    })
+    items = result.items
   } catch (err) {
     core.debug(
       `Could not list events for pod ${podName} (the 'events' permission may be missing): ${

From 1257123fb20fbec8310d42646fdd3b8a7a3fe8cf Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 10:13:12 +0800
Subject: [PATCH 07/18] fix: revert runner base image to match
 release/no_volumes (2.334.0)

- Remove --platform=linux/amd64: Jenkins build host is aarch64,
  without the flag the final image is ARM64, matching the working
  release/no_volumes build
- Revert to ghcr.io/actions/actions-runner:2.334.0 (same as
  release/no_volumes); the nju.edu.cn mirror at 2.335.1 caused
  silent 0-second exit with no logs (likely corrupted/wrong image)
- Remove USER root + chown (not needed, matches release/no_volumes)
- Update nodeSelector to arm64 to match ARM64 image architecture

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 Dockerfile                             |   6 +-
 tao/linux-arm64-npu-1-test_values.yaml | 232 +++++++++++++++++++++++++
 2 files changed, 233 insertions(+), 5 deletions(-)
 create mode 100644 tao/linux-arm64-npu-1-test_values.yaml

diff --git a/Dockerfile b/Dockerfile
index 04974605..9de8902d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,12 +6,8 @@ COPY . /app/
 
 RUN npm install && npm run bootstrap && npm run build-all
 
-FROM --platform=linux/amd64 ghcr.nju.edu.cn/actions/actions-runner:2.335.1
-
-USER root
+FROM ghcr.io/actions/actions-runner:2.334.0
 
 COPY --from=runner_builder /app/packages/k8s/dist/index.js /home/runner/k8s/index.js
 
-RUN chown runner:runner /home/runner/k8s/index.js
-
 USER runner
diff --git a/tao/linux-arm64-npu-1-test_values.yaml b/tao/linux-arm64-npu-1-test_values.yaml
new file mode 100644
index 00000000..67cbfc2a
--- /dev/null
+++ b/tao/linux-arm64-npu-1-test_values.yaml
@@ -0,0 +1,232 @@
+gha-runner-scale-set:
+
+  githubConfigUrl: https://github.com/ascend-gha-runners
+
+  githubConfigSecret: ascend-runner-mgmt-secret
+
+  listenerTemplate:
+    spec:
+      containers:
+        - name: listener
+          image: 
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+
+  containerMode:
+    type: "kubernetes-novolume"
+
+  template:
+    spec:
+      serviceAccountName: runner-service-account
+      schedulerName: volcano
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      imagePullSecrets:
+        - name: huawei-swr-image-pull-secret-model-gy
+      containers:
+        - name: runner
+          image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/runner-containers-hooks:release-no_volumes-9c3ea5
+          command: ["/home/runner/run.sh"]
+          securityContext:
+            capabilities:
+              add:
+                - SYS_RESOURCE
+          env:
+            - name: ACTIONS_RUNNER_CONTAINER_HOOKS
+              value: /home/runner/k8s/index.js
+            - name: NODE_OPTIONS
+              value: "--dns-result-order=ipv4first"
+            - name: ACTIONS_RUNNER_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
+              value: "false"
+            - name: ACTIONS_RUNNER_CONTAINER_HOOK_TEMPLATE
+              value: /home/runner/pod-templates/default.yaml
+            - name: ACTIONS_RUNNER_USE_KUBE_SCHEDULER
+              value: "false"
+            - name: ACTIONS_RUNNER_PREPARE_JOB_TIMEOUT_SECONDS
+              value: "3600"
+            - name: ACTIONS_RUNNER_IMAGE
+              value: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/runner-containers-hooks:release-no_volumes-9c3ea5
+            - name: ACTIONS_RUNNER_KUBERNETES_NAMESPACE
+              value: ascend-gha-runners
+          volumeMounts:
+            - name: work
+              mountPath: /home/runner/_work
+            - name: pod-templates
+              mountPath: /home/runner/pod-templates
+              readOnly: true
+            - name: shared-volume
+              mountPath: /root/.cache
+      volumes:
+        - name: work
+          ephemeral:
+            volumeClaimTemplate:
+              spec:
+                accessModes: ["ReadWriteMany"]
+                storageClassName: "sfsturbo-subpath-sc"
+                resources:
+                  requests:
+                    storage: 64Gi
+        - name: pod-templates
+          configMap:
+            name: pod-template-npu-1
+        - name: shared-volume
+          persistentVolumeClaim:
+            claimName: ascend-ci-share
+        - name: custom-index
+          persistentVolumeClaim:
+            claimName: custom-index
+        - name: app-data
+          emptyDir: {}
+  controllerServiceAccount:
+    namespace: arc-systems
+    name: arc-gha-rs-controller
+
+  listenerMetrics:
+    counters:
+      gha_started_jobs_total:
+        labels:
+          ["repository", "organization", "enterprise", "job_name", "event_name", "job_workflow_ref"]
+      gha_completed_jobs_total:
+        labels:
+          [
+            "repository",
+            "organization",
+            "enterprise",
+            "job_name",
+            "event_name",
+            "job_result",
+            "job_workflow_ref",
+          ]
+    gauges:
+      gha_assigned_jobs:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_running_jobs:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_registered_runners:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_busy_runners:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_min_runners:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_max_runners:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_desired_runners:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+      gha_idle_runners:
+        labels: ["name", "namespace", "repository", "organization", "enterprise"]
+    histograms:
+      gha_job_startup_duration_seconds:
+        labels:
+          ["repository", "organization", "enterprise", "job_name", "event_name","job_workflow_ref"]
+        buckets:
+          [
+            0.01,
+            0.05,
+            0.1,
+            0.5,
+            1.0,
+            2.0,
+            3.0,
+            4.0,
+            5.0,
+            6.0,
+            7.0,
+            8.0,
+            9.0,
+            10.0,
+            12.0,
+            15.0,
+            18.0,
+            20.0,
+            25.0,
+            30.0,
+            40.0,
+            50.0,
+            60.0,
+            70.0,
+            80.0,
+            90.0,
+            100.0,
+            110.0,
+            120.0,
+            150.0,
+            180.0,
+            210.0,
+            240.0,
+            300.0,
+            360.0,
+            420.0,
+            480.0,
+            540.0,
+            600.0,
+            900.0,
+            1200.0,
+            1800.0,
+            2400.0,
+            3000.0,
+            3600.0,
+          ]
+      gha_job_execution_duration_seconds:
+        labels:
+          [
+            "repository",
+            "organization",
+            "enterprise",
+            "job_name",
+            "event_name",
+            "job_result",
+            "job_workflow_ref"
+          ]
+        buckets:
+          [
+            0.01,
+            0.05,
+            0.1,
+            0.5,
+            1.0,
+            2.0,
+            3.0,
+            4.0,
+            5.0,
+            6.0,
+            7.0,
+            8.0,
+            9.0,
+            10.0,
+            12.0,
+            15.0,
+            18.0,
+            20.0,
+            25.0,
+            30.0,
+            40.0,
+            50.0,
+            60.0,
+            70.0,
+            80.0,
+            90.0,
+            100.0,
+            110.0,
+            120.0,
+            150.0,
+            180.0,
+            210.0,
+            240.0,
+            300.0,
+            360.0,
+            420.0,
+            480.0,
+            540.0,
+            600.0,
+            900.0,
+            1200.0,
+            1800.0,
+            2400.0,
+            3000.0,
+            3600.0,
+          ]
+

From 8071664586a746f9eb1a09338502688f0fc72d67 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 10:50:18 +0800
Subject: [PATCH 08/18] fix: deduplicate and clean up pod failure error output

- describePodFailure(): skip container waiting reasons already in
  UNRECOVERABLE_WAITING_REASONS (they appear in the caller's first
  line via getContainerErrors, printing them again is redundant)
- describePodFailure(): suppress terminated containers with exitCode=0
  (e.g. fs-init Completed) -- successful init containers are noise
- Add tao/ to .gitignore; remove accidentally committed values file

Result: InvalidImageName/ErrImagePull now shows as one clean block:
  Pod <name> has unrecoverable container errors: container "job": ErrImagePull - <message>
  Phase: Pending
  Condition Ready=False ...

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 .gitignore                             |   2 +-
 packages/k8s/src/k8s/index.ts          |  17 +-
 tao/linux-arm64-npu-1-test_values.yaml | 232 -------------------------
 3 files changed, 12 insertions(+), 239 deletions(-)
 delete mode 100644 tao/linux-arm64-npu-1-test_values.yaml

diff --git a/.gitignore b/.gitignore
index a48fc2b5..b33ad537 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,4 @@ node_modules/
 lib/
 dist/
 **/tests/_temp/**
-packages/k8s/tests/test-kind.yaml
\ No newline at end of file
+packages/k8s/tests/test-kind.yamltao/
diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 05b4e9fe..84993013 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -774,14 +774,19 @@ export async function describePodFailure(podName: string): Promise<string> {
   for (const cs of allStatuses) {
     const waiting = cs.state?.waiting
     if (waiting?.reason) {
-      lines.push(
-        `Container "${cs.name}" waiting: ${waiting.reason}${
-          waiting.message ? ` - ${waiting.message}` : ''
-        }`
-      )
+      // Skip reasons already surfaced by getContainerErrors() to avoid
+      // duplicating the same message in the caller's error string.
+      if (!UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) {
+        lines.push(
+          `Container "${cs.name}" waiting: ${waiting.reason}${
+            waiting.message ? ` - ${waiting.message}` : ''
+          }`
+        )
+      }
     }
     const terminated = cs.state?.terminated
-    if (terminated) {
+    // Only surface non-zero exits; exit 0 (e.g. fs-init Completed) is noise.
+    if (terminated && terminated.exitCode !== 0) {
       lines.push(
         `Container "${cs.name}" terminated: ${
           terminated.reason ?? 'Unknown'
diff --git a/tao/linux-arm64-npu-1-test_values.yaml b/tao/linux-arm64-npu-1-test_values.yaml
deleted file mode 100644
index 67cbfc2a..00000000
--- a/tao/linux-arm64-npu-1-test_values.yaml
+++ /dev/null
@@ -1,232 +0,0 @@
-gha-runner-scale-set:
-
-  githubConfigUrl: https://github.com/ascend-gha-runners
-
-  githubConfigSecret: ascend-runner-mgmt-secret
-
-  listenerTemplate:
-    spec:
-      containers:
-        - name: listener
-          image: 
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-
-  containerMode:
-    type: "kubernetes-novolume"
-
-  template:
-    spec:
-      serviceAccountName: runner-service-account
-      schedulerName: volcano
-      nodeSelector:
-        kubernetes.io/arch: arm64
-      imagePullSecrets:
-        - name: huawei-swr-image-pull-secret-model-gy
-      containers:
-        - name: runner
-          image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/runner-containers-hooks:release-no_volumes-9c3ea5
-          command: ["/home/runner/run.sh"]
-          securityContext:
-            capabilities:
-              add:
-                - SYS_RESOURCE
-          env:
-            - name: ACTIONS_RUNNER_CONTAINER_HOOKS
-              value: /home/runner/k8s/index.js
-            - name: NODE_OPTIONS
-              value: "--dns-result-order=ipv4first"
-            - name: ACTIONS_RUNNER_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
-              value: "false"
-            - name: ACTIONS_RUNNER_CONTAINER_HOOK_TEMPLATE
-              value: /home/runner/pod-templates/default.yaml
-            - name: ACTIONS_RUNNER_USE_KUBE_SCHEDULER
-              value: "false"
-            - name: ACTIONS_RUNNER_PREPARE_JOB_TIMEOUT_SECONDS
-              value: "3600"
-            - name: ACTIONS_RUNNER_IMAGE
-              value: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/runner-containers-hooks:release-no_volumes-9c3ea5
-            - name: ACTIONS_RUNNER_KUBERNETES_NAMESPACE
-              value: ascend-gha-runners
-          volumeMounts:
-            - name: work
-              mountPath: /home/runner/_work
-            - name: pod-templates
-              mountPath: /home/runner/pod-templates
-              readOnly: true
-            - name: shared-volume
-              mountPath: /root/.cache
-      volumes:
-        - name: work
-          ephemeral:
-            volumeClaimTemplate:
-              spec:
-                accessModes: ["ReadWriteMany"]
-                storageClassName: "sfsturbo-subpath-sc"
-                resources:
-                  requests:
-                    storage: 64Gi
-        - name: pod-templates
-          configMap:
-            name: pod-template-npu-1
-        - name: shared-volume
-          persistentVolumeClaim:
-            claimName: ascend-ci-share
-        - name: custom-index
-          persistentVolumeClaim:
-            claimName: custom-index
-        - name: app-data
-          emptyDir: {}
-  controllerServiceAccount:
-    namespace: arc-systems
-    name: arc-gha-rs-controller
-
-  listenerMetrics:
-    counters:
-      gha_started_jobs_total:
-        labels:
-          ["repository", "organization", "enterprise", "job_name", "event_name", "job_workflow_ref"]
-      gha_completed_jobs_total:
-        labels:
-          [
-            "repository",
-            "organization",
-            "enterprise",
-            "job_name",
-            "event_name",
-            "job_result",
-            "job_workflow_ref",
-          ]
-    gauges:
-      gha_assigned_jobs:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_running_jobs:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_registered_runners:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_busy_runners:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_min_runners:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_max_runners:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_desired_runners:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-      gha_idle_runners:
-        labels: ["name", "namespace", "repository", "organization", "enterprise"]
-    histograms:
-      gha_job_startup_duration_seconds:
-        labels:
-          ["repository", "organization", "enterprise", "job_name", "event_name","job_workflow_ref"]
-        buckets:
-          [
-            0.01,
-            0.05,
-            0.1,
-            0.5,
-            1.0,
-            2.0,
-            3.0,
-            4.0,
-            5.0,
-            6.0,
-            7.0,
-            8.0,
-            9.0,
-            10.0,
-            12.0,
-            15.0,
-            18.0,
-            20.0,
-            25.0,
-            30.0,
-            40.0,
-            50.0,
-            60.0,
-            70.0,
-            80.0,
-            90.0,
-            100.0,
-            110.0,
-            120.0,
-            150.0,
-            180.0,
-            210.0,
-            240.0,
-            300.0,
-            360.0,
-            420.0,
-            480.0,
-            540.0,
-            600.0,
-            900.0,
-            1200.0,
-            1800.0,
-            2400.0,
-            3000.0,
-            3600.0,
-          ]
-      gha_job_execution_duration_seconds:
-        labels:
-          [
-            "repository",
-            "organization",
-            "enterprise",
-            "job_name",
-            "event_name",
-            "job_result",
-            "job_workflow_ref"
-          ]
-        buckets:
-          [
-            0.01,
-            0.05,
-            0.1,
-            0.5,
-            1.0,
-            2.0,
-            3.0,
-            4.0,
-            5.0,
-            6.0,
-            7.0,
-            8.0,
-            9.0,
-            10.0,
-            12.0,
-            15.0,
-            18.0,
-            20.0,
-            25.0,
-            30.0,
-            40.0,
-            50.0,
-            60.0,
-            70.0,
-            80.0,
-            90.0,
-            100.0,
-            110.0,
-            120.0,
-            150.0,
-            180.0,
-            210.0,
-            240.0,
-            300.0,
-            360.0,
-            420.0,
-            480.0,
-            540.0,
-            600.0,
-            900.0,
-            1200.0,
-            1800.0,
-            2400.0,
-            3000.0,
-            3600.0,
-          ]
-

From 1bb105f63e2f2a38087b2643fdbad901b8af832f Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 10:59:07 +0800
Subject: [PATCH 09/18] feat: improve pod failure error output formatting
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Layer the error output into clearly separated sections:

  Pod <name> has unrecoverable container errors:
    ✗ container "job": ErrImagePull
      Error response from daemon: manifest for ... not found
  ────────────────────────────────────────────────────────────
  Pod status: Pending
    ✗ Ready=False (ContainersNotReady): containers with ...
    ✗ ContainersReady=False (ContainersNotReady): ...

  Recent warning events:
    [Failed] (x3) Error response from daemon: ...

Changes:
- getContainerErrors(): format each entry as indented ✗ lines with
  message detail on a separate indented line
- describePodFailure(): group output into sections (pod status,
  container details, warning events) separated by blank lines
- waitForPodPhases(): add ─── separator between summary and details
- describePodWarningEvents(): indent event lines consistently

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts | 95 +++++++++++++++++------------------
 1 file changed, 45 insertions(+), 50 deletions(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 84993013..9de5ac68 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -715,11 +715,10 @@ export function getContainerErrors(pod: k8s.V1Pod): string[] {
   for (const cs of allStatuses) {
     const waiting = cs.state?.waiting
     if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) {
-      errors.push(
-        `container "${cs.name}": ${waiting.reason}${
-          waiting.message ? ` - ${waiting.message}` : ''
-        }`
-      )
+      // Format as two indented lines: reason on first, message detail on second
+      const reason = `  ✗ container "${cs.name}": ${waiting.reason}`
+      const detail = waiting.message ? `    ${waiting.message}` : ''
+      errors.push(detail ? `${reason}\n${detail}` : reason)
     }
   }
   return errors
@@ -741,68 +740,70 @@ export async function describePodFailure(podName: string): Promise<string> {
     }`
   }
 
-  const lines: string[] = []
+  const sections: string[] = []
   const status = pod.status
 
-  if (status?.phase) {
-    lines.push(
-      `Phase: ${status.phase}${
-        status.reason ? ` (reason: ${status.reason})` : ''
-      }`
-    )
-  }
+  // --- Pod-level status ---
+  const podLines: string[] = []
+  const phaseLabel = status?.phase
+    ? `${status.phase}${status.reason ? ` (${status.reason})` : ''}`
+    : 'Unknown'
+  podLines.push(`Pod status: ${phaseLabel}`)
   if (status?.message) {
-    lines.push(`Message: ${status.message}`)
+    podLines.push(`  ${status.message}`)
   }
 
-  // Surface conditions that are blocking readiness, e.g. PodScheduled=False
-  // which is the only place a FailedScheduling shows up on the pod itself.
+  // Surface conditions that are False (e.g. PodScheduled=False from FailedScheduling)
   for (const cond of status?.conditions ?? []) {
     if (cond.status === 'False') {
-      lines.push(
-        `Condition ${cond.type}=False${
-          cond.reason ? ` (reason: ${cond.reason})` : ''
-        }${cond.message ? `: ${cond.message}` : ''}`
-      )
+      const reason = cond.reason ? ` (${cond.reason})` : ''
+      const msg = cond.message ? `: ${cond.message}` : ''
+      podLines.push(`  ✗ ${cond.type}=False${reason}${msg}`)
     }
   }
+  sections.push(podLines.join('\n'))
 
+  // --- Container-level status (non-zero exits and non-unrecoverable waits) ---
+  const unrecoverableReasons = getUnrecoverableWaitingReasons()
   const allStatuses = [
     ...(status?.initContainerStatuses ?? []),
     ...(status?.containerStatuses ?? [])
   ]
+  const containerLines: string[] = []
   for (const cs of allStatuses) {
     const waiting = cs.state?.waiting
     if (waiting?.reason) {
-      // Skip reasons already surfaced by getContainerErrors() to avoid
-      // duplicating the same message in the caller's error string.
-      if (!UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) {
-        lines.push(
-          `Container "${cs.name}" waiting: ${waiting.reason}${
-            waiting.message ? ` - ${waiting.message}` : ''
-          }`
-        )
+      // Skip reasons already surfaced by getContainerErrors() in the caller's
+      // first line to avoid printing the same error twice.
+      if (!unrecoverableReasons.has(waiting.reason)) {
+        const msg = waiting.message ? `\n    ${waiting.message}` : ''
+        containerLines.push(`  ✗ container "${cs.name}" waiting: ${waiting.reason}${msg}`)
       }
     }
     const terminated = cs.state?.terminated
     // Only surface non-zero exits; exit 0 (e.g. fs-init Completed) is noise.
     if (terminated && terminated.exitCode !== 0) {
-      lines.push(
-        `Container "${cs.name}" terminated: ${
-          terminated.reason ?? 'Unknown'
-        } (exit code ${terminated.exitCode})${
-          terminated.message ? ` - ${terminated.message}` : ''
-        }`
+      const reason = terminated.reason ?? 'Unknown'
+      const msg = terminated.message ? `\n    ${terminated.message}` : ''
+      containerLines.push(
+        `  ✗ container "${cs.name}" terminated: ${reason} (exit code ${terminated.exitCode})${msg}`
       )
     }
   }
+  if (containerLines.length) {
+    sections.push(`Container details:\n${containerLines.join('\n')}`)
+  }
 
-  lines.push(...(await describePodWarningEvents(podName)))
+  // --- Warning events ---
+  const eventLines = await describePodWarningEvents(podName)
+  if (eventLines.length) {
+    sections.push(`Recent warning events:\n${eventLines.join('\n')}`)
+  }
 
-  if (!lines.length) {
+  if (!sections.length) {
     return `No additional diagnostic information available for pod ${podName}`
   }
-  return lines.join('\n')
+  return sections.join('\n\n')
 }
 
 // Reads the most recent Warning events for a pod. Best-effort: listing events
@@ -839,13 +840,11 @@ async function describePodWarningEvents(podName: string): Promise<string[]> {
   const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS)
   const lines = recent.map(e => {
     const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
-    return `Event [Warning] ${e.reason ?? ''}${count}: ${
-      e.message ?? ''
-    }`.trim()
+    return `  [${e.reason ?? 'Warning'}]${count} ${e.message ?? ''}`.trimEnd()
   })
   if (warnings.length > recent.length) {
     lines.unshift(
-      `(showing ${recent.length} of ${warnings.length} warning events)`
+      `  (showing ${recent.length} of ${warnings.length} warning events)`
     )
   }
   return lines
@@ -871,7 +870,7 @@ export async function waitForPodPhases(
     if (!backOffPhases.has(phase)) {
       const details = await describePodFailure(podName)
       throw new Error(
-        `Pod ${podName} is unhealthy with phase status ${phase}\n${details}`
+        `Pod ${podName} is unhealthy (phase: ${phase})\n${'─'.repeat(60)}\n${details}`
       )
     }
 
@@ -882,9 +881,7 @@ export async function waitForPodPhases(
     if (containerErrors.length > 0) {
       const details = await describePodFailure(podName)
       throw new Error(
-        `Pod ${podName} has unrecoverable container errors: ${containerErrors.join(
-          '; '
-        )}\n${details}`
+        `Pod ${podName} has unrecoverable container errors:\n${containerErrors.join('\n')}\n${'─'.repeat(60)}\n${details}`
       )
     }
 
@@ -893,12 +890,10 @@ export async function waitForPodPhases(
     } catch (error) {
       // BackOffManager throws "backoff timeout" when maxTimeSeconds is exceeded.
       // Don't surface that bare message: collect diagnostics first so the user
-      // can see WHY the pod never became ready. This is the safest improvement
-      // -- it only runs when we were going to fail anyway and changes no
-      // success/failure decision.
+      // can see WHY the pod never became ready.
       const details = await describePodFailure(podName)
       throw new Error(
-        `Pod ${podName} is unhealthy: timed out after ${maxTimeSeconds}s in phase ${phase}\n${details}`
+        `Pod ${podName} timed out after ${maxTimeSeconds}s (phase: ${phase})\n${'─'.repeat(60)}\n${details}`
       )
     }
   }

From bbb2d3361c4ee227292c2470a0b22aa1d7a330bb Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 11:19:14 +0800
Subject: [PATCH 10/18] fix: clean up error layering and stray gitignore entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- prepare-job.ts: use err.message instead of String(err) to avoid
  double "Error: Error:" prefix; change separator to newline so the
  multi-section detail block renders on its own lines:
    pod failed to come online:
    Pod <name> has unrecoverable container errors:
      ✗ container "job": ErrImagePull
        <message>
    ────────────────────────────────────
    Pod status: Pending
      ✗ Ready=False ...
- .gitignore: fix corrupted line (tao/ was appended without newline
  to test-kind.yaml entry); restore original content, drop tao/
  (the values file should not be tracked in this repo)
- Dockerfile: restore to byte-identical match with release/no_volumes
  (only had trailing-newline diff, no content change)

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 .gitignore                            |  2 +-
 Dockerfile                            | 27 ++++++++++++++-------------
 packages/k8s/src/hooks/prepare-job.ts |  6 +++++-
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/.gitignore b/.gitignore
index b33ad537..a48fc2b5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,4 @@ node_modules/
 lib/
 dist/
 **/tests/_temp/**
-packages/k8s/tests/test-kind.yamltao/
+packages/k8s/tests/test-kind.yaml
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 9de8902d..6f758a7b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,13 +1,14 @@
-FROM swr.cn-north-4.myhuaweicloud.com/opensourceway/node:latest AS runner_builder
-
-WORKDIR /app
-
-COPY . /app/
-
-RUN npm install && npm run bootstrap && npm run build-all
-
-FROM ghcr.io/actions/actions-runner:2.334.0
-
-COPY --from=runner_builder /app/packages/k8s/dist/index.js /home/runner/k8s/index.js
-
-USER runner
+FROM swr.cn-north-4.myhuaweicloud.com/opensourceway/node:latest AS runner_builder
+
+WORKDIR /app
+
+COPY . /app/
+
+RUN npm install && npm run bootstrap && npm run build-all
+
+FROM ghcr.io/actions/actions-runner:2.334.0
+
+COPY --from=runner_builder /app/packages/k8s/dist/index.js /home/runner/k8s/index.js
+
+USER runner
+
diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts
index 28453c17..44aef8c0 100644
--- a/packages/k8s/src/hooks/prepare-job.ts
+++ b/packages/k8s/src/hooks/prepare-job.ts
@@ -112,7 +112,11 @@ export async function prepareJob(
     )
   } catch (err) {
     await prunePods()
-    throw new Error(`pod failed to come online with error: ${err}`)
+    // Unwrap nested "Error: " prefix so the message renders as:
+    //   pod failed to come online:
+    //   <detail from waitForPodPhases, already formatted with sections>
+    const detail = err instanceof Error ? err.message : String(err)
+    throw new Error(`pod failed to come online:\n${detail}`)
   }
 
   await execCpToPod(createdPod.metadata.name, runnerWorkspace, '/__w')

From 974ae50b4ab491f6c522c091c7a9a34daf3f3ddd Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 16:35:01 +0800
Subject: [PATCH 11/18] feat: detect unrecoverable pod event errors for
 fast-fail
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add event-based fast-fail detection alongside the existing container
waiting-reason check in waitForPodPhases().

Problem: some failures never surface in container.status.state.waiting
.reason — the container stays stuck in ContainerCreating — so the hook
polled until the 3600s timeout. Examples:
  - FailedMount: hostPath directory missing, missing Secret/ConfigMap volume
  - FailedScheduling: no schedulable node (resource/nodeSelector/affinity)
  - FailedBinding: PVC cannot bind (no PV, wrong storageClass)

Changes:
- UNRECOVERABLE_EVENT_REASONS: new exported Set with the three reasons above
- getUnrecoverableEventReasons(): mirrors getUnrecoverableWaitingReasons(),
  extended at runtime via ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS
  env var (comma-separated, additive)
- getPodEventErrors(): queries the event API for Warning events whose
  reason is in the unrecoverable set; deduplicates by reason; gracefully
  degrades to [] when the optional events RBAC permission is missing
- waitForPodPhases(): call getPodEventErrors() alongside getContainerErrors()
  each poll; fail fast when either returns results
- Error message updated: "unrecoverable container errors" →
  "unrecoverable errors" to cover both detection paths
- Tests: add coverage for FailedMount/FailedScheduling/FailedBinding event
  detection, env-var extension, RBAC degradation, and deduplication

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts                 |  98 ++++-
 .../k8s/tests/wait-for-pod-phases-test.ts     | 373 +++++++++++++-----
 2 files changed, 376 insertions(+), 95 deletions(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index 9de5ac68..de0c5842 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -682,6 +682,28 @@ export const UNRECOVERABLE_WAITING_REASONS = new Set([
   'CreateContainerError'
 ])
 
+// Pod *event* reasons (from the event stream, not container status) that
+// indicate a permanent failure the pod will never recover from on its own.
+// These surface as Warning events on the pod (visible via `kubectl describe`)
+// rather than in container.status.state.waiting.reason, so they need a separate
+// list + a separate (async) detection path.
+//
+//   - FailedMount: a volume cannot be mounted (e.g. hostPath `type: Directory`
+//     pointing at a path that does not exist on the node, missing PVC, missing
+//     Secret/ConfigMap volume). The container stays in `ContainerCreating`
+//     waiting state, which is NOT in UNRECOVERABLE_WAITING_REASONS, so without
+//     this check the hook polls until the 3600s timeout.
+//   - FailedScheduling: the scheduler cannot place the pod (insufficient
+//     cpu/memory/gpu, node selector / affinity mismatch, no matching nodes).
+//     The pod stays Pending with PodScheduled=False forever.
+//   - FailedBinding: a PVC could not be bound (no matching PV, storage class
+//     misconfiguration). Usually paired with FailedMount once the pod retries.
+export const UNRECOVERABLE_EVENT_REASONS = new Set([
+  'FailedMount',
+  'FailedScheduling',
+  'FailedBinding'
+])
+
 // Maximum number of Warning events to include in a pod's failure description so
 // that the GitHub Actions log is not flooded.
 const MAX_DIAGNOSTIC_EVENTS = 10
@@ -705,6 +727,24 @@ export function getUnrecoverableWaitingReasons(): Set<string> {
   return reasons
 }
 
+// Mirrors getUnrecoverableWaitingReasons() but for pod *event* reasons. The
+// env var ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS adds to (never removes
+// from) the built-in UNRECOVERABLE_EVENT_REASONS set.
+export function getUnrecoverableEventReasons(): Set<string> {
+  const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS']
+  if (!extra) {
+    return UNRECOVERABLE_EVENT_REASONS
+  }
+  const reasons = new Set(UNRECOVERABLE_EVENT_REASONS)
+  for (const reason of extra.split(',')) {
+    const trimmed = reason.trim()
+    if (trimmed) {
+      reasons.add(trimmed)
+    }
+  }
+  return reasons
+}
+
 export function getContainerErrors(pod: k8s.V1Pod): string[] {
   const errors: string[] = []
   const unrecoverableReasons = getUnrecoverableWaitingReasons()
@@ -724,6 +764,53 @@ export function getContainerErrors(pod: k8s.V1Pod): string[] {
   return errors
 }
 
+// Inspects the pod's Warning events for reasons in UNRECOVERABLE_EVENT_REASONS
+// (e.g. FailedMount when a hostPath directory does not exist). Unlike container
+// waiting reasons, these appear only in the event stream, so a separate API
+// call is required. Best-effort: if events cannot be listed (e.g. the optional
+// 'events' RBAC permission is missing) it returns an empty list instead of
+// blocking fast-fail detection of container-level errors. Deduplicates by
+// reason so a repeatedly retried FailedMount is reported once.
+export async function getPodEventErrors(podName: string): Promise<string[]> {
+  const unrecoverableReasons = getUnrecoverableEventReasons()
+  if (unrecoverableReasons.size === 0) {
+    return []
+  }
+
+  let items: k8s.CoreV1Event[]
+  try {
+    const result = await k8sApi.listNamespacedEvent({
+      namespace: namespace(),
+      fieldSelector: `involvedObject.name=${podName}`
+    })
+    items = result.items
+  } catch (err) {
+    core.debug(
+      `Could not list events for pod ${podName} during fast-fail check: ${
+        err instanceof Error ? err.message : String(err)
+      }`
+    )
+    return []
+  }
+
+  const errors: string[] = []
+  const seenReasons = new Set<string>()
+  for (const e of items) {
+    if (e.type !== 'Warning' || !e.reason || !unrecoverableReasons.has(e.reason)) {
+      continue
+    }
+    if (seenReasons.has(e.reason)) {
+      continue
+    }
+    seenReasons.add(e.reason)
+    const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
+    const reason = `  ✗ event: ${e.reason}${count}`
+    const detail = e.message ? `    ${e.message}` : ''
+    errors.push(detail ? `${reason}\n${detail}` : reason)
+  }
+  return errors
+}
+
 // describePodFailure aggregates everything that might explain why a pod is not
 // healthy into a single human-readable string. It makes NO success/failure
 // judgement of its own (a terminated exitCode=0 container is just reported as
@@ -878,10 +965,17 @@ export async function waitForPodPhases(
     // error (e.g. ImagePullBackOff) -- fail fast with diagnostics instead of
     // waiting out the full timeout.
     const containerErrors = getContainerErrors(pod)
-    if (containerErrors.length > 0) {
+    // Also check the pod's Warning events for unrecoverable event reasons such
+    // as FailedMount (hostPath directory missing). These never show up in
+    // container.status.state.waiting.reason -- the container stays in
+    // ContainerCreating -- so without this check the hook would poll until the
+    // 3600s timeout.
+    const eventErrors = await getPodEventErrors(podName)
+    if (containerErrors.length > 0 || eventErrors.length > 0) {
+      const allErrors = [...containerErrors, ...eventErrors]
       const details = await describePodFailure(podName)
       throw new Error(
-        `Pod ${podName} has unrecoverable container errors:\n${containerErrors.join('\n')}\n${'─'.repeat(60)}\n${details}`
+        `Pod ${podName} has unrecoverable errors:\n${allErrors.join('\n')}\n${'─'.repeat(60)}\n${details}`
       )
     }
 
diff --git a/packages/k8s/tests/wait-for-pod-phases-test.ts b/packages/k8s/tests/wait-for-pod-phases-test.ts
index 9ff2e3ec..60758615 100644
--- a/packages/k8s/tests/wait-for-pod-phases-test.ts
+++ b/packages/k8s/tests/wait-for-pod-phases-test.ts
@@ -2,8 +2,11 @@ import * as k8s from '@kubernetes/client-node'
 import {
   describePodFailure,
   getContainerErrors,
+  getPodEventErrors,
+  getUnrecoverableEventReasons,
   getUnrecoverableWaitingReasons,
   parsePodPhase,
+  UNRECOVERABLE_EVENT_REASONS,
   UNRECOVERABLE_WAITING_REASONS,
   waitForPodPhases
 } from '../src/k8s'
@@ -38,6 +41,30 @@ function waitingContainer(
   } as k8s.V1ContainerStatus
 }
 
+// Build a Warning event with the fields getPodEventErrors() inspects.
+function buildEvent(
+  reason: string,
+  message: string,
+  opts: { type?: string; count?: number } = {}
+): k8s.CoreV1Event {
+  return {
+    type: opts.type ?? 'Warning',
+    reason,
+    message,
+    count: opts.count,
+    lastTimestamp: new Date('2026-01-01T00:00:00Z')
+  } as k8s.CoreV1Event
+}
+
+// @kubernetes/client-node 1.x object-style API returns the object directly
+// (not wrapped in { body }), so mocks must do the same.
+function podResult(pod: k8s.V1Pod): never {
+  return pod as never
+}
+function eventResult(items: k8s.CoreV1Event[]): never {
+  return { items } as never
+}
+
 describe('parsePodPhase', () => {
   it('returns the phase when it is a known value', () => {
     expect(parsePodPhase(buildPod(PodPhase.RUNNING))).toBe(PodPhase.RUNNING)
@@ -74,11 +101,13 @@ describe('getContainerErrors', () => {
       const pod = buildPod(PodPhase.PENDING, {
         containerStatuses: [waitingContainer('job', reason)]
       })
-      expect(getContainerErrors(pod)).toEqual([`container "job": ${reason}`])
+      expect(getContainerErrors(pod)).toEqual([
+        `  ✗ container "job": ${reason}`
+      ])
     }
   })
 
-  it('includes the waiting message when present', () => {
+  it('includes the waiting message as an indented second line', () => {
     const pod = buildPod(PodPhase.PENDING, {
       containerStatuses: [
         waitingContainer(
@@ -89,7 +118,7 @@ describe('getContainerErrors', () => {
       ]
     })
     expect(getContainerErrors(pod)).toEqual([
-      'container "job": ErrImagePull - Back-off pulling image "does-not-exist:latest"'
+      '  ✗ container "job": ErrImagePull\n    Back-off pulling image "does-not-exist:latest"'
     ])
   })
 
@@ -99,8 +128,8 @@ describe('getContainerErrors', () => {
       containerStatuses: [waitingContainer('job', 'CreateContainerError')]
     })
     expect(getContainerErrors(pod)).toEqual([
-      'container "init": ImagePullBackOff',
-      'container "job": CreateContainerError'
+      '  ✗ container "init": ImagePullBackOff',
+      '  ✗ container "job": CreateContainerError'
     ])
   })
 
@@ -112,77 +141,104 @@ describe('getContainerErrors', () => {
       ]
     })
     expect(getContainerErrors(pod)).toEqual([
-      'container "bad": InvalidImageName'
+      '  ✗ container "bad": InvalidImageName'
     ])
   })
 })
 
-describe('waitForPodPhases', () => {
-  let readSpy: jest.SpyInstance
+describe('getPodEventErrors', () => {
   let eventSpy: jest.SpyInstance
 
   beforeEach(() => {
     process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default'
-    readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod')
-    // waitForPodPhases now calls describePodFailure on its failure paths, which
-    // lists events. Stub it out so the tests never hit a real cluster.
     eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent')
-    eventSpy.mockResolvedValue({ body: { items: [] } } as never)
   })
 
   afterEach(() => {
     jest.restoreAllMocks()
     delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+    delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS']
   })
 
-  it('returns once the pod reaches an awaited phase', async () => {
-    readSpy.mockResolvedValue({ body: buildPod(PodPhase.RUNNING) } as never)
+  it('returns no errors when there are no warning events', async () => {
+    eventSpy.mockResolvedValue(eventResult([]))
+    expect(await getPodEventErrors('my-pod')).toEqual([])
+  })
 
-    await expect(
-      waitForPodPhases(
-        'my-pod',
-        new Set([PodPhase.RUNNING]),
-        new Set([PodPhase.PENDING])
-      )
-    ).resolves.toBeUndefined()
+  it('returns no errors for warning events whose reason is not unrecoverable', async () => {
+    eventSpy.mockResolvedValue(
+      eventResult([
+        buildEvent('Unhealthy', 'Liveness probe failed'),
+        buildEvent('BackOff', 'container restart back-off')
+      ])
+    )
+    expect(await getPodEventErrors('my-pod')).toEqual([])
   })
 
-  it('surfaces unrecoverable container errors in the thrown message', async () => {
-    readSpy.mockResolvedValue({
-      body: buildPod(PodPhase.PENDING, {
-        containerStatuses: [
-          waitingContainer(
-            'job',
-            'ImagePullBackOff',
-            'Back-off pulling image "nope:latest"'
-          )
-        ]
-      })
-    } as never)
+  it('detects FailedMount (the hostPath Directory missing case)', async () => {
+    eventSpy.mockResolvedValue(
+      eventResult([
+        buildEvent(
+          'FailedMount',
+          'Unable to attach or mount volume "bad-hostpath": mount path "/this/path/does/not/exist" does not exist',
+          { count: 3 }
+        )
+      ])
+    )
+    expect(await getPodEventErrors('my-pod')).toEqual([
+      '  ✗ event: FailedMount (x3)\n    Unable to attach or mount volume "bad-hostpath": mount path "/this/path/does/not/exist" does not exist'
+    ])
+  })
 
-    await expect(
-      waitForPodPhases(
-        'my-pod',
-        new Set([PodPhase.RUNNING]),
-        new Set([PodPhase.PENDING])
-      )
-    ).rejects.toThrow(
-      'container "job": ImagePullBackOff - Back-off pulling image "nope:latest"'
+  it('detects every unrecoverable event reason', async () => {
+    eventSpy.mockResolvedValue(
+      eventResult([
+        buildEvent('FailedScheduling', '0/1 nodes are available'),
+        buildEvent('FailedBinding', 'no persistent volumes available'),
+        buildEvent('FailedMount', 'volume not found')
+      ])
     )
+    expect(await getPodEventErrors('my-pod')).toEqual([
+      '  ✗ event: FailedScheduling\n    0/1 nodes are available',
+      '  ✗ event: FailedBinding\n    no persistent volumes available',
+      '  ✗ event: FailedMount\n    volume not found'
+    ])
   })
 
-  it('throws with the phase when the pod is in a non-backoff phase', async () => {
-    readSpy.mockResolvedValue({ body: buildPod(PodPhase.FAILED) } as never)
+  it('ignores Normal-type events even if the reason matches', async () => {
+    eventSpy.mockResolvedValue(
+      eventResult([buildEvent('FailedMount', 'volume not found', { type: 'Normal' })])
+    )
+    expect(await getPodEventErrors('my-pod')).toEqual([])
+  })
 
-    // The message is no longer double-wrapped: it states the phase once and is
-    // followed by the diagnostic block from describePodFailure.
-    await expect(
-      waitForPodPhases(
-        'my-pod',
-        new Set([PodPhase.RUNNING]),
-        new Set([PodPhase.PENDING])
-      )
-    ).rejects.toThrow('Pod my-pod is unhealthy with phase status Failed')
+  it('deduplicates events that share the same reason', async () => {
+    eventSpy.mockResolvedValue(
+      eventResult([
+        buildEvent('FailedMount', 'first attempt', { count: 1 }),
+        buildEvent('FailedMount', 'second attempt', { count: 5 })
+      ])
+    )
+    // Only the first occurrence is kept.
+    expect(await getPodEventErrors('my-pod')).toEqual([
+      '  ✗ event: FailedMount\n    first attempt'
+    ])
+  })
+
+  it('honors extra reasons from the env var', async () => {
+    process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] =
+      'FailedPreStopHook'
+    eventSpy.mockResolvedValue(
+      eventResult([buildEvent('FailedPreStopHook', 'hook failed')])
+    )
+    expect(await getPodEventErrors('my-pod')).toEqual([
+      '  ✗ event: FailedPreStopHook\n    hook failed'
+    ])
+  })
+
+  it('degrades gracefully when listing events is forbidden', async () => {
+    eventSpy.mockRejectedValue(new Error('events is forbidden') as never)
+    expect(await getPodEventErrors('my-pod')).toEqual([])
   })
 })
 
@@ -215,11 +271,146 @@ describe('getUnrecoverableWaitingReasons', () => {
       containerStatuses: [waitingContainer('job', 'CrashLoopBackOff')]
     })
     expect(getContainerErrors(pod)).toEqual([
-      'container "job": CrashLoopBackOff'
+      '  ✗ container "job": CrashLoopBackOff'
     ])
   })
 })
 
+describe('getUnrecoverableEventReasons', () => {
+  afterEach(() => {
+    delete process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS']
+  })
+
+  it('returns the built-in defaults when the env var is unset', () => {
+    expect(getUnrecoverableEventReasons()).toEqual(UNRECOVERABLE_EVENT_REASONS)
+  })
+
+  it('adds extra reasons from the env var without dropping the defaults', () => {
+    process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS'] =
+      'FailedPreStopHook, FailedPostStartHook'
+    const reasons = getUnrecoverableEventReasons()
+    for (const builtin of Array.from(UNRECOVERABLE_EVENT_REASONS)) {
+      expect(reasons.has(builtin)).toBe(true)
+    }
+    expect(reasons.has('FailedPreStopHook')).toBe(true)
+    expect(reasons.has('FailedPostStartHook')).toBe(true)
+  })
+})
+
+describe('waitForPodPhases', () => {
+  let readSpy: jest.SpyInstance
+  let eventSpy: jest.SpyInstance
+
+  beforeEach(() => {
+    process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default'
+    readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod')
+    // waitForPodPhases now calls getPodEventErrors (lists events) on every poll
+    // for fast-fail detection, and describePodFailure on failure paths (also
+    // lists events). Stub it out so the tests never hit a real cluster.
+    eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent')
+    eventSpy.mockResolvedValue(eventResult([]))
+  })
+
+  afterEach(() => {
+    jest.restoreAllMocks()
+    delete process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+  })
+
+  it('returns once the pod reaches an awaited phase', async () => {
+    readSpy.mockResolvedValue(podResult(buildPod(PodPhase.RUNNING)))
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).resolves.toBeUndefined()
+  })
+
+  it('surfaces unrecoverable container errors in the thrown message', async () => {
+    readSpy.mockResolvedValue(
+      podResult(
+        buildPod(PodPhase.PENDING, {
+          containerStatuses: [
+            waitingContainer(
+              'job',
+              'ImagePullBackOff',
+              'Back-off pulling image "nope:latest"'
+            )
+          ]
+        })
+      )
+    )
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).rejects.toThrow('Pod my-pod has unrecoverable errors')
+  })
+
+  it('fast-fails on FailedMount events instead of polling to timeout', async () => {
+    // Container is in ContainerCreating (recoverable waiting reason), but the
+    // pod has a FailedMount Warning event -- this is the hostPath-missing case.
+    readSpy.mockResolvedValue(
+      podResult(
+        buildPod(PodPhase.PENDING, {
+          containerStatuses: [waitingContainer('job', 'ContainerCreating')]
+        })
+      )
+    )
+    eventSpy.mockResolvedValue(
+      eventResult([
+        buildEvent(
+          'FailedMount',
+          'mount path "/this/path/does/not/exist" does not exist',
+          { count: 4 }
+        )
+      ])
+    )
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).rejects.toThrow(
+      /has unrecoverable errors:[\s\S]*event: FailedMount \(x4\)/
+    )
+  })
+
+  it('fast-fails on FailedScheduling events', async () => {
+    readSpy.mockResolvedValue(podResult(buildPod(PodPhase.PENDING)))
+    eventSpy.mockResolvedValue(
+      eventResult([buildEvent('FailedScheduling', '0/1 nodes are available')])
+    )
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).rejects.toThrow(/event: FailedScheduling[\s\S]*0\/1 nodes are available/)
+  })
+
+  it('throws with the phase when the pod is in a non-backoff phase', async () => {
+    readSpy.mockResolvedValue(podResult(buildPod(PodPhase.FAILED)))
+
+    await expect(
+      waitForPodPhases(
+        'my-pod',
+        new Set([PodPhase.RUNNING]),
+        new Set([PodPhase.PENDING])
+      )
+    ).rejects.toThrow('Pod my-pod is unhealthy (phase: Failed)')
+  })
+})
+
 describe('describePodFailure', () => {
   let readSpy: jest.SpyInstance
   let eventSpy: jest.SpyInstance
@@ -229,7 +420,7 @@ describe('describePodFailure', () => {
     readSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'readNamespacedPod')
     eventSpy = jest.spyOn(k8s.CoreV1Api.prototype, 'listNamespacedEvent')
     // Default: no events. Individual tests override this.
-    eventSpy.mockResolvedValue({ body: { items: [] } } as never)
+    eventSpy.mockResolvedValue(eventResult([]))
   })
 
   afterEach(() => {
@@ -238,8 +429,8 @@ describe('describePodFailure', () => {
   })
 
   it('reports phase, failing conditions and terminated containers', async () => {
-    readSpy.mockResolvedValue({
-      body: {
+    readSpy.mockResolvedValue(
+      podResult({
         status: {
           phase: PodPhase.FAILED,
           reason: 'Evicted',
@@ -261,63 +452,59 @@ describe('describePodFailure', () => {
             }
           ]
         }
-      }
-    } as never)
+      } as k8s.V1Pod)
+    )
 
     const description = await describePodFailure('my-pod')
-    expect(description).toContain('Phase: Failed (reason: Evicted)')
+    expect(description).toContain('Pod status: Failed (Evicted)')
+    expect(description).toContain('The node was low on resource: memory')
     expect(description).toContain(
-      'Message: The node was low on resource: memory'
+      '✗ PodScheduled=False (Unschedulable): no nodes available'
     )
     expect(description).toContain(
-      'Condition PodScheduled=False (reason: Unschedulable): no nodes available'
-    )
-    expect(description).toContain(
-      'Container "job" terminated: OOMKilled (exit code 137)'
+      '✗ container "job" terminated: OOMKilled (exit code 137)'
     )
   })
 
   it('includes recent Warning events and skips Normal ones', async () => {
-    readSpy.mockResolvedValue({
-      body: { status: { phase: PodPhase.PENDING } }
-    } as never)
-    eventSpy.mockResolvedValue({
-      body: {
-        items: [
-          {
-            type: 'Normal',
-            reason: 'Scheduled',
-            message: 'assigned',
-            lastTimestamp: new Date('2026-01-01T00:00:00Z')
-          },
-          {
-            type: 'Warning',
-            reason: 'FailedScheduling',
-            message: '0/1 nodes are available',
-            count: 3,
-            lastTimestamp: new Date('2026-01-01T00:01:00Z')
-          }
-        ]
-      }
-    } as never)
+    readSpy.mockResolvedValue(
+      podResult({ status: { phase: PodPhase.PENDING } } as k8s.V1Pod)
+    )
+    eventSpy.mockResolvedValue(
+      eventResult([
+        {
+          type: 'Normal',
+          reason: 'Scheduled',
+          message: 'assigned',
+          lastTimestamp: new Date('2026-01-01T00:00:00Z')
+        } as k8s.CoreV1Event,
+        {
+          type: 'Warning',
+          reason: 'FailedScheduling',
+          message: '0/1 nodes are available',
+          count: 3,
+          lastTimestamp: new Date('2026-01-01T00:01:00Z')
+        } as k8s.CoreV1Event
+      ])
+    )
 
     const description = await describePodFailure('my-pod')
     expect(description).toContain(
-      'Event [Warning] FailedScheduling (x3): 0/1 nodes are available'
+      '[FailedScheduling] (x3) 0/1 nodes are available'
     )
-    expect(description).not.toContain('Scheduled')
+    expect(description).not.toContain('[Scheduled]')
   })
 
   it('degrades gracefully when listing events is forbidden', async () => {
-    readSpy.mockResolvedValue({
-      body: { status: { phase: PodPhase.PENDING } }
-    } as never)
+    readSpy.mockResolvedValue(
+      podResult({ status: { phase: PodPhase.PENDING } } as k8s.V1Pod)
+    )
     eventSpy.mockRejectedValue(new Error('events is forbidden') as never)
 
     const description = await describePodFailure('my-pod')
-    expect(description).toContain('Phase: Pending')
+    expect(description).toContain('Pod status: Pending')
     // No throw, and no event lines.
-    expect(description).not.toContain('Event [Warning]')
+    expect(description).not.toContain('[Warning]')
   })
 
   it('never throws when the pod cannot be read', async () => {

From 6c588869546aaba12374bbc2e0b081d2f7b20151 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 17:02:38 +0800
Subject: [PATCH 12/18] fix: extract k8s API message from raw HttpException for
 pod create errors

The k8s client throws HttpException with a multi-line message:
  HTTP-Code: 422
  Message: Unknown API Status Code!
  Body: "{\"message\":\"Pod is invalid: ...\"}"

Parse the "message" field from the embedded JSON body so the log shows:
  failed to create job pod:
    Pod "xxx" is invalid: spec.volumes[5].name: Duplicate value: "bad-hostpath"

Falls back to the raw string if parsing fails (e.g. non-422 errors or
network errors that don't have a JSON body).

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/hooks/prepare-job.ts | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts
index 44aef8c0..7e22298c 100644
--- a/packages/k8s/src/hooks/prepare-job.ts
+++ b/packages/k8s/src/hooks/prepare-job.ts
@@ -85,8 +85,18 @@ export async function prepareJob(
   } catch (err) {
     await prunePods()
     core.debug(`createPod failed: ${JSON.stringify(err)}`)
-    const message = (err as any)?.response?.body?.message || err
-    throw new Error(`failed to create job pod: ${message}`)
+    // The k8s client throws HttpException whose message is a multi-line string
+    // containing the raw HTTP dump. Extract the human-readable "message" field
+    // from the embedded JSON body so the log shows something like:
+    //   failed to create job pod:
+    //     spec.volumes[5].name: Duplicate value: "bad-hostpath"
+    // instead of the full HTTP dump.
+    const raw = err instanceof Error ? err.message : String(err)
+    const msgMatch = raw.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/s)
+    const detail = msgMatch
+      ? msgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, '\n')
+      : raw
+    throw new Error(`failed to create job pod:\n  ${detail}`)
   }
 
   if (!createdPod?.metadata?.name) {

From 90453565056457d8fab81f188c54280eda7e3ba3 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 17:23:13 +0800
Subject: [PATCH 13/18] fix: remove regex 's' flag incompatible with pre-ES2018
 tsconfig target
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The dotAll flag (s) requires ES2018+. The JSON body is single-line so
the flag is unnecessary — remove it to restore build compatibility.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/hooks/prepare-job.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts
index 7e22298c..ef22ec4e 100644
--- a/packages/k8s/src/hooks/prepare-job.ts
+++ b/packages/k8s/src/hooks/prepare-job.ts
@@ -92,7 +92,7 @@ export async function prepareJob(
     //     spec.volumes[5].name: Duplicate value: "bad-hostpath"
     // instead of the full HTTP dump.
     const raw = err instanceof Error ? err.message : String(err)
-    const msgMatch = raw.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/s)
+    const msgMatch = raw.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/)
     const detail = msgMatch
       ? msgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, '\n')
       : raw

From 75aaa7e8cc36d4b194a20d71759f2d3dcf95862c Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 17:44:27 +0800
Subject: [PATCH 14/18] fix: reliably extract k8s API error message from
 HttpException body
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace the broken regex approach with boundary-based extraction:
1. Find 'Body: "' and '"\nHeaders:' delimiters in the raw dump
2. Unescape the JSON string (\" → " and \ → \)
3. JSON.parse and return parsed.message

Verified against the actual 422 response format produced by the k8s
client (Body contains a JSON-encoded string with \" escaping inside).
Falls back to the full raw string on any parse failure.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/hooks/prepare-job.ts | 28 +++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts
index ef22ec4e..851ef05d 100644
--- a/packages/k8s/src/hooks/prepare-job.ts
+++ b/packages/k8s/src/hooks/prepare-job.ts
@@ -92,10 +92,30 @@ export async function prepareJob(
     //     spec.volumes[5].name: Duplicate value: "bad-hostpath"
     // instead of the full HTTP dump.
     const raw = err instanceof Error ? err.message : String(err)
-    const msgMatch = raw.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/)
-    const detail = msgMatch
-      ? msgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, '\n')
-      : raw
+    let detail = raw
+    // The k8s HttpException message is a multi-line dump:
+    //   HTTP-Code: 422
+    //   Body: "{\"kind\":\"Status\",\"message\":\"...\", ...}"
+    //   Headers: {...}
+    // Extract the Body JSON string, unescape it, and pull out "message".
+    try {
+      const bodyStart = raw.indexOf('Body: "')
+      const bodyEnd = raw.indexOf('"\nHeaders:')
+      if (bodyStart !== -1 && bodyEnd !== -1 && bodyEnd > bodyStart) {
+        const escaped = raw.substring(bodyStart + 7, bodyEnd)
+        // Body uses JSON string escaping: \" → " and \\ → \
+        const unescaped = escaped
+          .replace(/\\\\/g, '\x00') // protect \\ first
+          .replace(/\\"/g, '"')     // unescape \"
+          .replace(/\x00/g, '\\')  // restore \\
+        const parsed = JSON.parse(unescaped)
+        if (typeof parsed?.message === 'string') {
+          detail = parsed.message
+        }
+      }
+    } catch {
+      // Parsing failed — fall through and show the raw string
+    }
     throw new Error(`failed to create job pod:\n  ${detail}`)
   }
 

From 574910cd069c732af001df959f5df071e91224e4 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Sat, 27 Jun 2026 19:11:32 +0800
Subject: [PATCH 15/18] fix: use lastIndexOf to find Body boundary before
 Headers line
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous '"\nHeaders:' boundary failed when the newline between
Body and Headers was not a real newline in the error string. Instead,
find 'Headers:' first and then use lastIndexOf('"') to locate the
closing quote of the Body value — more robust against whitespace
variations in the HttpException dump format.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/hooks/prepare-job.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts
index 851ef05d..ccacd851 100644
--- a/packages/k8s/src/hooks/prepare-job.ts
+++ b/packages/k8s/src/hooks/prepare-job.ts
@@ -100,7 +100,12 @@ export async function prepareJob(
     // Extract the Body JSON string, unescape it, and pull out "message".
     try {
       const bodyStart = raw.indexOf('Body: "')
-      const bodyEnd = raw.indexOf('"\nHeaders:')
+      // The boundary may be '"\nHeaders:' (real newline) or the literal
+      // string ends before Headers — use the last '"' before 'Headers:' as fallback
+      const headersIdx = raw.indexOf('Headers:')
+      const bodyEnd = headersIdx !== -1
+        ? raw.lastIndexOf('"', headersIdx) - 0   // last " before Headers:
+        : raw.indexOf('"\nHeaders:')
       if (bodyStart !== -1 && bodyEnd !== -1 && bodyEnd > bodyStart) {
         const escaped = raw.substring(bodyStart + 7, bodyEnd)
         // Body uses JSON string escaping: \" → " and \\ → \

From 3d1ad69611ce3ff1678e4e9fc1407ab95a1656d6 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Mon, 29 Jun 2026 09:27:54 +0800
Subject: [PATCH 16/18] feat: add condition-based fast-fail as RBAC-free
 fallback for scheduling errors
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

getPodConditionErrors() checks pod.status.conditions without needing
the optional 'events' list RBAC permission:
  - PodScheduled=False (Unschedulable): nodeSelector mismatch, resource
    requests exceed limits, no matching node

In waitForPodPhases():
  - eventErrors (events-API based) is tried first; it's richer (carries
    event.count and detailed message)
  - conditionErrors is only added when eventErrors is empty, i.e. when
    the events RBAC permission is absent — avoids duplicate output

This means FailedScheduling fast-fail now works in both cases:
  - RBAC allows events   → FailedScheduling event detected
  - RBAC blocks events   → PodScheduled=False condition detected

FailedBinding still relies on the FailedMount event path (the PVC
binding failure eventually surfaces as a FailedMount pod event).

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts | 45 +++++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index de0c5842..e253819a 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -764,6 +764,33 @@ export function getContainerErrors(pod: k8s.V1Pod): string[] {
   return errors
 }
 
+// Inspects the pod's status.conditions for deterministic scheduling failures.
+// This is the RBAC-free companion to getPodEventErrors: pod conditions are
+// always readable without the optional 'events' list permission.
+//
+//   PodScheduled=False/Unschedulable: no node matched the pod's requirements
+//   (nodeSelector mismatch, insufficient resources). The scheduler sets this
+//   condition almost immediately and continues retrying. Self-healing IS
+//   possible (a node might free up), so this is a fallback used only when
+//   getPodEventErrors returns nothing (events RBAC unavailable). When events
+//   ARE available, the FailedScheduling event fires first and gates by count.
+//
+// Never throws; on any unexpected error it returns [].
+export function getPodConditionErrors(pod: k8s.V1Pod): string[] {
+  const errors: string[] = []
+  for (const cond of pod.status?.conditions ?? []) {
+    if (
+      cond.type === 'PodScheduled' &&
+      cond.status === 'False' &&
+      cond.reason === 'Unschedulable'
+    ) {
+      const msg = cond.message ? `\n    ${cond.message}` : ''
+      errors.push(`  ✗ condition: ${cond.type}=False (${cond.reason})${msg}`)
+    }
+  }
+  return errors
+}
+
 // Inspects the pod's Warning events for reasons in UNRECOVERABLE_EVENT_REASONS
 // (e.g. FailedMount when a hostPath directory does not exist). Unlike container
 // waiting reasons, these appear only in the event stream, so a separate API
@@ -965,14 +992,18 @@ export async function waitForPodPhases(
     // error (e.g. ImagePullBackOff) -- fail fast with diagnostics instead of
     // waiting out the full timeout.
     const containerErrors = getContainerErrors(pod)
-    // Also check the pod's Warning events for unrecoverable event reasons such
-    // as FailedMount (hostPath directory missing). These never show up in
-    // container.status.state.waiting.reason -- the container stays in
-    // ContainerCreating -- so without this check the hook would poll until the
-    // 3600s timeout.
+    // Check pod Warning events (e.g. FailedMount, FailedScheduling). Best-effort:
+    // silently returns [] when the optional 'events' RBAC permission is absent.
     const eventErrors = await getPodEventErrors(podName)
-    if (containerErrors.length > 0 || eventErrors.length > 0) {
-      const allErrors = [...containerErrors, ...eventErrors]
+    // Check pod conditions as RBAC-free fallback for scheduling failures
+    // (PodScheduled=False/Unschedulable). Deduplicates with eventErrors: if both
+    // fire for the same FailedScheduling, the eventErrors entry takes precedence
+    // (it has richer count/message), so we only add conditionErrors when eventErrors
+    // is empty (i.e. events RBAC is unavailable).
+    const conditionErrors =
+      eventErrors.length === 0 ? getPodConditionErrors(pod) : []
+    if (containerErrors.length > 0 || eventErrors.length > 0 || conditionErrors.length > 0) {
+      const allErrors = [...containerErrors, ...eventErrors, ...conditionErrors]
       const details = await describePodFailure(podName)
       throw new Error(
         `Pod ${podName} has unrecoverable errors:\n${allErrors.join('\n')}\n${'─'.repeat(60)}\n${details}`

From 37404902b910aec3520605410d90fab7a3f3de7e Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Mon, 29 Jun 2026 12:33:53 +0800
Subject: [PATCH 17/18] fix: always run condition-based fast-fail, not only
 when events are empty

The previous logic only called getPodConditionErrors() when
getPodEventErrors() returned [] (events RBAC unavailable). This created
a race: if events take a few extra seconds to propagate, the first
several polls would return eventErrors=[], run conditionErrors (and
detect Unschedulable), but later polls (after events appear) would skip
conditionErrors even though events may show an unrelated reason.

Change: always run both checks each poll iteration. Deduplicate with a
simple filter so FailedScheduling (event) + Unschedulable (condition)
for the same pod don't both appear in the error output.

This ensures PodScheduled=False/Unschedulable is detected reliably even
when events RBAC permission is absent, covering:
  - FailedScheduling (nodeSelector mismatch)
  - FailedScheduling (resource shortage including NPU)
  - FailedMount (pod stuck Pending because NPU is busy)

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/src/k8s/index.ts | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts
index e253819a..4339d3c2 100644
--- a/packages/k8s/src/k8s/index.ts
+++ b/packages/k8s/src/k8s/index.ts
@@ -992,16 +992,19 @@ export async function waitForPodPhases(
     // error (e.g. ImagePullBackOff) -- fail fast with diagnostics instead of
     // waiting out the full timeout.
     const containerErrors = getContainerErrors(pod)
-    // Check pod Warning events (e.g. FailedMount, FailedScheduling). Best-effort:
-    // silently returns [] when the optional 'events' RBAC permission is absent.
+    // Check pod Warning events (FailedMount, FailedScheduling, FailedBinding).
+    // Best-effort: returns [] when the optional events RBAC permission is absent.
     const eventErrors = await getPodEventErrors(podName)
-    // Check pod conditions as RBAC-free fallback for scheduling failures
-    // (PodScheduled=False/Unschedulable). Deduplicates with eventErrors: if both
-    // fire for the same FailedScheduling, the eventErrors entry takes precedence
-    // (it has richer count/message), so we only add conditionErrors when eventErrors
-    // is empty (i.e. events RBAC is unavailable).
-    const conditionErrors =
-      eventErrors.length === 0 ? getPodConditionErrors(pod) : []
+    // ALWAYS run condition check (not just as fallback when events are empty):
+    // conditions are set near-instantly by the scheduler while events may take
+    // a few extra seconds to propagate. Running both ensures we catch scheduling
+    // failures even if events appear slightly later. Deduplicate the output so
+    // the same failure isn't printed twice (event and condition both fire for
+    // FailedScheduling when RBAC works).
+    const rawConditionErrors = getPodConditionErrors(pod)
+    const conditionErrors = rawConditionErrors.filter(
+      c => !eventErrors.some(e => e.includes('FailedScheduling') && c.includes('Unschedulable'))
+    )
     if (containerErrors.length > 0 || eventErrors.length > 0 || conditionErrors.length > 0) {
       const allErrors = [...containerErrors, ...eventErrors, ...conditionErrors]
       const details = await describePodFailure(podName)

From 20e2ef25a6b59f729cc09cab6b18c61defe66357 Mon Sep 17 00:00:00 2001
From: Longwt123 <1075015495@qq.com>
Date: Tue, 30 Jun 2026 10:28:20 +0800
Subject: [PATCH 18/18] chore: bump js-yaml to 4.2.0 and uuid to 11.1.1, clean
 up jest setup comment

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
---
 packages/k8s/jest.setup.js | 1 -
 packages/k8s/package.json  | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/k8s/jest.setup.js b/packages/k8s/jest.setup.js
index a7551362..fca39576 100644
--- a/packages/k8s/jest.setup.js
+++ b/packages/k8s/jest.setup.js
@@ -1,2 +1 @@
-// eslint-disable-next-line filenames/match-regex, no-undef
 jest.setTimeout(500000)
diff --git a/packages/k8s/package.json b/packages/k8s/package.json
index 6821b7c1..1a33c6aa 100644
--- a/packages/k8s/package.json
+++ b/packages/k8s/package.json
@@ -18,10 +18,10 @@
     "@actions/io": "^1.1.3",
     "@kubernetes/client-node": "^1.3.0",
     "hooklib": "file:../hooklib",
-    "js-yaml": "^4.1.0",
+    "js-yaml": "^4.2.0",
     "shlex": "^3.0.0",
     "tar-fs": "^3.1.0",
-    "uuid": "^11.1.0"
+    "uuid": "^11.1.1"
   },
   "devDependencies": {
     "@babel/core": "^7.28.3",