From 0baef43134ed80837337a3977ced1533326321c5 Mon Sep 17 00:00:00 2001 From: tfhddd <2272751277@qq.com> Date: Wed, 4 Mar 2026 17:04:29 +0800 Subject: [PATCH 1/6] =?UTF-8?q?fix:=20=E6=8E=92=E6=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/k8s/src/hooks/prepare-job.ts | 118 ++++++++++++++++++++------ packages/k8s/src/index.ts | 9 +- 2 files changed, 99 insertions(+), 28 deletions(-) diff --git a/packages/k8s/src/hooks/prepare-job.ts b/packages/k8s/src/hooks/prepare-job.ts index 28453c17..d855056d 100644 --- a/packages/k8s/src/hooks/prepare-job.ts +++ b/packages/k8s/src/hooks/prepare-job.ts @@ -39,14 +39,25 @@ export async function prepareJob( args: PrepareJobArgs, responseFile ): Promise { + core.debug('[prepareJob] Step 1: validating args.container') if (!args.container) { throw new Error('Job Container is required.') } + core.debug( + `[prepareJob] container image: ${args.container.image}, services: ${args.services?.length ?? 0}` + ) + core.debug('[prepareJob] Step 2: pruning stale pods') await prunePods() + core.debug('[prepareJob] Step 2: prunePods done') + core.debug('[prepareJob] Step 3: reading extension from file') const extension = readExtensionFromFile() + core.debug( + `[prepareJob] Step 3: extension loaded: ${extension ? 'yes' : 'none'}` + ) + core.debug('[prepareJob] Step 4: building main container spec') let container: k8s.V1Container | undefined = undefined if (args.container?.image) { container = createContainerSpec( @@ -55,24 +66,33 @@ export async function prepareJob( true, extension ) + core.debug(`[prepareJob] Step 4: main container spec built: ${container.name}, image: ${container.image}`) + } else { + core.debug('[prepareJob] Step 4: no image specified, skipping main container') } + core.debug('[prepareJob] Step 5: building service container specs') let services: k8s.V1Container[] = [] if (args.services?.length) { services = args.services.map(service => { - return createContainerSpec( + const spec = createContainerSpec( service, generateContainerName(service.image), false, extension ) + core.debug(`[prepareJob] Step 5: service container spec built: ${spec.name}, image: ${spec.image}`) + return spec }) + } else { + core.debug('[prepareJob] Step 5: no services') } if (!container && !services?.length) { throw new Error('No containers exist, skipping hook invocation') } + core.debug(`[prepareJob] Step 6: creating job pod (name: ${getJobPodName()})`) let createdPod: k8s.V1Pod | undefined = undefined try { createdPod = await createJobPod( @@ -82,9 +102,10 @@ export async function prepareJob( args.container.registry, extension ) + core.debug(`[prepareJob] Step 6: job pod created: ${createdPod?.metadata?.name}`) } catch (err) { await prunePods() - core.debug(`createPod failed: ${JSON.stringify(err)}`) + core.debug(`[prepareJob] Step 6 FAILED — createPod error: ${JSON.stringify(err)}`) const message = (err as any)?.response?.body?.message || err throw new Error(`failed to create job pod: ${message}`) } @@ -92,68 +113,111 @@ export async function prepareJob( if (!createdPod?.metadata?.name) { throw new Error('created pod should have metadata.name') } + + const podName = createdPod.metadata.name + const timeoutSecs = getPrepareJobTimeoutSeconds() core.debug( - `Job pod created, waiting for it to come online ${createdPod?.metadata?.name}` + `[prepareJob] Step 7: waiting for pod "${podName}" to reach RUNNING phase (timeout: ${timeoutSecs}s)` ) - - const runnerWorkspace = dirname(process.env.RUNNER_WORKSPACE as string) - - let prepareScript: { containerPath: string; runnerPath: string } | undefined - if (args.container?.userMountVolumes?.length) { - prepareScript = prepareJobScript(args.container.userMountVolumes || []) - } - try { await waitForPodPhases( - createdPod.metadata.name, + podName, new Set([PodPhase.RUNNING]), new Set([PodPhase.PENDING]), - getPrepareJobTimeoutSeconds() + timeoutSecs ) + core.debug(`[prepareJob] Step 7: pod "${podName}" is RUNNING`) } catch (err) { await prunePods() throw new Error(`pod failed to come online with error: ${err}`) } - await execCpToPod(createdPod.metadata.name, runnerWorkspace, '/__w') + const runnerWorkspace = dirname(process.env.RUNNER_WORKSPACE as string) + core.debug( + `[prepareJob] Step 8: copying runnerWorkspace "${runnerWorkspace}" to pod "${podName}" at /__w` + ) + try { + await execCpToPod(podName, runnerWorkspace, '/__w') + core.debug('[prepareJob] Step 8: copy to /__w done') + } catch (err) { + core.debug(`[prepareJob] Step 8 FAILED — execCpToPod error: ${JSON.stringify(err)}`) + throw new Error(`failed to copy runner workspace to pod: ${err}`) + } + + let prepareScript: { containerPath: string; runnerPath: string } | undefined + if (args.container?.userMountVolumes?.length) { + core.debug( + `[prepareJob] Step 9: generating prepareJobScript for ${args.container.userMountVolumes.length} userMountVolumes` + ) + prepareScript = prepareJobScript(args.container.userMountVolumes || []) + core.debug( + `[prepareJob] Step 9: prepareScript containerPath: ${prepareScript.containerPath}` + ) + } else { + core.debug('[prepareJob] Step 9: no userMountVolumes, skipping prepareScript') + } if (prepareScript) { - await execPodStep( - ['sh', '-e', prepareScript.containerPath], - createdPod.metadata.name, - JOB_CONTAINER_NAME + core.debug( + `[prepareJob] Step 10: executing prepareScript in pod "${podName}" container "${JOB_CONTAINER_NAME}"` ) + try { + await execPodStep( + ['sh', '-e', prepareScript.containerPath], + podName, + JOB_CONTAINER_NAME + ) + core.debug('[prepareJob] Step 10: prepareScript execution done') + } catch (err) { + core.debug(`[prepareJob] Step 10 FAILED — execPodStep error: ${JSON.stringify(err)}`) + throw new Error(`failed to execute prepareScript in pod: ${err}`) + } + core.debug('[prepareJob] Step 11: copying userMountVolumes to pod') const promises: Promise[] = [] for (const vol of args?.container?.userMountVolumes || []) { + core.debug( + `[prepareJob] Step 11: copying "${vol.sourceVolumePath}" -> "${vol.targetVolumePath}"` + ) promises.push( execCpToPod( - createdPod.metadata.name, + podName, vol.sourceVolumePath, vol.targetVolumePath - ) + ).catch(err => { + core.debug( + `[prepareJob] Step 11 FAILED — copy "${vol.sourceVolumePath}" -> "${vol.targetVolumePath}": ${JSON.stringify(err)}` + ) + throw new Error( + `failed to copy volume "${vol.sourceVolumePath}" to pod: ${err}` + ) + }) ) } await Promise.all(promises) + core.debug('[prepareJob] Step 11: all userMountVolumes copied') } - core.debug('Job pod is ready for traffic') + core.debug('[prepareJob] Step 12: pod is ready for traffic') + core.debug( + `[prepareJob] Step 13: checking if container "${JOB_CONTAINER_NAME}" in pod "${podName}" is Alpine` + ) let isAlpine = false try { - isAlpine = await isPodContainerAlpine( - createdPod.metadata.name, - JOB_CONTAINER_NAME - ) + isAlpine = await isPodContainerAlpine(podName, JOB_CONTAINER_NAME) + core.debug(`[prepareJob] Step 13: isAlpine = ${isAlpine}`) } catch (err) { core.debug( - `Failed to determine if the pod is alpine: ${JSON.stringify(err)}` + `[prepareJob] Step 13 FAILED — isPodContainerAlpine error: ${JSON.stringify(err)}` ) const message = (err as any)?.response?.body?.message || err throw new Error(`failed to determine if the pod is alpine: ${message}`) } - core.debug(`Setting isAlpine to ${isAlpine}`) + + core.debug(`[prepareJob] Step 14: writing response file "${responseFile}"`) generateResponseFile(responseFile, args, createdPod, isAlpine) + core.debug('[prepareJob] Step 14: response file written — prepareJob complete') } function generateResponseFile( diff --git a/packages/k8s/src/index.ts b/packages/k8s/src/index.ts index 291e20d7..9fa4cfde 100644 --- a/packages/k8s/src/index.ts +++ b/packages/k8s/src/index.ts @@ -48,7 +48,14 @@ async function run(): Promise { throw new Error(`Command not recognized: ${command}`) } } catch (error) { - core.error(error as Error) + const errMsg = error instanceof Error ? error.stack || error.message : String(error) + core.error('='.repeat(60)) + core.error('HOOK FATAL ERROR — PrepareJob failed') + core.error('='.repeat(60)) + core.error(errMsg) + core.error('='.repeat(60)) + core.error('Sleeping 10 minutes to allow pod inspection for debugging...') + await new Promise(resolve => setTimeout(resolve, 10 * 60 * 1000)) process.exit(1) } } From 95896083d637cdb774e6c68e70ea2f8e7f7bda37 Mon Sep 17 00:00:00 2001 From: tfhddd <2272751277@qq.com> Date: Thu, 5 Mar 2026 10:45:58 +0800 Subject: [PATCH 2/6] =?UTF-8?q?fix:=20=E6=8E=92=E6=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/k8s/src/k8s/index.ts | 59 ++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 36696a73..f921c190 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -391,16 +391,57 @@ export async function execCpToPod( const exec = new k8s.Exec(kc) // Use tar to extract with --no-same-owner to avoid ownership issues. // Then use find to fix permissions. The -m flag helps but we also need to fix permissions after. + // stderr is NOT suppressed on tar so errors are visible in the runner debug log. const command = [ 'sh', '-c', - `tar xf - --no-same-owner -C ${shlex.quote(containerPath)} 2>/dev/null; ` + + `tar xf - --no-same-owner -C ${shlex.quote(containerPath)}; ` + `find ${shlex.quote(containerPath)} -type f -exec chmod u+rw {} \\; 2>/dev/null; ` + `find ${shlex.quote(containerPath)} -type d -exec chmod u+rwx {} \\; 2>/dev/null` ] - const readStream = tar.pack(runnerPath) + + // Pre-pack diagnostic: log host directory info to help diagnose Linux environment issues. + try { + const { readdirSync } = await import('fs') + const entries = readdirSync(runnerPath, { withFileTypes: true }) + const specialFiles = entries + .filter(e => !e.isFile() && !e.isDirectory() && !e.isSymbolicLink()) + .map(e => { + const type = e.isFIFO() ? 'fifo' + : e.isSocket() ? 'socket' + : e.isBlockDevice() ? 'blockdev' + : e.isCharacterDevice() ? 'chardev' + : 'unknown' + return `${e.name}(${type})` + }) + core.debug( + `[execCpToPod] host dir "${runnerPath}": totalEntries=${entries.length}, specialFiles=${specialFiles.join(',') || 'none'}` + ) + } catch (statErr) { + core.debug(`[execCpToPod] pre-pack readdir failed: ${statErr}`) + } + + // Track the last entry being packed so we know which file caused an error. + let lastPackedEntry = '' + const readStream = tar.pack(runnerPath, { + map: (header: tar.Headers) => { + lastPackedEntry = header.name + return header + } + }) const errStream = new WritableStreamBuffer() + await new Promise((resolve, reject) => { + // Without this listener, an error from tar.pack() (e.g. unreadable file) + // would be an unhandled 'error' event and crash the Node.js process. + readStream.on('error', (err: Error) => { + const e = err as NodeJS.ErrnoException + core.debug( + `[execCpToPod] tar.pack error: code=${e.code}, path="${e.path}", lastEntry="${lastPackedEntry}", message="${e.message}"` + ) + reject(new Error(`tar.pack error [${e.code}] on "${e.path ?? lastPackedEntry}": ${e.message}`)) + }) + exec .exec( namespace(), @@ -412,17 +453,25 @@ export async function execCpToPod( readStream, false, async status => { - if (errStream.size()) { + const stderr = errStream.getContentsAsString() || '' + core.debug( + `[execCpToPod] exec callback: status=${JSON.stringify(status)}, lastEntry="${lastPackedEntry}", stderr=${stderr || '(empty)'}` + ) + if (status.status === 'Failure' || errStream.size()) { reject( new Error( - `Error from execCpToPod - status: ${status.status}, details: \n ${errStream.getContentsAsString()}` + `Error from execCpToPod - status: ${JSON.stringify(status)}, stderr:\n${stderr}` ) ) + return } resolve(status) } ) - .catch(e => reject(e)) + .catch(e => { + core.debug(`[execCpToPod] exec.exec() rejected: ${e}`) + reject(e) + }) }) break } catch (error) { From c932078e9e128d71c6309c1d0726e4d4653b41ef Mon Sep 17 00:00:00 2001 From: tfhddd <2272751277@qq.com> Date: Thu, 5 Mar 2026 11:12:39 +0800 Subject: [PATCH 3/6] =?UTF-8?q?fix:=20=E6=8E=92=E6=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/k8s/src/index.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/k8s/src/index.ts b/packages/k8s/src/index.ts index 9fa4cfde..1de5057d 100644 --- a/packages/k8s/src/index.ts +++ b/packages/k8s/src/index.ts @@ -60,4 +60,19 @@ async function run(): Promise { } } +process.on('uncaughtException', (err: Error) => { + const e = err as NodeJS.ErrnoException + core.error(`[GLOBAL] uncaughtException: code=${e.code}, message=${e.message}`) + core.error(e.stack ?? String(e)) + process.exit(1) +}) + +process.on('unhandledRejection', (reason: unknown) => { + const err = reason instanceof Error ? reason : new Error(String(reason)) + const e = err as NodeJS.ErrnoException + core.error(`[GLOBAL] unhandledRejection: code=${e.code}, message=${e.message}`) + core.error(e.stack ?? String(e)) + process.exit(1) +}) + void run() From 3f938af92a00b35a3de6dce50698efae7e47573b Mon Sep 17 00:00:00 2001 From: tfhddd <2272751277@qq.com> Date: Thu, 5 Mar 2026 11:25:08 +0800 Subject: [PATCH 4/6] =?UTF-8?q?fix:=20=E6=8E=92=E6=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/k8s/src/index.ts | 5 ++++ packages/k8s/src/k8s/index.ts | 46 ++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/packages/k8s/src/index.ts b/packages/k8s/src/index.ts index 1de5057d..c3b1c64f 100644 --- a/packages/k8s/src/index.ts +++ b/packages/k8s/src/index.ts @@ -60,6 +60,11 @@ async function run(): Promise { } } +process.on('SIGTERM', () => { + core.error('[GLOBAL] SIGTERM received — process killed by runner (likely hung too long)') + process.exit(143) +}) + process.on('uncaughtException', (err: Error) => { const e = err as NodeJS.ErrnoException core.error(`[GLOBAL] uncaughtException: code=${e.code}, message=${e.message}`) diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index f921c190..7c1e68e3 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -400,26 +400,24 @@ export async function execCpToPod( `find ${shlex.quote(containerPath)} -type d -exec chmod u+rwx {} \\; 2>/dev/null` ] - // Pre-pack diagnostic: log host directory info to help diagnose Linux environment issues. - try { - const { readdirSync } = await import('fs') - const entries = readdirSync(runnerPath, { withFileTypes: true }) - const specialFiles = entries - .filter(e => !e.isFile() && !e.isDirectory() && !e.isSymbolicLink()) - .map(e => { - const type = e.isFIFO() ? 'fifo' - : e.isSocket() ? 'socket' - : e.isBlockDevice() ? 'blockdev' - : e.isCharacterDevice() ? 'chardev' - : 'unknown' - return `${e.name}(${type})` - }) - core.debug( - `[execCpToPod] host dir "${runnerPath}": totalEntries=${entries.length}, specialFiles=${specialFiles.join(',') || 'none'}` - ) - } catch (statErr) { - core.debug(`[execCpToPod] pre-pack readdir failed: ${statErr}`) - } + // Pre-pack diagnostic: recursively find special files (socket/fifo/device) under runnerPath. + // Top-level check is insufficient — blocking files are often in subdirectories. + await new Promise(resolve => { + const findProc = spawn('find', [ + runnerPath, '-not', '-type', 'f', + '-not', '-type', 'd', + '-not', '-type', 'l' + ], { stdio: ['ignore', 'pipe', 'ignore'] }) + let found = '' + findProc.stdout.on('data', (chunk: Buffer) => { found += chunk.toString() }) + findProc.on('close', () => { + core.debug( + `[execCpToPod] recursive special files under "${runnerPath}": ${found.trim() || 'none'}` + ) + resolve() + }) + findProc.on('error', () => resolve()) + }) // Track the last entry being packed so we know which file caused an error. let lastPackedEntry = '' @@ -442,6 +440,11 @@ export async function execCpToPod( reject(new Error(`tar.pack error [${e.code}] on "${e.path ?? lastPackedEntry}": ${e.message}`)) }) + readStream.on('end', () => { + core.debug(`[execCpToPod] tar.pack stream ended, lastEntry="${lastPackedEntry}"`) + }) + + core.debug('[execCpToPod] calling exec.exec() to open WebSocket...') exec .exec( namespace(), @@ -468,6 +471,9 @@ export async function execCpToPod( resolve(status) } ) + .then(() => { + core.debug(`[execCpToPod] WebSocket established, streaming tar to pod...`) + }) .catch(e => { core.debug(`[execCpToPod] exec.exec() rejected: ${e}`) reject(e) From c903f5193eb19836046fa84994763370549801d7 Mon Sep 17 00:00:00 2001 From: tfhddd <2272751277@qq.com> Date: Thu, 5 Mar 2026 11:36:27 +0800 Subject: [PATCH 5/6] =?UTF-8?q?fix:=20=E6=8E=92=E6=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/k8s/src/k8s/index.ts | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index 7c1e68e3..ed6a2f1a 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -430,6 +430,11 @@ export async function execCpToPod( const errStream = new WritableStreamBuffer() await new Promise((resolve, reject) => { + // Guard against double-settle: statusCallback, ws.close, and ws.error may all fire. + let settled = false + const safeResolve = (v: unknown): void => { if (!settled) { settled = true; resolve(v) } } + const safeReject = (e: unknown): void => { if (!settled) { settled = true; reject(e) } } + // Without this listener, an error from tar.pack() (e.g. unreadable file) // would be an unhandled 'error' event and crash the Node.js process. readStream.on('error', (err: Error) => { @@ -437,7 +442,7 @@ export async function execCpToPod( core.debug( `[execCpToPod] tar.pack error: code=${e.code}, path="${e.path}", lastEntry="${lastPackedEntry}", message="${e.message}"` ) - reject(new Error(`tar.pack error [${e.code}] on "${e.path ?? lastPackedEntry}": ${e.message}`)) + safeReject(new Error(`tar.pack error [${e.code}] on "${e.path ?? lastPackedEntry}": ${e.message}`)) }) readStream.on('end', () => { @@ -461,22 +466,38 @@ export async function execCpToPod( `[execCpToPod] exec callback: status=${JSON.stringify(status)}, lastEntry="${lastPackedEntry}", stderr=${stderr || '(empty)'}` ) if (status.status === 'Failure' || errStream.size()) { - reject( + safeReject( new Error( `Error from execCpToPod - status: ${JSON.stringify(status)}, stderr:\n${stderr}` ) ) return } - resolve(status) + safeResolve(status) } ) - .then(() => { + .then(ws => { core.debug(`[execCpToPod] WebSocket established, streaming tar to pod...`) + // @kubernetes/client-node does NOT call statusCallback when the WebSocket + // closes without a status frame (e.g. API-server timeout, abrupt network drop). + // Without these handlers the Promise hangs forever, the event loop drains, + // and Node.js exits silently without writing the response file. + ws.on('close', (code: number, reason: Buffer) => { + core.debug( + `[execCpToPod] WebSocket closed: code=${code}, reason="${reason?.toString()}", callbackFired=${settled}` + ) + safeReject( + new Error(`WebSocket closed before status callback: code=${code}, reason="${reason?.toString()}"`) + ) + }) + ws.on('error', (err: Error) => { + core.debug(`[execCpToPod] WebSocket error: ${err}`) + safeReject(err) + }) }) .catch(e => { core.debug(`[execCpToPod] exec.exec() rejected: ${e}`) - reject(e) + safeReject(e) }) }) break From 3a12e69cd181355eb13a235d1d2b1e79ee8b2b79 Mon Sep 17 00:00:00 2001 From: tfhddd <2272751277@qq.com> Date: Thu, 5 Mar 2026 12:40:47 +0800 Subject: [PATCH 6/6] =?UTF-8?q?fix:=20=E6=8E=92=E6=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/k8s/src/k8s/index.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index ed6a2f1a..a09d926b 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -477,17 +477,27 @@ export async function execCpToPod( } ) .then(ws => { - core.debug(`[execCpToPod] WebSocket established, streaming tar to pod...`) + core.debug( + `[execCpToPod] WebSocket established, protocol="${ws.protocol}", streaming tar to pod...` + ) // @kubernetes/client-node does NOT call statusCallback when the WebSocket - // closes without a status frame (e.g. API-server timeout, abrupt network drop). - // Without these handlers the Promise hangs forever, the event loop drains, - // and Node.js exits silently without writing the response file. + // closes without a status frame. This happens with k8s protocol < v5.channel.k8s.io: + // when stdin ends, handleStandardInput() calls ws.close() directly, which closes + // the entire WebSocket before the server can send the status on channel 3. ws.on('close', (code: number, reason: Buffer) => { + const reasonStr = reason?.toString() || '' core.debug( - `[execCpToPod] WebSocket closed: code=${code}, reason="${reason?.toString()}", callbackFired=${settled}` + `[execCpToPod] WebSocket closed: code=${code}, reason="${reasonStr}", settled=${settled}` ) + if (code === 1000) { + // Normal close — exec command likely finished. statusCallback may not fire + // if the server protocol < v5.channel.k8s.io. Resolve here and let the + // hash-verification loop below confirm the copy actually landed in the pod. + safeResolve(undefined) + return + } safeReject( - new Error(`WebSocket closed before status callback: code=${code}, reason="${reason?.toString()}"`) + new Error(`WebSocket closed unexpectedly: code=${code}, reason="${reasonStr}"`) ) }) ws.on('error', (err: Error) => {