From aa904fe497044560f99a6ecb5fa98158e38a5274 Mon Sep 17 00:00:00 2001 From: ZhangYang Date: Mon, 20 Apr 2026 19:56:51 +0800 Subject: [PATCH] feat: enhance github-action-diagnose skill with MCP integration and Python scripts - Add Python scripts (collect_failed_runs.py, diagnose_job.py, batch_diagnose.py, fetch_run.py, mcp_client.py) - Add README.md with usage guide - Integrate MCP client for LTS cluster log querying - Update SKILL.md to support both bash and Python scripts - Remove deprecated JS scripts (collect-failed-runs.js, diagnose-ci-runs.js) --- .../github-action-diagnose/README.md | 141 ++ .../github-action-diagnose/SKILL.md | 481 ++++++- .../github-action-diagnose/evals/evals.json | 50 +- .../references/ascend-troubleshooting.md | 1 - .../references/classification-guide.md | 76 ++ .../references/common-patterns.md | 271 ++++ .../references/vllm-ascend.md | 152 +++ .../scripts/batch_diagnose.py | 331 +++++ .../scripts/collect_failed_runs.py | 340 +++++ .../scripts/diagnose_job.py | 1214 +++++++++++++++++ .../scripts/fetch-run.sh | 211 +++ .../scripts/fetch_run.py | 423 ++++++ .../scripts/mcp_client.py | 241 ++++ 13 files changed, 3823 insertions(+), 109 deletions(-) create mode 100644 skills/infrastructure/github-action-diagnose/README.md create mode 100644 skills/infrastructure/github-action-diagnose/references/classification-guide.md create mode 100644 skills/infrastructure/github-action-diagnose/references/common-patterns.md create mode 100644 skills/infrastructure/github-action-diagnose/references/vllm-ascend.md create mode 100644 skills/infrastructure/github-action-diagnose/scripts/batch_diagnose.py create mode 100644 skills/infrastructure/github-action-diagnose/scripts/collect_failed_runs.py create mode 100644 skills/infrastructure/github-action-diagnose/scripts/diagnose_job.py create mode 100644 skills/infrastructure/github-action-diagnose/scripts/fetch-run.sh create mode 100644 skills/infrastructure/github-action-diagnose/scripts/fetch_run.py create mode 100644 skills/infrastructure/github-action-diagnose/scripts/mcp_client.py diff --git a/skills/infrastructure/github-action-diagnose/README.md b/skills/infrastructure/github-action-diagnose/README.md new file mode 100644 index 0000000..c3294d8 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/README.md @@ -0,0 +1,141 @@ +# GitHub Action CI 诊断 Skill + +> 专为昇腾（Ascend）NPU 集群（ARC/K8s）基础设施设计的 CI 故障自动诊断工具。 + +## 功能 + +- 自动收集失败 CI 的日志、Annotations、PR 变更等信息 +- 通过 AI（Qwen3.6-Plus）分析根因，分类故障类型 +- 支持查询 LTS 集群日志（MCP），获取 Runner Pod 完整日志 +- 输出结构化诊断报告（定性、根因、责任方、建议） +- 支持单条诊断和批量诊断 + +## 快速开始 + +### 前置要求 + +- Python 3.12+ +- `gh` CLI（已登录并认证） +- 百炼 API Key（用于调用 Qwen 模型） +- GitHub Token（用于调用 GitHub API） + +### 安装依赖 + +```bash +pip install requests openpyxl openai +``` + +### 1. 收集失败的 CI + +```bash +# 收集过去 12 小时，排除 lint 相关 +python scripts/collect_failed_runs.py --hours 12 --exclude-jobs lint --token + +# 收集指定日期范围 +python scripts/collect_failed_runs.py --from 2026-04-20 --to 2026-04-20 --exclude-jobs lint --token +``` + +输出：`Fail_CI_Problem/failed-runs-YYYY-MM-DD.xlsx` + +### 2. 诊断单个 Job + +```bash +python scripts/diagnose_job.py --url "https://github.com/vllm-project/vllm-ascend/actions/runs/xxx/job/yyy" --api-key +``` + +### 3. 批量诊断 + +```bash +python scripts/batch_diagnose.py --input Fail_CI_Problem/failed-runs-2026-04-20.xlsx --api-key + +# 只诊断前 N 条（测试用） +python scripts/batch_diagnose.py --input Fail_CI_Problem/failed-runs-2026-04-20.xlsx --api-key --limit 5 +``` + +结果写回 xlsx 文件的"诊断结果"列，支持断点续传。 + +## 工具列表 + +### GitHub 数据获取（6 个） + +| 工具 | 用途 | +|------|------| +| `fetch_run_script` | 运行 `fetch_run.py` 脚本，一次性获取 Job 信息、Annotations、预过滤日志 | +| `fetch_job_info` | 获取 Job 元数据（名称、Runner、失败步骤） | +| `fetch_job_logs` | 获取 GitHub Actions 日志（按关键词过滤） | +| `fetch_annotations` | 获取 GitHub Annotations（通常直接揭示根因） | +| `fetch_run_info` | 获取 Run 元数据（PR、分支、工作流名称） | +| `fetch_pr_diff` | 获取 PR 变更文件列表 | + +### MCP 集群日志查询（4 个） + +| 工具 | 用途 | +|------|------| +| `mcp_get_runner_logs` | **一键获取 Runner Pod 日志**（自动查找 Pod → 导出 → 下载） | +| `mcp_list_pods` | 列出集群中的 Pod | +| `mcp_export_logs` | 创建日志导出任务 | +| `mcp_get_export` | 查询导出结果 | + +### 通用工具（5 个） + +| 工具 | 用途 | +|------|------| +| `bash_execute` | 执行任意 shell 命令 | +| `read_file` | 读取本地文件 | +| `grep_file` | 搜索文件内容 | +| `glob_file` | 按模式查找文件 | +| `write_file` | 写入文件 | + +## 诊断流程 + +``` +用户输入 Job URL + ↓ +Step 1: fetch_run_script → 获取 GitHub 日志、Annotations、PR 信息 + ↓ +AI 分析：信息够吗？ + ├─ 够 → 直接输出诊断报告 + └─ 不够 → 调用 mcp_get_runner_logs 获取集群日志 + ↓ + AI 分析集群日志 + ↓ + 输出最终诊断报告 +``` + +## 故障分类 + +| 类型 | 说明 | 责任方 | +|------|------|--------| +| A | 基础设施/环境故障（网络超时、NPU 硬件、OOM、K8s 调度） | 基础设施团队 | +| B | 代码 Bug（PR 引入的异常） | PR 作者 | +| C | 精度回归 | PR 作者 | +| D | YAML/配置错误 | PR 作者/CI 维护者 | +| E | 疑难/概率性问题 | 需进一步排查 | + +## MCP 配置 + +MCP 服务地址：`http://150.158.143.223:30089/mcp` +默认日志源：`ascend-ci-log` +默认 namespace：`vllm-project` + +## 文件结构 + +``` +scripts/ +├── collect_failed_runs.py # 收集失败 CI，输出 xlsx +├── diagnose_job.py # 单条诊断脚本（AI agentic loop） +├── batch_diagnose.py # 批量诊断脚本（从 xlsx 读取，结果写回） +├── fetch_run.py # 获取 Job 日志和元数据（Python 版） +├── fetch-run.sh # 获取 Job 日志和元数据（Bash 版） +├── mcp_client.py # MCP 客户端封装 +``` + +## 费用估算 + +| 指标 | 数值 | +|------|------| +| 单条 Token 消耗 | ~1-3 万 | +| 单条费用 | ~¥0.02-0.06 | +| 100 条费用 | ~¥2-6 | + +定价：Qwen3.6-Plus 输入 2 元/百万 Token，输出 12 元/百万 Token。 diff --git a/skills/infrastructure/github-action-diagnose/SKILL.md b/skills/infrastructure/github-action-diagnose/SKILL.md index 5e1fc11..92cd1f6 100644 --- a/skills/infrastructure/github-action-diagnose/SKILL.md +++ b/skills/infrastructure/github-action-diagnose/SKILL.md @@ -1,99 +1,450 @@ --- name: github-action-diagnose -description: 诊断昇腾（Ascend）NPU 集群上 GitHub Actions 执行失败的原因，定位基础设施故障与根因分析 -allowed-tools: Bash(gh run view:*), Bash(gh run list:*), Bash(gh api:*), Bash(kubectl get:*), Bash(kubectl describe:*), Bash(kubectl logs:*), Bash(kubectl config:*), Read, Grep +description: GitHub Action CI 失败自动诊断，专为昇腾（Ascend）NPU 集群（ARC/K8s）基础设施设计。当用户提到 CI 失败、流水线挂掉、CI 报错、构建失败、测试失败、PR CI 不过、CI 排队超时、nightly 失败、runner 异常等情况时立即触发。自动分析日志、定位物理节点、分类根因（基础设施/代码Bug/精度回归）、识别责任方，将诊断报告写入文件。即使用户只是粘贴了一段错误日志或 CI 链接，也应触发此 skill。 +compatibility: + tools: + - Bash + - Read + - Write + - Glob + - Grep --- -## Your task +# CI 故障自动诊断 Skill -诊断昇腾（Ascend）NPU 集群上 GitHub Actions 的执行失败。仅做故障定性和根因分析，不提供修复建议。 -所有只读操作（gh CLI、kubectl 查询、日志读取）直接执行，无需用户确认。 +## 使用场景 -本技能采用 5 步诊断流程：Step 0 输入识别 + 4 步诊断逻辑（生命周期分诊 → 物理节点溯源与环境故障定位 → 非环境问题判定 → 输出报告）。 +- Ascend NPU 集群（ARC/K8s）上的 GitHub Actions 运行失败 +- PR CI 不过、nightly 任务失败、runner 异常等需要快速定位根因的场景 +- 区分故障责任方：基础设施团队（环境/硬件）vs PR 作者（代码/精度） -### Step 0: 识别输入并获取日志 +## 前置要求 -分析用户提供的输入，自动识别类型： +- `gh` CLI 已安装并已完成 GitHub 认证（`gh auth login`） +- 有目标仓库的读权限（用于拉取 run 日志） +- `kubectl` 已配置（用于物理节点溯源，无权限时跳过该步骤） -- **GitHub Actions Run URL**（包含 `/actions/runs/`）：提取 owner/repo 和 run_id，执行 `gh run view --repo --log-failed` 获取失败日志。 -- **Run ID**（纯数字）：需要用户确认 repo 或从当前目录推断，然后执行 `gh run view --log-failed`。 -- **粘贴的日志内容**（非 URL 非纯数字）：直接作为日志内容进行分析。 +## 核心原则 -如果 `--log-failed` 信息不足，自动尝试 `gh run view --log` 获取完整日志。 -同时获取 run 的元信息：`gh run view --repo --json jobs,name,status,conclusion,headBranch,event`。 +- **静默执行**：直接运行所有只读操作（`gh` CLI、日志读取、`kubectl` 查询），不要逐步询问权限 +- **先跑后说**：收集完所有信息后，一次性生成完整报告 +- **直接输出**：诊断结果直接在对话中输出，不写入文件（Action Plan JSON 除外） +- **责任明确**：每个问题必须标注责任方（基础设施团队 / PR 作者） +- **仅在变更操作前确认**：修改代码、提交 commit、执行 `kubectl delete/patch` 才需询问 -### Step 1: 生命周期分诊 +--- + +## Step 0：解析用户输入并确定诊断范围 + +根据用户输入确定诊断范围： + +| 输入类型 | 示例 | 诊断范围 | +|---------|------|---------| +| Run ID | `23282406275` | 诊断 Run 中所有失败的 Job | +| Run URL | `github.com/.../actions/runs/23282406275` | 诊断 Run 中所有失败的 Job | +| Job URL | `github.com/.../actions/runs/.../jobs/xxx` | **只诊断这一个 Job** | + +> **重要**：如果用户提供了具体的 Job URL，**只分析这一个 Job**，不要去拉取整个 Run 的其他 Job 日志。这是为了节省时间和 token。 + +--- + +## Step 1：静默收集上下文 + +### 1a. 确定诊断范围 + +- **如果用户提供了 Job URL**：直接使用该 Job ID，跳过 Run 级别分析 +- **如果用户提供了 Run ID/URL**：分析该 Run 中所有失败的 Job + +### 1b. 收集日志 + +**必须使用脚本获取日志**，不要直接用 `gh run view --job --log`，原因： +1. 日志可能很大（几百MB），导致 API 超时或工具调用被中断 +2. tiling 编译警告会淹没真正的根因错误 +3. 脚本已内置预过滤逻辑，只输出关键错误行 + +**Job URL 场景（只诊断单个 Job）**： +```bash +bash /scripts/fetch-run.sh --job [owner/repo] +# 例如：bash scripts/fetch-run.sh --job 68954806226 vllm-project/vllm-ascend +# 或（无 bash 环境时）：python scripts/fetch_run.py --job 68954806226 vllm-project/vllm-ascend +``` + +**Run ID/URL 场景（诊断所有失败 Job）**： +```bash +bash /scripts/fetch-run.sh [owner/repo] +# 例如：bash scripts/fetch-run.sh 23326177540 +# 或（无 bash 环境时）：python scripts/fetch_run.py --run 23326177540 +# 默认 repo 为 vllm-project/vllm-ascend +``` + +**Run ID/URL 场景（诊断所有失败 Job）**： +```bash +python /scripts/fetch_run.py --run [owner/repo] +# 例如：python scripts/fetch_run.py --run 23326177540 +# 或： bash scripts/fetch-run.sh 23326177540 +# 默认 repo 为 vllm-project/vllm-ascend +``` + +**脚本输出内容**： +- Runner 名称、起止时间、结论 +- 失败步骤列表（哪个 step 挂了） +- Annotations（通常直接揭示根因，如 ETIMEDOUT / exit code） +- Annotations 已明确根因时自动跳过日志拉取 +- 预过滤的运行时错误（RuntimeError/AssertionError/EngineDeadError/OOM 等） +- 预过滤的编译/安装错误（依赖冲突、git dubious ownership 等） +- PR 变更文件列表 + +脚本位置：`/scripts/fetch_run.py`（Python，跨平台）或 `/scripts/fetch-run.sh`（Bash） + +**Step 1c：依赖解析错误优先检查（Install 阶段失败时必做）** + +脚本日志不足以定位根因时，**优先使用 Step 1d 中的依赖专用命令**搜索依赖解析错误，这是最容易被遗漏的根因： + +**为什么重要**：依赖不可用（如 `modelscope==1.35.1` 不存在）会导致后续所有步骤连锁失败，日志中会出现大量 "error"、"failed"，但真正的根因只在依赖解析步骤的开头。 + +### Step 1d. 按需补充的命令（脚本输出不足以定位根因时才用） + +> **脚本已内置以下逻辑，90% 场景无需手动补充命令：** +> - 失败步骤列表（哪个 step 挂了） +> - Annotations（ETIMEDOUT / exit code 等直接结论） +> - Annotations 已明确根因时自动跳过日志拉取 +> - PR 变更文件列表 + +仅在脚本输出仍不足以定位根因时，才手动补充： + +```bash +# 依赖解析错误专用 +gh run view --job --log --repo | grep -iE "unsatisfiable|No solution found|no version of|modelscope.*error" +``` + +### Step 1e. 集群日志查询（MCP，当 GitHub 日志不足时） + +当 GitHub Actions 日志不足以定位根因（如只显示 exit code 1 但无具体错误），可使用 MCP 查询 LTS 集群日志： + +```bash +# 一键获取 Runner Pod 日志 +mcp_get_runner_logs(runner_name="linux-aarch64-a3-0-ggwx6-runner-qbq72", start="2026-03-27 21:20:00", end="2026-03-27 22:00:00") + +# 或分步调用： +mcp_list_pods(namespace="vllm-project", start="...", end="...") +mcp_export_logs(namespace="vllm-project", pod_name="...", keywords="RuntimeError|OOM", start="...", end="...") +mcp_get_export(export_id="...") +``` + +**MCP 服务地址**：`http://150.158.143.223:30089/mcp` +**默认日志源**：`ascend-ci-log` +**默认 namespace**：`vllm-project`（与仓库名对应） + +--- + +## Step 2：生命周期快速分诊 + +根据**失败发生的步骤**做初步判断： + +| 失败步骤 | 初步定性 | +|---------|---------| +| `Set up job` | 基础设施 — Runner 调度失败 | +| `Initialize containers` | 基础设施 — 容器运行时异常 | +| `Checkout` / `Install dependencies` | 基础设施 — 网络/挂载/权限 | +| `Run test` / `Build` | **需同时执行 Step 3 和 Step 4**，不可只做其中之一 | + +> **重要**：失败在 `Run test` / `Build` 阶段时，必须同时检查环境故障（Step 3）和代码问题（Step 4），不因"看起来是代码问题"而跳过环境检查，也不因"多 Job 同时失败"而跳过代码检查。 + +**多个 Job 同步以相同步骤失败 → 强烈的基础设施信号**，但仍需完成代码侧的快速排查。 + +--- + +## Step 3：物理节点溯源（A 类必做，Run step 失败时也应尝试） + +### 3a. 获取 runner_name + +从 `Set up job` 日志或 API 获取，格式通常为： +``` +linux-aarch64-a3-2-x51bm-runner-ksnwc + ↑节点池 ↑物理机编号 ↑pod随机后缀 +``` + +### 3b. 定位 Namespace + +```bash +# 自动发现 runner pod 所在 namespace（不确定时使用） +kubectl get pods --all-namespaces | grep +``` + +### 3c. 查物理节点（区分两种场景） + +**Pod 未销毁**： +```bash +kubectl get pod -n \ + -o custom-columns=NODE:.spec.nodeName +``` + +**Pod 已销毁**：从完整日志中搜索调度记录： +```bash +# 先拉完整日志 +gh run view --job --log --repo > full.log +grep "Successfully assigned to" full.log +# 或在 Loki/ELK 中搜索相同关键词 +``` + +### 3d. 多机任务 — 优先定位 Master 节点（Rank 0） + +当任务涉及多机（multi-node）且某节点报 `Timeout` 时，**不要孤立分析报错节点**，先找 Rank 0： + +```bash +# 方法一：从日志中找 MASTER_ADDR 环境变量 +grep "MASTER_ADDR" full.log + +# 方法二：从 RANK_TABLE_FILE 中找 rank_id=0 对应的 device_ip +grep -A5 '"rank_id": "0"' +``` + +找到 Master 节点后，优先检查其日志： +- 是否有 `Unexpected Exit` +- 是否有 NPU 驱动报错（`ERR99999`、`error code 507035`） +- 是否有进程崩溃（exit `-9`、`Bus error`） + +Master 有问题 → 从节点只是连锁超时，根因在 Master。 + +### 3e. 确认 NPU 健康 + +```bash +npu-smi info # 在对应节点上执行 +kubectl get nodes --kubeconfig=<集群kubeconfig> +``` + +--- + +## Step 4：根因分类 + +### 类型 A：基础设施 / 环境故障 +信号：容器启动失败、网络超时、NPU 硬件报错（`ERR99999`）、OOM（`Bus error` / `Killed` / exit `-9`）、exit code 255（K8s 强制终止）、ModelScope 下载超时、多机 Timeout（见 Step 3d）、`shm_broadcast` 超时 + `EngineDeadError`（Nightly 资源调度不稳定，见 `references/common-patterns.md`） -根据失败发生的步骤进行初步分类： +**责任方**：基础设施团队 -| 失败步骤 | 初步定性 | 说明 | -|----------|---------|------| -| Set up job | 环境问题 | Runner 无法拉起/调度失败 | -| Initialize containers | 环境问题 | 容器运行时异常/镜像拉取失败 | -| Checkout / Setup Environment | 环境问题 | 网络/挂载/权限故障 | -| Run steps（pytest/lint 等） | 需进一步分析 | Runner 已就绪，进入 Step 2-3 细分 | +### 类型 B：代码 Bug（PR 引入） +信号：exit code 1、Python 异常堆栈（`UnboundLocalError` / `AssertionError` / `AttributeError`）、失败与 PR diff 直接对应、重跑仍失败、UT 卡死 -如果失败在前三类步骤，直接标记为环境问题并进入 Step 2。 -如果失败在 Run steps，需要同时执行 Step 2 和 Step 3 来区分环境故障与代码问题。 +**责任方**：PR 作者 -### Step 2: 物理节点溯源 + 环境故障深度定位 +### 类型 C：精度回归 +信号：`Accuracy of ... is X, lower than Y`、精度跌幅 > 5% -#### 2a. 物理节点溯源 -当判定为环境故障或怀疑硬件问题时，定位物理节点： +**责任方**：PR 作者（需与算法团队确认） -1. 从 `Set up job` 日志或 `gh api` 获取 `runner_name`（即 Pod 名称）。 -2. 从 `runner_name` 前缀或 Job Labels 识别节点池（如 `a3-0`）。 -3. 自动发现 namespace：`kubectl get pods --all-namespaces --no-headers | grep ` 提取 namespace。 -4. 反查物理机： - - Pod 未销毁：`kubectl get pod -n -o custom-columns=NODE:.spec.nodeName --no-headers` - - Pod 已销毁：在日志中搜索 `Successfully assigned to `。 +### 类型 D：YAML / 配置错误 +信号：`undefined variable "False"`、workflow 语法报错 -#### 2b. 环境故障深度定位 -检查以下故障模式： +**责任方**：PR 作者 / CI 维护者 -- **驱动/硬件失效**：`npu-smi info` 报错、`ERR99999`、`error code 507035`、`Device not found`。 -- **资源溢出 (OOM)**：系统级 `Killed` 信号、`Bus error`（SHM 不足）、`No space left on device`。 -- **多机连锁超时**：多机任务中某节点报 `Timeout`，优先识别并检查 Master 节点。Master 节点识别方法： - - 检查 `RANK_TABLE_FILE` 中 `rank_id=0` 对应的节点 - - 或在日志中搜索 `master_addr` / `MASTER_ADDR` 环境变量指向的节点 - - 确认 Master 节点是否有 `Unexpected Exit` 或驱动报错 +### 类型 E：疑难 / 概率性问题 +信号：偶发挂死（如 triton ascend 概率挂）、无明确异常堆栈、重跑有时通过 -### Step 3: 非环境问题判定 +**常见错误模式速查**：`references/common-patterns.md` +**vllm-ascend 专项**：`references/vllm-ascend.md` -如果 Step 2 未发现环境/硬件故障，检查以下非环境因素： +--- + +## Step 5：输出报告 + +直接在对话中输出诊断结果，格式如下（每个失败 Job 一节）： + +``` +# CI 故障诊断报告 + +**Run**: [Workflow名称] #[Run ID] +**PR**: [仓库/PR号] ([分支名]) +**时间**: [开始] ~ [结束] + +--- + +## 故障一：[Job 名称] + +- **定性**: [环境问题 / 代码Bug / 精度回归 / 配置错误 / 疑难] +- **根因**: [一句话直接原因，如"K8s 容器运行时在测试阶段强制终止，exit code 255"] +- **关键标识**: `[最关键的一行错误，如 ERR99999 / AssertionError / exit code 255]` +- **责任方**: [基础设施团队 / PR 作者] +- **建议**: [重跑 / 修改 XX 文件 XX 行 / 上报运维 / 检查 Master 节点 XX] +- **节点**: [仅硬件故障时填写：Runner Pod名 → 物理节点名，其他类型省略] + +--- + +[其他 Job 以同样格式继续] +``` + +**输出精简原则**： +- 不粘贴大段原始日志，只引用最关键的一行错误标识 +- 不描述诊断过程（不写"我们检查了A，发现B"），直接给结论 +- 节点池、物理机等环境信息仅在硬件故障时填写，其他类型省略 +- 若同一 Run 中多个 Job 根因相同，可合并为一条说明 +- **不输出步骤分析表格，不输出汇总表格** + +--- + +## Step 6：等待人工确认 + +输出诊断结果后停止，等待确认： +- "确认" / "没问题" → 生成 Action Plan（仅类型 B/C/D） +- 提出修正 → 在对话中重新输出修订后的诊断结果；若修正揭示了漏判或误判的模式，同步更新 `references/` 中对应的参考文件，防止下次重犯 +- 基础设施问题 → 提供操作建议后结束，不执行代码修改 + +> **重要**：不要跳过 Step 6 等待确认，直接进入 Step 7 + +--- + +## Step 7：生成 Action Plan + +用户确认后，生成结构化 Action Plan JSON 文件供后续 AI 修复使用。 + +> **注意**：只有代码 Bug/配置错误/精度回归 需要生成 Action Plan，基础设施问题不需要。 + +### Step 7.1：检查是否需要生成 + +| 问题类型 | 是否生成 Action Plan | 说明 | +|---------|-------------------|------| +| 类型 B（代码Bug） | ✅ 是 | PR 引入的 Bug，需要修改代码 | +| 类型 C（精度回归） | ✅ 是 | 需要调整参数或回滚 | +| 类型 D（YAML/配置错误） | ✅ 是 | 需要修改配置 | +| 类型 A（环境/基础设施） | ❌ 否 | 基础设施问题，不涉及代码修改 | +| 类型 E（疑难） | ❌ 否 | 偶发问题，重跑尝试 | + +### Step 7.2：生成 JSON 文件 + +``` +输出文件：/reports/action-plan-.json +``` + +```json +{ + "run_id": "", + "pr": "#", + "jobs_to_fix": [ + { + "job_id": "", + "job_name": "", + "issue_type": "code_bug|config_error|accuracy_regression", + "root_cause": "<一句话描述>", + "fix_actions": [ + { + "type": "code_change", + "file": "<文件路径>", + "action": "modify|add|remove", + "location": "<文件:行号 或 函数名>", + "original": "<原内容（可选）>", + "replacement": "<新内容>", + "reason": "<为什么这么改>" + } + ], + "verification": { + "command": "<验证命令>", + "expected": "<预期结果>" + } + } + ] +} +``` + +### Step 7.3：验证 Action Plan + +生成后，在对话中输出： +- Action Plan 已写入文件路径 +- 总结每个 fix_action（文件、修改内容） -- **YAML 语法错误**：如 `undefined variable "False"`（应为小写 `false`）。 -- **业务逻辑报错**：`AssertionError`、Python Traceback 指向业务源码、测试用例失败。 -- **依赖问题**：pip/npm 安装失败但非网络原因（版本冲突、包不存在）。 +用户确认后执行实际代码修改。 -### Step 4: 输出诊断报告 +--- + +## 参考资料 + +- **常见错误模式速查**：`references/common-patterns.md`（网络超时/容器崩溃/UT卡死/triton挂/NPU硬件等） +- **vllm-ascend 专用**：`references/vllm-ascend.md`（Runner 类型、内部服务、workflow 触发逻辑） +- **分类判断详细逻辑**：`references/classification-guide.md` + +--- + +## 常见误判与规避 + +### 1. tiling 编译警告 ≠ 测试失败根因 + +**现象**：日志中大量 `Register tiling func failed`、`Get op tiling func failed` +**实际情况**：这是 CANN SDK 在编译 optiling 时的 DEBUG 输出，属于正常行为 +**规避**：grep 时排除 tiling 相关模式，优先捕获 `RuntimeError`、`AssertionError`、`exit code 255` + +### 2. 日志"淹没"问题 -根据诊断结果选择输出模式： +**现象**：高volume日志（如 tiling DEBUG）掩盖低volume但真正的根因 +**规避**：先按错误类型/阶段分离日志输出，高优先级错误（OOM/AssertionError）先展示 -#### 情况 A：确认为基础设施/硬件故障 +### 3. exit code 255 = K8s 终止 -严格按照以下格式输出： +**现象**：测试实际已失败，但日志最后是 `command terminated with exit code 255` +**实际情况**：exit code 255 通常是 K8s 强制终止，说明测试进程已崩溃 +**分析**：往前搜索 `RuntimeError`、`AssertionError` 找到真正失败原因 +--- + +## 使用示例 + +### 示例 1：通过 GitHub Actions Run URL 诊断 + +``` +/github-action-diagnose https://github.com/my-org/my-repo/actions/runs/12345678901 ``` -# GitHub Action 故障诊断报告 -## 1. 故障概览 -- **任务名称**: [Job Name] -- **Run ID**: [Run ID] -- **故障定性**: 环境问题 / 硬件故障 / 集群调度异常 -- **判定依据**: [驱动报错/调度异常/硬件死锁/OOM 等] +Skill 会自动提取 `owner/repo` 和 `run_id`，执行 `gh run view --log-failed`，进入完整诊断流程。 -## 2. 详细诊断 -- **失败步骤**: [Set up job / Initialize containers / Run steps 等] -- **报错原文**: [引用关键日志片段] -- **物理节点**: [Runner 名] -> [Pod] -> [物理机/节点池] -- **根因分析**: [深层原因说明] +### 示例 2：通过 Run ID 诊断（在 repo 目录下） + +``` +/github-action-diagnose 12345678901 ``` -#### 情况 B：环境正常，非基础设施问题 +从当前目录推断 repo，然后执行诊断。 + +### 示例 3：直接粘贴日志内容 + +``` +/github-action-diagnose +2024-01-15T10:23:45.123Z [error] npu-smi info: ERR99999 Device not found +2024-01-15T10:23:45.456Z [error] error code 507035 +``` + +--- + +## 注意事项 + +- 本 skill 仅做故障**定性与根因分析**，不输出修复建议或操作手册 +- 所有只读操作（`gh` CLI、`kubectl` 查询、日志读取）直接执行，无需用户确认 +- 诊断报告中的物理节点信息依赖 `kubectl` 访问权限，无权限时跳过节点溯源步骤 +- 参考资料位于 `references/` 目录，包括常见错误模式、vllm-ascend 专项说明、Ascend NPU 故障排查手册 + +## 相关 Skills + +- [claude-bot](../claude-bot/claude.md) — GitHub Issues/PR 中的 AI 助手，可触发本诊断流程 + +--- + +## 更新日志 + +### v2.0.0 (2026-03-31) +- 重构诊断流程：扩展至 7 步（原 5 步），增加精度回归、代码 Bug、Action Plan 生成阶段 +- 新增责任方标注（基础设施团队 / PR 作者） +- 新增 `references/` 参考资料目录（common-patterns、vllm-ascend、classification-guide、ascend-troubleshooting） +- 输出格式由纯报告改为结构化 Action Plan JSON（代码问题时） +- 新增常见误判规避说明（tiling 警告、日志淹没、exit code 255） + +### v1.0.0 (2026-03-16) +- 初始版本：5 步诊断流程（输入识别 → 生命周期分诊 → 节点溯源 → 非环境判定 → 输出报告） +- 支持 URL / Run ID / 粘贴日志三种输入方式 +- 支持基础设施故障（情况A）和非基础设施问题（情况B）两种输出模式 + +--- + +## 作者 + +- v1.0.0: @hdsong2 +- v2.0.0: @zhangyang -采用自然语言简要反馈： -1. **结论**：直接告知"未发现环境/基础设施故障"。 -2. **依据**：简述分析了哪些环节（调度、镜像拉取、NPU 挂载等均正常）。 +## 最后更新 -注意：两种情况均不输出修复建议或操作手册。 +2026-03-31 diff --git a/skills/infrastructure/github-action-diagnose/evals/evals.json b/skills/infrastructure/github-action-diagnose/evals/evals.json index 6f78850..1a4a35c 100644 --- a/skills/infrastructure/github-action-diagnose/evals/evals.json +++ b/skills/infrastructure/github-action-diagnose/evals/evals.json @@ -1,53 +1,17 @@ { - "skill_name": "github-action-diagnose", + "skill_name": "ci-diagnosis", "evals": [ { "id": 1, - "prompt": "https://github.com/tile-ai/tilelang-ascend/actions/runs/22718535502", - "expected_output": "识别为基础设施故障：镜像拉取失败（image pull failure），输出结构化诊断报告", - "files": [], - "assertions": [ - { - "name": "输出结构化报告（情况A格式）", - "description": "输出包含 '# GitHub Action 故障诊断报告' 标题，说明技能正确识别为基础设施故障并使用了规定格式" - }, - { - "name": "识别为镜像拉取失败", - "description": "输出中包含 '镜像' 或 'image pull' 或 'pull' 相关关键词，说明技能正确定性了故障类型" - }, - { - "name": "包含故障定性字段", - "description": "输出包含 '故障定性' 字段，符合规定的报告结构" - }, - { - "name": "不包含修复建议", - "description": "输出不包含 '建议' 或 '修复' 或 '解决方案' 等字样，符合技能只做诊断不提供修复建议的要求" - } - ] + "prompt": "我们的 PR CI 挂了，Run ID 是 23326177540，repo 是 vllm-project/vllm-ascend，帮我诊断一下哪里出问题了", + "expected_output": "诊断所有失败 Job，写入 reports/diagnosis-23326177540.md，对话输出简短摘要和汇总表格", + "files": [] }, { "id": 2, - "prompt": "https://github.com/sgl-project/sglang/actions/runs/22573924024/job/65388415192", - "expected_output": "识别为基础设施故障：网络问题，输出结构化诊断报告", - "files": [], - "assertions": [ - { - "name": "输出结构化报告（情况A格式）", - "description": "输出包含 '# GitHub Action 故障诊断报告' 标题，说明技能正确识别为基础设施故障并使用了规定格式" - }, - { - "name": "识别为网络问题", - "description": "输出中包含 '网络' 或 'network' 或 'timeout' 或 'connection' 相关关键词，说明技能正确定性了故障类型" - }, - { - "name": "包含故障定性字段", - "description": "输出包含 '故障定性' 字段，符合规定的报告结构" - }, - { - "name": "不包含修复建议", - "description": "输出不包含 '建议' 或 '修复' 或 '解决方案' 等字样，符合技能只做诊断不提供修复建议的要求" - } - ] + "prompt": "这个 Job 一直失败，帮我看下：https://github.com/vllm-project/vllm-ascend/actions/runs/23282406275/job/67875153370", + "expected_output": "只诊断这一个 Job（不拉取整个 Run），写入 reports/diagnosis-job-67875153370.md，对话输出简短摘要和汇总表格", + "files": [] } ] } diff --git a/skills/infrastructure/github-action-diagnose/references/ascend-troubleshooting.md b/skills/infrastructure/github-action-diagnose/references/ascend-troubleshooting.md index 60118bf..ee57dea 100644 --- a/skills/infrastructure/github-action-diagnose/references/ascend-troubleshooting.md +++ b/skills/infrastructure/github-action-diagnose/references/ascend-troubleshooting.md @@ -36,4 +36,3 @@ kubectl logs -n kube-system -l app=ascend-device-plugin Look for: - `get npu device count failed` - `npu device is unhealthy` - diff --git a/skills/infrastructure/github-action-diagnose/references/classification-guide.md b/skills/infrastructure/github-action-diagnose/references/classification-guide.md new file mode 100644 index 0000000..6991964 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/references/classification-guide.md @@ -0,0 +1,76 @@ +# 问题分类判断树 + +分类体系与 SKILL.md 保持一致：**类型 A**（基础设施）/ **类型 B**（代码Bug）/ **类型 C**（精度回归）/ **类型 D**（配置错误）/ **类型 E**（疑难） + +--- + +## 第一步：失败发生在哪个阶段？ + +| 失败步骤 | 初步定性 | +|---------|---------| +| `Set up job` / `Initialize containers` | **类型 A** — 直接归基础设施，无需看日志 | +| `Checkout` / `Install dependencies` | **倾向类型 A**，但需确认是网络还是版本号写错（见场景 B） | +| `Stream logs` / `Upload logs` | **类型 A** — 测试本身可能已通过，是 Runner 与 GitHub 通信失败 | +| `Run test` / `Build` | **需同时排查 A 和 B**，不可只做其中之一 | + +--- + +## 第二步：是类型 A 还是类型 B？（Run test/Build 阶段失败时） + +**倾向类型 A（基础设施）的信号：** +- 多个 PR / 多个 Job 同时失败，且根因相同 +- 重跑后通过 +- 错误指向网络、存储、NPU 硬件（`ERR99999`）、OOM、exit code 255 +- `shm_broadcast` 超时 + `EngineDeadError`（Nightly 调度不稳定） +- 无 Python 异常堆栈，进程被外部终止 + +**倾向类型 B（代码 Bug）的信号：** +- 有明确 Python 异常堆栈，且指向 PR 新增/修改的代码 +- 重跑仍然失败（稳定复现） +- 其他 PR 没有同样的失败 +- PR diff 与失败路径直接对应 + +**两者都有信号时 → 先排查 A，再排查 B**（基础设施问题可以掩盖代码问题，反之不成立） + +--- + +## 第三步：细分类型 + +### 类型 C：精度回归 +- 有明确精度数值报错：`Accuracy of ... is X, lower than Y` +- 跌幅 < 3%：先重跑确认是否 flaky +- 跌幅 > 5%：PR 引入功能性回归，检查推理路径改动 + +### 类型 D：配置/YAML 错误 +- `undefined variable "False"`（YAML 大小写） +- workflow 语法报错（actionlint） +- 路径/权限配置错误 + +### 类型 E：疑难 / 概率性 +- 无明确异常堆栈，进程静默挂死 +- 重跑有时通过、有时挂 +- 常见：triton ascend 概率挂、CPU offload 死锁 + +--- + +## 常见混淆场景 + +**场景 A：`AssertionError` 是代码 Bug 还是精度回归？** +- `AssertionError: Test0:` / `model_utils.py` 断言 → **类型 C**，精度回归 +- `AssertionError` 指向功能逻辑（如形状不匹配、返回值错误）→ **类型 B**，代码 Bug + +**场景 B：依赖失败 — 网络问题还是版本号写错？** +- `No solution found` / `no version of xxx==Y.Y.Y` → **类型 A**，版本不存在于镜像源（或写错） +- `connection refused` / 超时 → **类型 A**，网络/基础设施 + +**场景 C：NPU function error — 硬件还是代码兼容性？** +- `error code 507xxx` / `ERR99999` → **类型 A**，硬件/驱动故障 +- `error code 107xxx`（如 107030）→ CANN runtime 参数非法，**不是硬件信号**；若发生在 `model init` 且 PR 是版本升级/main2main → **类型 B**（新版本 init 路径与 torch_npu 不兼容） + +**场景 D：`EngineDeadError` — 代码 Bug 还是环境？** +- 伴随 `shm_broadcast` 60s 超时连续出现，且为 Nightly 任务 → **类型 A**，资源调度不稳定 +- 启动阶段即崩溃，有明确 Python 堆栈 → **类型 B** + +**场景 E：exit code 255** +- 通常是 K8s 强制终止，本身不是根因 +- 往前找 `RuntimeError`、`AssertionError` 才是真正失败原因 diff --git a/skills/infrastructure/github-action-diagnose/references/common-patterns.md b/skills/infrastructure/github-action-diagnose/references/common-patterns.md new file mode 100644 index 0000000..ba1a8f8 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/references/common-patterns.md @@ -0,0 +1,271 @@ +# 常见 CI 错误模式速查（昇腾 NPU / ARC / K8s 环境） + +## 基础设施类 + +### NPU 硬件错误（⚠️ 注意 error code 范围） +``` +ERR99999 +error code 507035 +npu-smi info: device X not found +``` +- **定性**：类型 A，硬件层故障 +- **分析**：用 `npu-smi info` 确认 NPU 状态；查看是否是特定节点的问题 +- **处理**：标记该节点为 SchedulingDisabled，上报硬件团队 +- **⚠️ 易误判**：`NPU function error: aclrtMemcpy, error code is 107xxx`（如 107030）**不是硬件错误**，是 CANN runtime 参数非法；若发生在 `model init` 阶段且 PR 是版本升级/main2main → 定性为**类型 B**（代码兼容性），参见 classification-guide.md 场景 D + +### 容器启动超时（Initialize containers 阶段） +``` +[error] Executing the custom container implementation failed. +Please contact your self hosted runner administrator. +``` +日志在 `12:50:13` 开始执行 `k8s/index.js` 但 1 小时后无任何进展： +- **定性**：类型 A +- **可能原因**：镜像拉取超时（网络抖动/仓库限流）、NPU 资源争用（节点上 NPU 全被占用）、K8s 容器编排脚本卡死（API Server 响应慢） +- **溯源**：查 Runner Pod 状态，反查物理节点，确认镜像拉取是否有失败事件 + +### 网络代理绕过（Install dependencies 阶段） +``` +Connecting to github.com|20.205.243.166|:443... connected. +Unable to establish SSL connection. +Error: command terminated with exit code 4 +``` +``` +ERROR 618: jwt:expired. +Error: command terminated with exit code 8 +``` +- **定性**：类型 A，基础设施/网络 +- **根因**：安装脚本用 `wget` 直连 `github.com`，未经集群内代理（如 `gh-proxy.test.osinfra.cn`）转发，导致 SSL 失败 / JWT 签名 URL 过期 +- **触发位置**：通常在 `scripts/ci/npu/npu_ci_install_dependency.sh` +- **修复方案 A（推荐）**：将下载 URL 替换为代理地址： + ```bash + GH_PROXY="https://gh-proxy.test.osinfra.cn" + wget "${GH_PROXY}/https://github.com/..." + ``` +- **修复方案 B**：将依赖预缓存到内部 cache-service，彻底脱离 GitHub 实时依赖 + +### OOM / 内存不足 +``` +Bus error (核心已转储) +Killed +exit code -9 +``` +- **定性**：类型 A +- **Bus error**：通常是 SHM（共享内存）不足，多机/多卡训练时常见 +- **Killed / exit code -9**：系统 OOM Killer 终止进程 +- **处理**：检查 Pod 的内存 limit 配置；检查是否有其他 Pod 占用了节点内存 + +### 多机任务连锁超时 +``` +[Rank X] Timeout waiting for ... +``` +- **定性**：类型 A，但需找主节点（Master / Rank 0） +- **分析**：某 Rank 报 Timeout 不一定是该 Rank 的问题——优先检查 **Master 节点**日志，是否有 `Unexpected Exit` 或驱动报错 + +### Nightly 资源调度不稳定（Worker 进程挂死） +``` +No available shared memory broadcast block found in 60 seconds. +EngineDeadError: EngineCore encountered an issue. +``` +- **定性**：类型 A，基础设施 / 资源调度不稳定 +- **背景**：Nightly 多机任务中，Worker 进程被调度抢占或 NPU 资源争用导致卡死，`shm_broadcast` 60s 超时连续触发，最终 EngineCore 死亡。`EngineDeadError` 本身不是根因，是连锁结果 +- **判断依据**：`shm_broadcast` 超时日志连续出现（每 60s 一次）；任务为 Nightly（非 PR CI）；早期推理吞吐日志正常，说明模型加载成功 +- **与代码 Bug 的区分**：代码 Bug 通常在启动阶段或首次推理即失败，且有明确的 Python 异常堆栈；此模式为运行一段时间后才崩溃，且无代码层异常 +- **处理**：直接重跑；若持续复现，检查对应节点的 NPU 调度竞争情况 + +### Runner 调度失败（Set up job 阶段） +``` +Waiting for runner... (timeout) +No runners available +``` +- **定性**：类型 A +- **分析**：查看 ARC Scale Set 是否有 listener Pod 在运行；查看 Pending Pod 数量；检查节点是否有 `SchedulingDisabled` + +--- + +## 代码类错误 + +### UnboundLocalError（典型陷阱） +``` +UnboundLocalError: cannot access local variable 'xxx' where it is not associated with a value +``` +- **定性**：类型 B，代码 Bug +- **典型模式**：`try: import lib except ImportError: pass`，之后直接使用 `lib` 变量，在 except 路径中 `lib` 未被赋值 +- **修复模式**： + ```python + try: + import huggingface_hub + _HF_HUB_AVAILABLE = True + except ImportError: + _HF_HUB_AVAILABLE = False + + # 使用时： + if _HF_HUB_AVAILABLE: + try: + ... + except huggingface_hub.errors.LocalEntryNotFoundError: + ... + ``` + +### 精度回归 +``` +AssertionError: 0.515 not greater than or equal to 0.59 +Accuracy of /root/.cache/.../ModelName is 0.515, is lower than 0.59 +``` +``` +FAILED tests/e2e/singlecard/test_quantization.py::test_qwen3_w8a8_quant - AssertionError: Test0: +tests/e2e/model_utils.py:53: AssertionError +``` +- **定性**：类型 C，精度回归 +- **分析方向**：找 PR 中对推理路径的改动（量化/反量化逻辑、speculative decoding、token 采样、接受逻辑） +- **注意**：`AssertionError: TestN:` 来自 `model_utils.py` 的输出正确性断言，也是精度回归，不是功能性代码 Bug；跌幅 > 5-13% 通常是功能性回归，不是随机波动 + +### YAML 语法错误 +``` +undefined variable "False" +``` +- **定性**：类型 D +- **根因**：YAML 中 `True`/`False` 应为小写 `true`/`false` + +### 测试断言失败 +``` +AssertionError +FAIL: TestXxx +Expected: xxx, Got: yyy +``` +- **分析步骤**： + 1. 判断是测试写错（预期值不合理）还是被测逻辑错 + 2. 对比 PR diff 找到变更点 + 3. 重跑确认是否 flaky + +--- + +### ModelScope 网络不稳定（Run test 阶段） +``` +Downloading model from modelscope... +ConnectionError: ('Connection aborted.', RemoteDisconnected('...')) +urllib3.exceptions.ReadTimeoutError +HTTPSConnectionPool: Read timed out +``` +- **定性**：类型 A，网络/基础设施 +- **背景**：vllm-ascend 测试使用 ModelScope 下载模型（`VLLM_USE_MODELSCOPE=True`），模型缓存在 `/root/.cache/modelscope/`。网络抖动会导致下载中断，表现为 Run test 阶段报网络超时而非代码异常 +- **判断依据**：失败在 Run test 早期（模型加载前），错误为网络类而非 Python 异常；其他 Job / 其他时间重跑可通过 +- **处理**：直接重跑。如频繁出现，检查集群到 ModelScope CDN 的网络质量；考虑将常用模型预缓存到节点本地，避免每次实时下载 + +### UT 卡死 / 超时（主线合入脏代码） +``` +# 无明显报错，Job 运行 2-3 小时后超时取消 +Canceling since a higher priority waiting request ... exists +# 或超时被 K8s 强制终止 +command terminated with exit code 255 +``` +- **定性**：类型 B，代码 Bug（通常是主线合入了有问题的代码） +- **背景**：UT 正常运行时间约 30-60 分钟。如果 UT 运行超过 2 小时仍无进展，通常是某个测试死锁或无限等待，而非基础设施问题 +- **与基础设施超时的区分**：基础设施超时发生在 `Initialize containers` 阶段；UT 卡死发生在 `Run unit test` 阶段，且 Job 已正常启动并运行了较长时间 +- **排查步骤**： + 1. 查看 UT 日志的最后输出，定位卡在哪个测试文件/函数 + 2. 检查最近合入 main 的 commit，是否包含涉及锁、进程通信、分布式初始化的改动 + 3. 本地运行该测试用例（单独运行，加超时）复现 +- **处理**：定位并 revert 引入死锁的 commit，或修复相关代码 + +### CPU Offload 概率性挂死 +``` +# 无异常堆栈，进程在 Run e2e test 阶段静默挂起 +Error executing in Docker Container: 1 +Executing the custom container implementation failed. +``` +- **定性**：类型 E，疑难/概率性问题 +- **背景**：CPU offload 功能在特定配置下存在概率性死锁，进程不报错但不返回，被容器超时机制终止，表现为 exit code 1 +- **与真实测试失败的区分**： + - 概率挂：日志在测试中途截断，无 `FAILED` / `AssertionError` 等 pytest 汇总行 + - 真实失败：有明确的 `FAILED tests/xxx.py::test_xxx` + 异常堆栈 +- **处理**：先重跑 1-2 次确认概率性；能通过则上报 CPU offload 模块追踪死锁路径 + +### 依赖解析失败（Install 阶段） +``` +× No solution found when resolving dependencies: +╰─▶ Because there is no version of modelscope==1.35.1 and you require + modelscope==1.35.1, we can conclude that your requirements are unsatisfiable. +``` +``` +ERROR: Could not find a version that satisfies the requirement xxx +ERROR: No matching distribution found for xxx +``` +- **定性**：类型 A，依赖源配置问题 +- **背景**：依赖版本在 PyPI / 镜像源中不存在，导致依赖解析失败，后续步骤连锁失败 +- **⚠️ 常见误判**：依赖解析失败后会产生大量 "error"、"failed" 级联日志（如 `find: '/usr/lib/python3': No such file`），容易被误判为 Python 环境问题或镜像缺失 +- **关键诊断**：搜索 `unsatisfiable|No solution found|no version of|requirements are unsatisfiable` +- **处理**：检查版本号是否正确、依赖是否已发布到配置的镜像源 + +### vllm-ascend 安装失败（Install 阶段 build error） +``` +Installing build dependencies: finished with status 'error' +error: subprocess-exited-with-error +× installing build dependencies did not run successfully. exit code: 1 +ERROR: Failed to build 'file:///__w/vllm-ascend/vllm-ascend/vllm-empty' +``` +- **定性**：类型 A 或 B，需区分 +- **背景**：`Install vllm-project/vllm-ascend` 或 `Install vllm from source` 阶段，csrc 编译或 build dependency 安装失败 +- **⚠️ 常见误判**：同步出现的 apt `403 Forbidden`（针对 `jammy-backports`/`jammy-security`）是次要现象，不是根因，这两个仓库为可选安全更新源，403 不影响核心依赖安装 +- **区分根因**： + - 检查 PR 是否修改了 csrc/CMakeLists/setup.py → 可能是 PR 引入（类型 B） + - 与最近主线无关、其他 PR 同期也失败 → 可能是环境依赖版本问题（类型 A） +- **处理**：在对应硬件（310p/A3）环境本地复现；检查 build dependency 版本兼容性 + +### CANN tiling 编译警告（⚠️ 不是根因） +``` +[ERROR][xxx] Register tiling func failed, please check the class name. +[ERROR][xxx] Get op tiling func failed, please check the soc version %d +[ERROR][xxx] Do op tiling failed, cannot find soc version. +``` +- **定性**：编译阶段正常输出，**不是测试失败根因** +- **背景**：这些日志来自 CANN SDK 的 tiling 注册过程，属于 DEBUG 级别的编译输出，不代表运行时失败 +- **⚠️ 常见误判**：这些日志数量大（可达数百行），容易被 grep 脚本捕获后误认为是根因 +- **识别方法**： + - 时间戳：在 `Install vllm-ascend` 阶段（编译期）出现 + - 日志级别：DEBUG 或普通 ERROR，无 Traceback + - 真正的根因需要往后找 `RuntimeError`、`AssertionError`、`exit code 255` +- **处理**：忽略 tiling 警告，继续分析测试阶段的运行时错误 + +### Git dubious ownership（Install 阶段） +``` +git introspection failed: fatal: detected dubious ownership in repository at '/__w/xxx/xxx' +``` +- **定性**：类型 A，基础设施 — Git 安全检查阻止构建 +- **背景**：Git 仓库所有者与当前用户不匹配，通常发生在容器内运行 CI Job 时 +- **关键诊断**：搜索 `dubious ownership` +- **处理**：在 workflow 或容器启动脚本中添加 `git config --global --add safe.directory '*'` + +### Triton Ascend 概率性挂死 +``` +# 无异常堆栈，进程静默挂起 +# 或： +Traceback (most recent call last): + ... +RuntimeError: [Triton] ... +# 或 Job 超时，无有意义的错误信息 +``` +- **定性**：类型 E，疑难/概率性问题 +- **背景**：triton_ascend 在特定算子或配置下存在概率性挂死，表现为进程不报错但永远不返回，最终被超时机制终止 +- **判断依据**： + - 重跑有时通过、有时挂 + - 失败点随机（不固定在某个测试） + - 日志截断，无完整的 Python 异常堆栈 + - 涉及 triton_ascend 编译或 kernel 执行阶段 +- **处理**： + 1. 先重跑 1-2 次确认是否概率性（能通过则大概率是此类问题） + 2. 记录卡死时最后执行的算子/测试，上报 triton_ascend 团队 + 3. 短期可跳过该测试或降低并发度规避；长期需 triton_ascend 侧修复 + +--- + +## Runner Pod 命名规范 + +ARC/K8s 环境中 Runner Pod 名称的结构： +``` +linux-aarch64-a3-2-x51bm-runner-ksnwc + ↑架构 ↑节点池 ↑ ↑随机后缀 + a3-2 = 节点池编号 + x51bm = 物理机标识（有时） +``` + +常见节点池：`a3-0`、`a3-2`、`a3-4` 等，对应不同的 ARC Runner Scale Set 和物理机组。 diff --git a/skills/infrastructure/github-action-diagnose/references/vllm-ascend.md b/skills/infrastructure/github-action-diagnose/references/vllm-ascend.md new file mode 100644 index 0000000..1608b8c --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/references/vllm-ascend.md @@ -0,0 +1,152 @@ +# vllm-ascend 仓库专用参考 + +仓库：`vllm-project/vllm-ascend` +这是 vLLM 在华为昇腾（Ascend）NPU 上的适配层，基于 ARC/K8s + GitHub Actions。 + +--- + +## Runner 类型与节点池 + +vllm-ascend 有两种 runner，溯源方式不同： + +### 静态 Runner（固定节点，长期运行） +| Runner 名 | 硬件 | 用途 | +|-----------|------|------| +| `linux-aarch64-a2b3-0` | Atlas 800 A2B3 | 轻量 job（trigger 解析、变更检测） | +| `linux-aarch64-a2b3-1` | Atlas 800 A2B3 | E2E singlecard 测试 | +| `linux-aarch64-a3-2` | Atlas 800 A3 | A3 单机测试 | +| `linux-aarch64-a3-4` | Atlas 800 A3 | A3 4 卡测试 | +| `linux-aarch64-310p-1` | Atlas 300I Pro (310P) | 310P 单卡测试 | +| `linux-aarch64-310p-4` | Atlas 300I Pro (310P) | 310P 4 卡测试 | +| `linux-amd64-cpu-8-hk` | x86 CPU（香港） | lint / pre-commit（无 NPU） | + +静态 runner 出问题：pod 不会销毁，可直接 `kubectl get pod` 查节点。 + +### ARC 动态 Runner（K8s 弹性扩容） +用于 nightly 等高并发场景，pod 名格式： +``` +linux-aarch64-a3-2-x51bm-runner-ksnwc + ↑架构 ↑节点池 ↑随机后缀 +``` +节点池编号（`a3-0`、`a3-2` 等）对应不同物理机组。 +动态 runner 任务完成后 pod 会销毁，需从日志系统（Loki/ELK）查调度记录。 + +--- + +## 内部服务 + +这些是集群内部地址，出现在 workflow 日志中属于正常配置，**如果访问失败则是基础设施问题**： + +| 服务 | 地址 | 用途 | +|------|------|------| +| PyPI 缓存 | `http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple` | 所有 `uv pip install` 走这里 | +| apt 镜像 | `cache-service.nginx-pypi-cache.svc.cluster.local:8081` | apt 源替换 | +| GitHub 代理 | `https://gh-proxy.test.osinfra.cn` | git clone / wget GitHub 资源 | +| Ascend PyPI | `https://mirrors.huaweicloud.com/ascend/repos/pypi` | Ascend 专用包（CANN 等） | + +**关键诊断规则：** +- `Install dependencies` 失败 + 报 `cache-service` 连接超时 → **基础设施/集群内网问题**，不是代码问题 +- `wget github.com` 直连失败 → 脚本未走 `gh-proxy` → **脚本配置问题**（代码/CI维护者） +- `git clone github.com` 失败 → 检查是否配置了 `url insteadOf gh-proxy` + +--- + +## 主要 Workflow 及触发条件 + +| Workflow | 触发 | Runner | 说明 | +|---------|------|--------|------| +| `pr_test_light.yaml` | 每个 PR 到 main/*-dev/releases/v* | a2b3-1 + amd64 | lint + e2e-singlecard-light | +| `pr_test_full.yaml` | PR 打 label 或手动触发 | a2b3-1 + a3-2/4 | 完整测试套件 | +| `schedule_nightly_test_a3.yaml` | 每天 23:45 北京时间 | a3 动态 runner | nightly 全量测试 | +| `_pre_commit.yml` | 被上述 workflow 调用 | `linux-amd64-cpu-8-hk` | lint：actionlint / markdownlint / mypy | + +**PR CI 触发变更检测逻辑（重要）：** +PR CI 中有一个 `changes` job，用 `dorny/paths-filter` 判断哪些测试需要跑： +- 改了 `vllm_ascend/**`、`tests/e2e/**`、`CMakeLists.txt`、`setup.py` 等 → 触发 e2e 测试 +- 只改了 `tests/ut/**` → 只触发 unit test +- 只改了文档等 → 只跑 lint + +因此：**如果测试没有被触发，先检查变更文件是否命中了对应的 path filter**，不要误判为 CI 基础设施问题。 + +--- + +## 测试套件结构 + +测试用 `run_suite.py` 统一调度，套件定义在 `.github/workflows/scripts/config.yaml`： + +| 套件名 | 说明 | +|--------|------| +| `e2e-singlecard-light` | PR CI 默认跑的轻量单卡测试 | +| `e2e-singlecard` | 完整单卡测试 | +| `e2e-2card-light` | 2 卡轻量测试 | +| `e2e-multicard-2-cards` | 2 卡多卡测试 | +| `e2e-multicard-4-cards` | 4 卡多卡测试 | + +测试文件路径：`tests/e2e/singlecard/`、`tests/e2e/multicard/`、`tests/ut/` + +**定位具体失败用例：** +```bash +# 从 gh 日志中找到失败的具体 test 文件和函数名 +gh run view --job --log-failed --repo vllm-project/vllm-ascend | grep -E "FAILED|ERROR|assert" +``` + +--- + +## 环境与依赖特殊说明 + +### 包管理器：uv（不是 pip） +所有依赖安装用 `uv pip install`，报错格式与 pip 略有不同： +``` +# uv 典型报错 +error: Distribution ... not found (no matching distribution) +error: Failed to download ... Connection refused +``` + +### 模型来源：ModelScope（不是 HuggingFace） +所有测试环境设置了： +``` +VLLM_USE_MODELSCOPE: True +HF_HUB_OFFLINE: 1 +``` +模型缓存在 runner 节点的 `/root/.cache/modelscope/`（不是 `~/.cache/huggingface/`）。 +精度回归分析时，模型路径格式为： +``` +/root/.cache/modelscope/hub/xxx/ModelName +``` + +### CANN 环境激活 +Workflow 统一配置了： +```yaml +defaults: + run: + shell: bash -el {0} +``` +这是因为 CANN toolkit 的环境变量需要通过 shell profile 激活。如果某个 step 没有用这个 shell 配置，可能导致 NPU 驱动不可用。 + +--- + +## 常见 PR CI 失败场景速查 + +### Lint 失败（pre-commit job） +- **Runner**：`linux-amd64-cpu-8-hk`（CPU，无 NPU，不需要节点溯源） +- **常见错误**：mypy 类型检查失败、markdownlint 格式问题、actionlint workflow 语法问题 +- **责任方**：PR 作者 +- **修复**：本地运行 `pre-commit run --all-files` 复现 + +### Install dependencies 失败 +- 先判断是哪个依赖源出问题：`cache-service`（基础设施）还是 GitHub 直连（脚本配置） +- `uv pip install` 从 `cache-service` 失败 → 基础设施问题，重跑 +- `wget`/`git clone` 从 `github.com` 直连失败 → 检查 gh-proxy 配置 + +### 测试精度回归 +vllm-ascend 的精度测试会检查模型推理结果是否达到阈值，常见断言： +```python +assert accuracy >= threshold, f"Accuracy of {model_path} is {accuracy}, is lower than {threshold}" +``` +- 跌幅 < 3%：可能是随机性，先确认是否 flaky（重跑） +- 跌幅 > 5%：PR 引入了功能性回归，检查 speculative decoding、采样逻辑、kernel 改动 + +### 多机任务失败（nightly multi-node） +- nightly A3 的多机测试中，Rank N timeout 不代表 Rank N 出问题 +- 排查顺序：Master（Rank 0）→ 其他 Rank +- 检查 Master 的日志：`Unexpected Exit`、驱动报错、`ERR99999` diff --git a/skills/infrastructure/github-action-diagnose/scripts/batch_diagnose.py b/skills/infrastructure/github-action-diagnose/scripts/batch_diagnose.py new file mode 100644 index 0000000..f1c4ed0 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/scripts/batch_diagnose.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +batch_diagnose.py - 批量诊断 xlsx 文件中的 Job URL，结果写回原文件 + +用法: + python batch_diagnose.py --input Fail_CI_Problem/failed-runs-2026-04-17.xlsx + python batch_diagnose.py --input Fail_CI_Problem/failed-runs-2026-04-17.xlsx --api-key sk-xxx + +特性: + - 串行逐条诊断（简单可靠） + - 断点续传（跳过已诊断的行） + - 每完成一条立即保存（防止中断丢失） + - 进度显示（[3/20] 诊断中...） + +依赖: + pip install openpyxl openai + gh CLI 已安装并认证 +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +import openpyxl + +SCRIPT_DIR = Path(__file__).parent +sys.path.insert(0, str(SCRIPT_DIR)) + +# Force UTF-8 for subprocess +os.environ["PYTHONIOENCODING"] = "utf-8" + +from diagnose_job import ( + SKILL_DIR, + TOOLS, + load_skill_content, + parse_job_url, + execute_tool, + call_qwen, +) + +# 导入 MCP 工具 +from diagnose_job import ( + mcp_list_pods_tool, + mcp_export_logs_tool, + mcp_get_export_tool, + mcp_get_runner_logs_tool, +) + +RESULT_COLUMN = "诊断结果" + + +def parse_args(): + parser = argparse.ArgumentParser(description="批量诊断 xlsx 中的 Job URL") + parser.add_argument( + "--input", + required=True, + help="输入的 xlsx 文件路径", + ) + parser.add_argument( + "--api-key", + default=os.environ.get("DASHSCOPE_API_KEY", ""), + help="百炼 API Key", + ) + parser.add_argument( + "--model", + default="qwen3.6-plus", + help="模型名称（默认 qwen3.6-plus）", + ) + parser.add_argument( + "--max-turns", + type=int, + default=15, + help="每条最大工具调用轮数", + ) + parser.add_argument( + "--skill-dir", + default=str(SKILL_DIR), + help="Skill 目录路径", + ) + parser.add_argument( + "--limit", + type=int, + default=0, + help="只诊断前 N 条（0 表示全部）", + ) + return parser.parse_args() + + +def build_system_prompt(skill_dir: str, gh_path: str) -> str: + skill_content = load_skill_content(skill_dir) + + env_info = f"""环境信息： +- 操作系统: Windows (PowerShell 5.1) +- gh CLI 路径: {gh_path or "C:\\Program Files\\GitHub CLI\\gh.exe"} +- 没有 bash、python3、curl 等 Unix 工具 +- 使用 cmd 命令或 PowerShell 命令 +- 使用 findstr 替代 grep +- 使用 type 替代 cat +- 使用 dir 替代 ls""" + + return f"""你是一个 CI 故障诊断专家。请严格按照以下 SKILL 文档的指引，诊断用户提供的失败 Job。 + + +{skill_content} + + +{env_info} + +## 推荐诊断流程 + +**第一步（必须）：调用 fetch_run_script** +- `fetch_run_script(repo, job_id)` 会优先运行 `fetch-run.sh`（bash），失败则回退到 `fetch_run.py` +- 一次性输出： + - Job 基本信息、Runner、失败步骤 + - PR 上下文和变更文件 + - Annotations（通常直接揭示根因） + - 预过滤的错误日志（如需要） +- **这个工具的输出通常已足够完成诊断** + +**第二步（按需）：当 GitHub 日志不足以定位根因时，使用 MCP 查询集群日志** +- 场景：GitHub 日志只显示 exit code 1，但不知道具体原因 +- `mcp_get_runner_logs(runner_name, start, end, keywords)` - 一键获取 Runner Pod 日志 + - runner_name: 从 fetch_run_script 输出中获取 + - start/end: 从 fetch_run_script 输出的时间范围获取，格式 YYYY-MM-DD HH:MM:SS + - **keywords: 必须传，根据错误类型选择过滤词**： + - 超时/网络: `"ETIMEDOUT|timeout|connection refused"` + - 内存: `"OOM|OutOfMemory|Killed"` + - 运行时错误: `"RuntimeError|AssertionError|EngineDeadError"` + - 编译错误: `"error:|failed|fatal"` + - 不确定: `"Error|Failed|exit code|Exception"` + +**重要规则：** +1. **必须先调用 fetch_run_script**，它的输出通常足以完成诊断 +2. **fetch_run_script 输出后，如果信息足够，直接输出诊断报告，不要再调用其他工具** +3. 只有在 GitHub 日志不足以定位根因时，才使用 MCP 查询集群日志 +4. 最终诊断结果直接输出，不要调用多余的工具 +5. 不粘贴大段原始日志，只引用最关键的一行错误标识 +6. 每个问题标注责任方（基础设施团队 / PR 作者） + +Skill 目录: {skill_dir} +当前工作目录: {os.getcwd()}""" + + +def _find_gh() -> str: + try: + where = subprocess.run( + "where gh.exe", + shell=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if where.stdout.strip(): + return where.stdout.strip().split("\n")[0] + except Exception: + pass + return "C:\\Program Files\\GitHub CLI\\gh.exe" + + +def diagnose_single( + job_url: str, + system_prompt: str, + api_key: str, + model: str, + max_turns: int, +) -> tuple[str, dict]: + """诊断单个 Job URL，返回 (诊断报告文本, token用量)""" + repo, job_id = parse_job_url(job_url) + total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": f"请诊断这个失败的 Job: {job_url}\n\n仓库: {repo}\nJob ID: {job_id}", + }, + ] + + for turn in range(1, max_turns + 1): + message = call_qwen(messages, model, api_key, TOOLS) + + usage = message.get("usage", {}) + total_usage["prompt_tokens"] += usage.get("prompt_tokens", 0) + total_usage["completion_tokens"] += usage.get("completion_tokens", 0) + total_usage["total_tokens"] += usage.get("total_tokens", 0) + + if message.get("tool_calls"): + tool_calls = message["tool_calls"] + tool_results = [] + + for tc in tool_calls: + tool_name = tc["function"]["name"] + tool_args = json.loads(tc["function"]["arguments"]) + result = execute_tool(tool_name, tool_args) + + tool_results.append( + { + "role": "tool", + "tool_call_id": tc["id"], + "content": result, + } + ) + + messages.append(message) + messages.extend(tool_results) + + else: + return message.get("content", "无输出"), total_usage + + return f"（达到最大轮数限制 {max_turns}，诊断未完成）", total_usage + + +def main(): + args = parse_args() + + if not args.api_key: + print("错误: 未设置 API Key") + print("请设置环境变量 DASHSCOPE_API_KEY 或使用 --api-key 参数") + sys.exit(1) + + xlsx_path = Path(args.input) + if not xlsx_path.exists(): + print(f"错误: 文件不存在 {xlsx_path}") + sys.exit(1) + + gh_path = _find_gh() + system_prompt = build_system_prompt(args.skill_dir, gh_path) + + wb = openpyxl.load_workbook(str(xlsx_path)) + ws = wb.active + + headers = [cell.value for cell in ws[1]] + if RESULT_COLUMN not in headers: + headers.append(RESULT_COLUMN) + ws.cell(row=1, column=len(headers), value=RESULT_COLUMN) + + url_col_idx = headers.index("Job URL") + 1 + result_col_idx = headers.index(RESULT_COLUMN) + 1 + + total_rows = ws.max_row - 1 + pending_urls = [] + + for row in range(2, ws.max_row + 1): + url = ws.cell(row=row, column=url_col_idx).value + result = ws.cell(row=row, column=result_col_idx).value + if not url: + continue + if result and str(result).strip(): + print(f"[跳过] 第 {row - 1} 行: {url[:60]}... (已有诊断结果)") + else: + pending_urls.append((row, url)) + + if args.limit > 0: + pending_urls = pending_urls[: args.limit] + print(f"（限制前 {args.limit} 条）\n") + + if not pending_urls: + print("\n所有行已诊断完毕，无需处理。") + return + + print(f"\n共 {len(pending_urls)} 条待诊断，总计 {total_rows} 行\n") + + success_count = 0 + fail_count = 0 + grand_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + for i, (row, url) in enumerate(pending_urls, 1): + print(f"{'=' * 50}") + print(f"[{i}/{len(pending_urls)}] 诊断: {url}") + print(f"{'=' * 50}") + + try: + start = time.time() + result, usage = diagnose_single( + job_url=url, + system_prompt=system_prompt, + api_key=args.api_key, + model=args.model, + max_turns=args.max_turns, + ) + elapsed = time.time() - start + + grand_usage["prompt_tokens"] += usage["prompt_tokens"] + grand_usage["completion_tokens"] += usage["completion_tokens"] + grand_usage["total_tokens"] += usage["total_tokens"] + + ws.cell(row=row, column=result_col_idx, value=result) + wb.save(str(xlsx_path)) + + print( + f"\n✅ 完成 ({elapsed:.0f}s) | Token: {usage['total_tokens']:,} (输入 {usage['prompt_tokens']:,} / 输出 {usage['completion_tokens']:,})" + ) + success_count += 1 + + except Exception as e: + error_msg = f"诊断失败: {str(e)}" + ws.cell(row=row, column=result_col_idx, value=error_msg) + wb.save(str(xlsx_path)) + + print(f"\n❌ {error_msg}") + fail_count += 1 + + print() + + # 计算费用 (Qwen3.6-Plus 中国内地: 输入 2元/百万, 输出 12元/百万) + input_cost = grand_usage["prompt_tokens"] / 1_000_000 * 2 + output_cost = grand_usage["completion_tokens"] / 1_000_000 * 12 + total_cost = input_cost + output_cost + + print(f"\n{'=' * 50}") + print(f"批量诊断完成") + print(f" 成功: {success_count}") + print(f" 失败: {fail_count}") + print(f" Token 用量: {grand_usage['total_tokens']:,}") + print(f" 输入: {grand_usage['prompt_tokens']:,}") + print(f" 输出: {grand_usage['completion_tokens']:,}") + print( + f" 预估费用: ¥{total_cost:.4f} (输入 ¥{input_cost:.4f} + 输出 ¥{output_cost:.4f})" + ) + print(f" 输出: {xlsx_path}") + print(f"{'=' * 50}") + + +if __name__ == "__main__": + main() diff --git a/skills/infrastructure/github-action-diagnose/scripts/collect_failed_runs.py b/skills/infrastructure/github-action-diagnose/scripts/collect_failed_runs.py new file mode 100644 index 0000000..39ac7f8 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/scripts/collect_failed_runs.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +""" +collect_failed_runs.py - 收集 GitHub 仓库一段时间内失败的 CI，支持按 Job 名称过滤 + +用法: + python scripts/collect_failed_runs.py --hours 24 --exclude-jobs lint + python scripts/collect_failed_runs.py --from 2024-01-01 --to 2024-01-31 + python scripts/collect_failed_runs.py --hours 24 --exclude-jobs lint check-style --output custom.xlsx + +依赖: + pip install openpyxl requests +""" + +import argparse +import os +import sys +import re +from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode + +import requests +from openpyxl import Workbook +from openpyxl.styles import Font + + +def parse_args(): + parser = argparse.ArgumentParser(description="收集 GitHub 仓库一段时间内失败的 CI") + parser.add_argument( + "--repo", + default="vllm-project/vllm-ascend", + help="GitHub 仓库，格式 owner/repo", + ) + parser.add_argument("--from", dest="from_date", help="起始日期 YYYY-MM-DD") + parser.add_argument("--to", dest="to_date", help="结束日期 YYYY-MM-DD") + parser.add_argument("--hours", type=int, help="过去多少小时") + parser.add_argument( + "--exclude-jobs", + nargs="*", + default=[], + help="排除的 Job 名称（支持正则）", + ) + parser.add_argument("--output", help="输出文件路径") + parser.add_argument( + "--token", + default=os.environ.get("GITHUB_TOKEN", ""), + help="GitHub Token", + ) + parser.add_argument( + "--per-page", + type=int, + default=100, + help="每页数量", + ) + return parser.parse_args() + + +def get_time_range(args): + now = datetime.now(timezone.utc) + + if args.from_date and args.to_date: + since = datetime.strptime( + args.from_date + "T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc) + until = datetime.strptime( + args.to_date + "T23:59:59Z", "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc) + elif args.from_date: + since = datetime.strptime( + args.from_date + "T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc) + until = now + elif args.hours: + since = now - timedelta(hours=args.hours) + until = now + else: + since = now - timedelta(hours=2) + until = now + + return since.isoformat().replace("+00:00", "Z"), until.isoformat().replace( + "+00:00", "Z" + ) + + +def github_get(url_path, token, params=None): + url = f"https://api.github.com{url_path}" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "collect-failed-runs-script", + } + if token: + headers["Authorization"] = f"Bearer {token}" + + resp = requests.get(url, headers=headers, params=params) + resp.raise_for_status() + return resp.json() + + +def fetch_failed_runs(repo, since, until, token, per_page): + all_runs = [] + page = 1 + + while True: + params = { + "status": "failure", + "per_page": per_page, + "page": page, + "created": f"{since}..{until}", + } + + url_path = f"/repos/{repo}/actions/runs" + data = github_get(url_path, token, params) + runs = data.get("workflow_runs", []) + + if not runs: + break + + all_runs.extend(runs) + + if len(runs) < per_page: + break + page += 1 + + return all_runs + + +def fetch_run_jobs(repo, run_id, token): + all_jobs = [] + page = 1 + + while True: + params = { + "per_page": 100, + "page": page, + } + + url_path = f"/repos/{repo}/actions/runs/{run_id}/jobs" + data = github_get(url_path, token, params) + jobs = data.get("jobs", []) + + if not jobs: + break + + all_jobs.extend(jobs) + + if len(jobs) < 100: + break + page += 1 + + return all_jobs + + +def should_exclude_run(jobs, exclude_patterns): + if not exclude_patterns: + return False + + failed_jobs = [j for j in jobs if j.get("conclusion") == "failure"] + if not failed_jobs: + return True + + for job in failed_jobs: + job_name = job.get("name", "") + is_excluded = any( + re.search(pattern, job_name, re.IGNORECASE) for pattern in exclude_patterns + ) + if not is_excluded: + return False + + return True + + +def get_excluded_job_names(jobs, exclude_patterns): + excluded = [] + for job in jobs: + if job.get("conclusion") != "failure": + continue + job_name = job.get("name", "") + is_excluded = any( + re.search(pattern, job_name, re.IGNORECASE) for pattern in exclude_patterns + ) + if is_excluded: + excluded.append(job_name) + return excluded + + +def calc_duration(iso_str1, iso_str2): + try: + d1 = datetime.fromisoformat(iso_str1.replace("Z", "+00:00")) + d2 = datetime.fromisoformat(iso_str2.replace("Z", "+00:00")) + diff_seconds = int((d2 - d1).total_seconds()) + hours = diff_seconds // 3600 + minutes = (diff_seconds % 3600) // 60 + seconds = diff_seconds % 60 + + parts = [] + if hours > 0: + parts.append(f"{hours}h") + if minutes > 0: + parts.append(f"{minutes}m") + parts.append(f"{seconds}s") + return " ".join(parts) + except Exception: + return "N/A" + + +def format_time(iso_str): + try: + dt = datetime.fromisoformat(iso_str.replace("Z", "+00:00")) + return dt.strftime("%Y-%m-%d %H:%M:%S") + except Exception: + return iso_str + + +def write_xlsx(records, output_path): + wb = Workbook() + ws = wb.active + ws.title = "失败记录" + + headers = [ + "Job URL", + "Run ID", + "创建时间", + "失败时间", + "运行时间", + "工作流名称", + "分支", + "用户", + ] + ws.append(headers) + + for r in records: + for job in r["failed_jobs"]: + ws.append( + [ + job["url"], + r["run_id"], + r["created_at"], + r["updated_at"], + r["duration"], + r["workflow_name"], + r["branch"], + r["user"], + ] + ) + + col_widths = [80, 18, 22, 22, 15, 30, 25, 20] + for i, width in enumerate(col_widths, 1): + ws.column_dimensions[ + chr(64 + i) if i <= 26 else "A" + chr(64 + i - 26) + ].width = width + + wb.save(output_path) + + +def main(): + args = parse_args() + + if not args.token: + print("警告: 未设置 GitHub Token，可能触发 API 限流。") + print("请设置环境变量 GITHUB_TOKEN 或使用 --token 参数。") + + since, until = get_time_range(args) + print(f"仓库: {args.repo}") + print(f"时间范围: {since} ~ {until}") + if args.exclude_jobs: + print(f"排除 Job: {', '.join(args.exclude_jobs)}") + print() + + print("获取失败的 Runs...") + runs = fetch_failed_runs(args.repo, since, until, args.token, args.per_page) + print(f"共获取到 {len(runs)} 个失败的 Run") + + records = [] + skipped_count = 0 + skipped_reasons = [] + + for i, run in enumerate(runs, 1): + run_id = run["id"] + print(f" 处理 [{i}/{len(runs)}] Run #{run_id}... ", end="", flush=True) + + jobs = fetch_run_jobs(args.repo, run_id, args.token) + + if should_exclude_run(jobs, args.exclude_jobs): + excluded_names = get_excluded_job_names(jobs, args.exclude_jobs) + skipped_count += 1 + skipped_reasons.append( + f"Run #{run_id} 被排除 (失败 Job: {', '.join(excluded_names)})" + ) + print("跳过 (被过滤)") + continue + + user = run.get("triggering_actor", {}).get("login", "N/A") + failed_jobs = [ + { + "url": f"https://github.com/{args.repo}/actions/runs/{run_id}/job/{j['id']}", + } + for j in jobs + if j.get("conclusion") == "failure" + ] + + records.append( + { + "url": f"https://github.com/{args.repo}/actions/runs/{run_id}", + "run_id": run_id, + "created_at": format_time(run["created_at"]), + "updated_at": format_time(run["updated_at"]), + "duration": calc_duration(run["created_at"], run["updated_at"]), + "workflow_name": run.get("name", "N/A"), + "branch": run.get("head_branch", "N/A"), + "user": user, + "failed_jobs": failed_jobs, + } + ) + print("保留") + + print() + print(f"保留 {len(records)} 条记录，跳过 {skipped_count} 条") + + if skipped_reasons: + print("\n跳过的 Run:") + for reason in skipped_reasons: + print(f" - {reason}") + + if not records: + print("\n没有符合条件的记录，不生成文件。") + return + + output_path = args.output + if not output_path: + output_dir = "Fail_CI_Problem" + os.makedirs(output_dir, exist_ok=True) + today = datetime.now().strftime("%Y-%m-%d") + output_path = os.path.join(output_dir, f"failed-runs-{today}.xlsx") + + write_xlsx(records, output_path) + print(f"\n已写入: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/skills/infrastructure/github-action-diagnose/scripts/diagnose_job.py b/skills/infrastructure/github-action-diagnose/scripts/diagnose_job.py new file mode 100644 index 0000000..295d069 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/scripts/diagnose_job.py @@ -0,0 +1,1214 @@ +#!/usr/bin/env python3 +""" +diagnose_job.py - 通过百炼 API 调用 Qwen3.6-Plus，使用 SKILL 引擎诊断失败的 CI Job + +用法: + python diagnose_job.py --url "https://github.com/owner/repo/actions/runs/xxx/job/yyy" + python diagnose_job.py --url "https://github.com/owner/repo/actions/runs/xxx/job/yyy" --skill-dir /path/to/skill + +依赖: + pip install dashscope openai + gh CLI 已安装并认证 +""" + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import io +from pathlib import Path +from typing import Optional + +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") +sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8") + + +SKILL_DIR = Path(__file__).parent.parent # scripts 的父目录即 skill 根目录 +SKILL_FILES = { + "SKILL.md": "SKILL.md", + "references/common-patterns.md": "references/common-patterns.md", + "references/vllm-ascend.md": "references/vllm-ascend.md", + "references/classification-guide.md": "references/classification-guide.md", + "references/ascend-troubleshooting.md": "references/ascend-troubleshooting.md", +} + +BASH_EXECUTE = { + "type": "function", + "function": { + "name": "bash_execute", + "description": "Execute a bash command and return its stdout and stderr. Use this to run scripts, fetch logs, query kubectl, etc.", + "parameters": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute", + } + }, + "required": ["command"], + }, + }, +} + +READ_FILE = { + "type": "function", + "function": { + "name": "read_file", + "description": "Read the content of a file. Use this to read reference files, scripts, or templates.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute or relative path to the file", + } + }, + "required": ["path"], + }, + }, +} + +GREP_FILE = { + "type": "function", + "function": { + "name": "grep_file", + "description": "Search for a pattern in file contents. Returns matching lines with file paths and line numbers.", + "parameters": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Regex pattern to search for", + }, + "path": { + "type": "string", + "description": "Directory or file to search in (default: current directory)", + }, + "include": { + "type": "string", + "description": "File pattern to include (e.g. '*.md', '*.sh')", + }, + }, + "required": ["pattern"], + }, + }, +} + +GLOB_FILE = { + "type": "function", + "function": { + "name": "glob_file", + "description": "Find files matching a glob pattern. Returns list of matching file paths.", + "parameters": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Glob pattern (e.g. '*.md', 'scripts/*.sh')", + }, + "path": { + "type": "string", + "description": "Directory to search in (default: current directory)", + }, + }, + "required": ["pattern"], + }, + }, +} + +WRITE_FILE = { + "type": "function", + "function": { + "name": "write_file", + "description": "Write content to a file. Use this to create reports, action plans, or temporary files.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Path to the file to write", + }, + "content": { + "type": "string", + "description": "Content to write to the file", + }, + }, + "required": ["path", "content"], + }, + }, +} + +FETCH_RUN_SCRIPT = { + "type": "function", + "function": { + "name": "fetch_run_script", + "description": "Run the fetch_run.py script to collect job metadata, annotations, and filtered logs. This is the PRIMARY tool for gathering CI failure data. It outputs: job info, runner name, failed steps, annotations, and pre-filtered error lines. ALWAYS use this FIRST before any other data collection.", + "parameters": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "Owner/repo, e.g. vllm-project/vllm-ascend", + }, + "job_id": {"type": "string", "description": "GitHub Job ID"}, + }, + "required": ["repo", "job_id"], + }, + }, +} + +FETCH_JOB_INFO = { + "type": "function", + "function": { + "name": "fetch_job_info", + "description": "Fetch job metadata including name, conclusion, runner_name, started_at, completed_at, run_id, and steps. Use this FIRST to understand the job before fetching logs.", + "parameters": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "Owner/repo, e.g. vllm-project/vllm-ascend", + }, + "job_id": {"type": "string", "description": "GitHub Job ID"}, + }, + "required": ["repo", "job_id"], + }, + }, +} + +FETCH_JOB_LOGS = { + "type": "function", + "function": { + "name": "fetch_job_logs", + "description": "Fetch filtered job logs. Returns only key error lines (RuntimeError, AssertionError, exit code, ETIMEDOUT, OOM, etc.) to avoid overwhelming output. Use this instead of raw log fetch.", + "parameters": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "Owner/repo, e.g. vllm-project/vllm-ascend", + }, + "job_id": {"type": "string", "description": "GitHub Job ID"}, + "filter_type": { + "type": "string", + "enum": [ + "runtime_errors", + "compile_errors", + "upload_errors", + "network_errors", + "all_errors", + ], + "description": "Type of errors to filter for", + }, + }, + "required": ["repo", "job_id"], + }, + }, +} + +FETCH_ANNOTATIONS = { + "type": "function", + "function": { + "name": "fetch_annotations", + "description": "Fetch GitHub check-run annotations for a job. Often directly reveals root cause (ETIMEDOUT, exit code, etc.). Check this BEFORE fetching logs.", + "parameters": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "Owner/repo, e.g. vllm-project/vllm-ascend", + }, + "job_id": {"type": "string", "description": "GitHub Job ID"}, + }, + "required": ["repo", "job_id"], + }, + }, +} + +FETCH_RUN_INFO = { + "type": "function", + "function": { + "name": "fetch_run_info", + "description": "Fetch workflow run metadata including PR number, branch, display_title, and all job statuses. Use to understand run context and PR changes.", + "parameters": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "Owner/repo, e.g. vllm-project/vllm-ascend", + }, + "run_id": {"type": "string", "description": "GitHub Run ID"}, + }, + "required": ["repo", "run_id"], + }, + }, +} + +FETCH_PR_DIFF = { + "type": "function", + "function": { + "name": "fetch_pr_diff", + "description": "Fetch list of changed files in a PR. Use for classification (code bug vs infrastructure).", + "parameters": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "Owner/repo, e.g. vllm-project/vllm-ascend", + }, + "pr_number": {"type": "string", "description": "PR number"}, + }, + "required": ["repo", "pr_number"], + }, + }, +} + +MCP_LIST_PODS = { + "type": "function", + "function": { + "name": "mcp_list_pods", + "description": "通过 LTS 日志服务查询集群中的 Pod 列表。用于定位 Runner Pod 或查找相关 Pod 名称。需要 source、namespace 和时间范围。", + "parameters": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "description": "Kubernetes namespace，如 vllm-project", + }, + "start": { + "type": "string", + "description": "开始时间，格式 YYYY-MM-DD HH:MM:SS", + }, + "end": { + "type": "string", + "description": "结束时间，格式 YYYY-MM-DD HH:MM:SS", + }, + "source": { + "type": "string", + "description": "LTS 日志源（默认 ascend-ci-log）", + }, + }, + "required": ["namespace", "start", "end"], + }, + }, +} + +MCP_EXPORT_LOGS = { + "type": "function", + "function": { + "name": "mcp_export_logs", + "description": "创建日志导出任务，按 Pod 名称和关键词过滤。返回 export_id 用于后续查询结果。", + "parameters": { + "type": "object", + "properties": { + "namespace": {"type": "string", "description": "Kubernetes namespace"}, + "pod_name": { + "type": "string", + "description": "Pod 名称（支持模糊匹配）", + }, + "keywords": { + "type": "string", + "description": "搜索关键词，如 RuntimeError|ETIMEDOUT|OOM", + }, + "start": { + "type": "string", + "description": "开始时间，格式 YYYY-MM-DD HH:MM:SS", + }, + "end": { + "type": "string", + "description": "结束时间，格式 YYYY-MM-DD HH:MM:SS", + }, + "source": { + "type": "string", + "description": "LTS 日志源（默认 ascend-ci-log）", + }, + }, + "required": ["namespace", "pod_name", "start", "end"], + }, + }, +} + +MCP_GET_EXPORT = { + "type": "function", + "function": { + "name": "mcp_get_export", + "description": "查询日志导出任务的结果。传入 export_id 获取日志内容或下载链接。", + "parameters": { + "type": "object", + "properties": { + "export_id": {"type": "string", "description": "导出任务 ID"}, + }, + "required": ["export_id"], + }, + }, +} + +MCP_GET_RUNNER_LOGS = { + "type": "function", + "function": { + "name": "mcp_get_runner_logs", + "description": "一键获取 Runner Pod 的日志。自动完成查找 Pod → 导出 → 下载全流程。只需提供 Runner 名称和时间范围。", + "parameters": { + "type": "object", + "properties": { + "runner_name": { + "type": "string", + "description": "Runner 名称，如 linux-aarch64-a3-0-ggwx6-runner-qbq72", + }, + "namespace": { + "type": "string", + "description": "Kubernetes namespace（默认 vllm-project）", + }, + "start": { + "type": "string", + "description": "开始时间，格式 YYYY-MM-DD HH:MM:SS", + }, + "end": { + "type": "string", + "description": "结束时间，格式 YYYY-MM-DD HH:MM:SS", + }, + "keywords": {"type": "string", "description": "搜索关键词（可选）"}, + }, + "required": ["runner_name", "start", "end"], + }, + }, +} + +TOOLS = [ + FETCH_RUN_SCRIPT, + MCP_GET_RUNNER_LOGS, + MCP_LIST_PODS, + MCP_EXPORT_LOGS, + MCP_GET_EXPORT, + FETCH_JOB_INFO, + FETCH_JOB_LOGS, + FETCH_ANNOTATIONS, + FETCH_RUN_INFO, + FETCH_PR_DIFF, + BASH_EXECUTE, + READ_FILE, + GREP_FILE, + GLOB_FILE, + WRITE_FILE, +] + + +def parse_args(): + parser = argparse.ArgumentParser(description="诊断失败的 CI Job") + parser.add_argument( + "--url", + required=True, + help="GitHub Job URL，如 https://github.com/owner/repo/actions/runs/xxx/job/yyy", + ) + parser.add_argument( + "--skill-dir", + default=str(SKILL_DIR), + help="Skill 目录路径（包含 SKILL.md 和 references/）", + ) + parser.add_argument( + "--api-key", + default=os.environ.get("DASHSCOPE_API_KEY", ""), + help="百炼 API Key", + ) + parser.add_argument( + "--model", + default="qwen3.6-plus", + help="模型名称（默认 qwen3.6-plus）", + ) + parser.add_argument( + "--max-turns", + type=int, + default=20, + help="最大工具调用轮数（防止无限循环）", + ) + return parser.parse_args() + + +def parse_job_url(url: str) -> tuple[str, str]: + """从 Job URL 提取 owner/repo 和 job_id""" + match = re.search(r"github\.com/([^/]+/[^/]+)/actions/runs/\d+/job/(\d+)", url) + if not match: + # 尝试 Run URL 格式 + match_run = re.search(r"github\.com/([^/]+/[^/]+)/actions/runs/(\d+)", url) + if match_run: + return match_run.group(1), match_run.group(2) + raise ValueError(f"无法解析 URL: {url}") + return match.group(1), match.group(2) + + +def load_skill_content(skill_dir: str) -> str: + """加载 SKILL.md + references/*.md 作为 system prompt""" + parts = [] + skill_path = Path(skill_dir) + + for name, rel_path in SKILL_FILES.items(): + file_path = skill_path / rel_path + if file_path.exists(): + content = file_path.read_text(encoding="utf-8") + parts.append(f"## {name}\n\n{content}") + else: + print(f"警告: 文件不存在 {name} ({file_path})", file=sys.stderr) + + return "\n\n".join(parts) + + +def bash_execute(command: str) -> str: + """执行 bash 命令""" + try: + result = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + timeout=120, + encoding="utf-8", + errors="replace", + ) + output = result.stdout + if result.stderr: + output += result.stderr + return output if output else "(无输出)" + except subprocess.TimeoutExpired: + return "命令执行超时（120秒）" + except Exception as e: + return f"执行错误: {str(e)}" + + +def read_file_tool(path: str) -> str: + """读取文件内容""" + try: + p = Path(path) + if not p.exists(): + return f"文件不存在: {path}" + content = p.read_text(encoding="utf-8", errors="replace") + if len(content) > 50000: + return ( + content[:50000] + + f"\n... (内容过长，已截断，总长度 {len(content)} 字符)" + ) + return content + except Exception as e: + return f"读取错误: {str(e)}" + + +def grep_file_tool(pattern: str, path: str = ".", include: str = None) -> str: + """搜索文件内容""" + try: + cmd = ["rg", "--no-heading", "--line-number", "--color=never"] + if include: + cmd.extend(["-g", include]) + cmd.extend([pattern, path]) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + output = result.stdout + if not output and result.returncode != 0: + # 回退到 Python 实现 + import glob as glob_mod + + files = list(glob_mod.glob(f"{path}/**/*", recursive=True)) + if include: + files = [f for f in files if Path(f).match(include)] + matches = [] + for f in files: + if Path(f).is_file(): + try: + content = Path(f).read_text(encoding="utf-8", errors="ignore") + for i, line in enumerate(content.splitlines(), 1): + if re.search(pattern, line): + matches.append(f"{f}:{i}:{line}") + except Exception: + pass + output = "\n".join(matches) + return output if output else f"未找到匹配: {pattern}" + except Exception as e: + return f"搜索错误: {str(e)}" + + +def glob_file_tool(pattern: str, path: str = ".") -> str: + """查找匹配 glob 的文件""" + try: + import glob as glob_mod + + files = glob_mod.glob(f"{path}/{pattern}", recursive=True) + return "\n".join(files) if files else f"未找到匹配文件: {pattern}" + except Exception as e: + return f"搜索错误: {str(e)}" + + +def write_file_tool(path: str, content: str) -> str: + """写入文件""" + try: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + return f"已写入: {path} ({len(content)} 字符)" + except Exception as e: + return f"写入错误: {str(e)}" + + +def _run_gh(args: list, repo: str = "") -> str: + """执行 gh 命令，自动查找路径""" + gh_candidates = [ + os.environ.get("GH_PATH", ""), + "C:\\Program Files\\GitHub CLI\\gh.exe", + ] + try: + where = subprocess.run( + "where gh.exe", + shell=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if where.stdout.strip(): + gh_candidates.insert(0, where.stdout.strip().split("\n")[0]) + except Exception: + pass + + for gh in gh_candidates: + if gh and Path(gh).exists(): + cmd = [gh] + args + if repo: + cmd.extend(["--repo", repo]) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=60, + encoding="utf-8", + errors="replace", + ) + return result.stdout + result.stderr + except Exception as e: + return f"gh 执行错误: {e}" + + return "gh CLI 未找到" + + +def fetch_job_info_tool(repo: str, job_id: str) -> str: + """获取 Job 元数据""" + output = _run_gh(["api", f"repos/{repo}/actions/jobs/{job_id}"]) + try: + data = json.loads(output) + steps_info = "" + for s in data.get("steps", []): + if s.get("conclusion") not in ("success", "skipped", None): + steps_info += ( + f" [{s['conclusion'].upper():8}] Step {s['number']}: {s['name']}\n" + ) + + return ( + f"名称: {data.get('name', 'N/A')}\n" + f"结论: {data.get('conclusion', 'N/A')}\n" + f"Runner: {data.get('runner_name', 'unknown')}\n" + f"时间: {data.get('started_at', '?')} ~ {data.get('completed_at', '?')}\n" + f"Run ID: {data.get('run_id', 'N/A')}\n" + f"失败步骤:\n{steps_info or ' (无)'}" + ) + except json.JSONDecodeError: + return output[:2000] + + +def fetch_job_logs_tool( + repo: str, job_id: str, filter_type: str = "runtime_errors" +) -> str: + """获取过滤后的 Job 日志""" + filters = { + "runtime_errors": "RuntimeError OutOfMemoryError AssertionError EngineDeadError exit code 255 synchronized memcpy", + "compile_errors": "error: failed fatal error unsatisfiable dubious", + "upload_errors": "upload artifact CreateArtifact ETIMEDOUT", + "network_errors": "ETIMEDOUT timeout refused ConnectionResetError", + "all_errors": "RuntimeError AssertionError EngineDeadError exit code ETIMEDOUT OutOfMemoryError fatal ERROR", + } + pattern = filters.get(filter_type, filters["runtime_errors"]) + + gh_path = _find_gh() + log_file = f"_diag_job_{job_id}.log" + + # 先下载完整日志 + subprocess.run( + f'"{gh_path}" run view --job {job_id} --log --repo {repo} > "{log_file}" 2>nul', + shell=True, + capture_output=True, + timeout=120, + ) + + # 用 findstr 过滤 + cmd = f'findstr /i "{pattern}" "{log_file}"' + result = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + timeout=30, + encoding="utf-8", + errors="replace", + ) + output = result.stdout + + # 清理临时文件 + try: + Path(log_file).unlink() + except Exception: + pass + + if not output: + return f"（未找到 {filter_type} 类型的错误日志）" + + lines = output.strip().split("\n") + if len(lines) > 50: + return "\n".join(lines[:50]) + f"\n... (共 {len(lines)} 行，已截断)" + return output + + +def _find_gh() -> str: + """查找 gh.exe 路径""" + try: + where = subprocess.run( + "where gh.exe", + shell=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if where.stdout.strip(): + return where.stdout.strip().split("\n")[0] + except Exception: + pass + return "C:\\Program Files\\GitHub CLI\\gh.exe" + + +def fetch_annotations_tool(repo: str, job_id: str) -> str: + """获取 Annotations""" + output = _run_gh(["api", f"repos/{repo}/check-runs/{job_id}/annotations"]) + try: + data = json.loads(output) + if not data: + return "（无 Annotations）" + result = [] + for a in data: + level = a.get("annotation_level", "?").upper() + msg = a.get("message", "").splitlines()[0] + result.append(f"[{level}] {msg}") + return "\n".join(result) + except json.JSONDecodeError: + return output[:2000] + + +def fetch_run_info_tool(repo: str, run_id: str) -> str: + """获取 Run 元数据""" + output = _run_gh(["api", f"repos/{repo}/actions/runs/{run_id}"]) + try: + data = json.loads(output) + pr_info = "" + if data.get("pull_requests"): + pr = data["pull_requests"][0] + pr_info = f"\nPR: #{pr['number']} - {pr['title']}" + return ( + f"工作流: {data.get('name', 'N/A')}\n" + f"标题: {data.get('display_title', 'N/A')}\n" + f"分支: {data.get('head_branch', 'N/A')}\n" + f"事件: {data.get('event', 'N/A')}\n" + f"结论: {data.get('conclusion', 'N/A')}{pr_info}" + ) + except json.JSONDecodeError: + return output[:2000] + + +def fetch_pr_diff_tool(repo: str, pr_number: str) -> str: + """获取 PR 变更文件列表""" + output = _run_gh(["pr", "diff", pr_number, "--repo", repo, "--name-only"]) + files = output.strip().split("\n") + return "\n".join(f" - {f}" for f in files if f) or "（无变更文件）" + + +def fetch_run_script_tool(repo: str, job_id: str) -> str: + """调用 fetch_run.py 脚本（优先），失败则回退到 fetch-run.sh""" + py_path = SKILL_DIR / "scripts" / "fetch_run.py" + sh_path = SKILL_DIR / "scripts" / "fetch-run.sh" + + # 优先使用 Python（跨平台，编码正确） + if py_path.exists(): + try: + result = subprocess.run( + [sys.executable, str(py_path), "--job", job_id, repo], + capture_output=True, + text=True, + timeout=120, + encoding="utf-8", + errors="replace", + ) + output = result.stdout + if result.stderr: + output += result.stderr + if output.strip(): + return output + except subprocess.TimeoutExpired: + return "fetch_run.py 执行超时（120秒）" + except Exception as e: + pass # 回退到 bash + + # 回退到 bash + if sh_path.exists(): + cmd = f'bash "{sh_path}" --job {job_id} {repo}' + result = bash_execute(cmd) + if result and "错误" not in result and "not found" not in result.lower(): + return result + + return f"错误: fetch_run.py 和 fetch-run.sh 均不存在或执行失败" + + +def _get_mcp_client(): + """获取 MCP 客户端实例""" + sys.path.insert(0, str(SKILL_DIR / "scripts")) + from mcp_client import MCPClient + + return MCPClient() + + +def mcp_list_pods_tool( + namespace: str, start: str, end: str, source: str = "ascend-ci-log" +) -> str: + """列出 Pod""" + try: + client = _get_mcp_client() + result = client.list_pods(namespace, start, end, source) + if "error" in result: + return f"MCP 错误: {result['error']}" + pods = result.get("result", {}).get("pods", []) + return "\n".join(f" - {p}" for p in pods) if pods else "（未找到 Pod）" + except Exception as e: + return f"MCP 调用失败: {e}" + + +def mcp_export_logs_tool( + namespace: str, + pod_name: str, + start: str, + end: str, + keywords: str = "", + source: str = "ascend-ci-log", +) -> str: + """导出日志""" + try: + client = _get_mcp_client() + result = client.export_logs(namespace, pod_name, keywords, start, end, source) + if "error" in result: + return f"MCP 错误: {result['error']}" + export_id = result.get("export_id", "N/A") + status = result.get("status", "pending") + return f"导出任务已创建\n export_id: {export_id}\n 状态: {status}\n\n请使用 mcp_get_export 查询结果" + except Exception as e: + return f"MCP 调用失败: {e}" + + +def mcp_get_export_tool(export_id: str) -> str: + """查询导出结果""" + try: + client = _get_mcp_client() + status = client.get_export_status(export_id) + if "error" in status: + return f"MCP 错误: {status['error']}" + phase = status.get("progress", {}).get("phase", "unknown") + percent = status.get("progress", {}).get("percent", 0) + logs_written = status.get("progress", {}).get("logs_written", 0) + + if phase != "completed": + return f"导出进度: {phase} ({percent}%), 已写入 {logs_written} 行" + + download_url = status.get("result", {}).get("download_url", "") + if download_url: + content = client.download_export(status) + if len(content) > 50000: + return content[:50000] + f"\n... (内容过长，已截断)" + return content + return "无下载链接" + except Exception as e: + return f"MCP 调用失败: {e}" + + +def mcp_get_runner_logs_tool( + runner_name: str, + start: str, + end: str, + namespace: str = "vllm-project", + keywords: str = "", +) -> str: + """一键获取 Runner 日志""" + try: + client = _get_mcp_client() + result = client.get_runner_logs(runner_name, namespace, start, end, keywords) + if not result: + return "未获取到日志" + return result + except Exception as e: + return f"MCP 调用失败: {e}" + + +def execute_tool(tool_name: str, arguments: dict) -> str: + """执行工具调用""" + if tool_name == "fetch_run_script": + return fetch_run_script_tool( + arguments.get("repo", ""), arguments.get("job_id", "") + ) + elif tool_name == "mcp_list_pods": + return mcp_list_pods_tool( + arguments.get("namespace", ""), + arguments.get("start", ""), + arguments.get("end", ""), + arguments.get("source", "ascend-ci-log"), + ) + elif tool_name == "mcp_export_logs": + return mcp_export_logs_tool( + arguments.get("namespace", ""), + arguments.get("pod_name", ""), + arguments.get("start", ""), + arguments.get("end", ""), + arguments.get("keywords", ""), + arguments.get("source", "ascend-ci-log"), + ) + elif tool_name == "mcp_get_export": + return mcp_get_export_tool(arguments.get("export_id", "")) + elif tool_name == "mcp_get_runner_logs": + return mcp_get_runner_logs_tool( + arguments.get("runner_name", ""), + arguments.get("start", ""), + arguments.get("end", ""), + arguments.get("namespace", "vllm-project"), + arguments.get("keywords", ""), + ) + elif tool_name == "fetch_job_info": + return fetch_job_info_tool( + arguments.get("repo", ""), arguments.get("job_id", "") + ) + elif tool_name == "fetch_job_logs": + return fetch_job_logs_tool( + arguments.get("repo", ""), + arguments.get("job_id", ""), + arguments.get("filter_type", "runtime_errors"), + ) + elif tool_name == "fetch_annotations": + return fetch_annotations_tool( + arguments.get("repo", ""), arguments.get("job_id", "") + ) + elif tool_name == "fetch_run_info": + return fetch_run_info_tool( + arguments.get("repo", ""), arguments.get("run_id", "") + ) + elif tool_name == "fetch_pr_diff": + return fetch_pr_diff_tool( + arguments.get("repo", ""), arguments.get("pr_number", "") + ) + elif tool_name == "bash_execute": + return bash_execute(arguments.get("command", "")) + elif tool_name == "read_file": + return read_file_tool(arguments.get("path", "")) + elif tool_name == "grep_file": + return grep_file_tool( + arguments.get("pattern", ""), + arguments.get("path", "."), + arguments.get("include"), + ) + elif tool_name == "glob_file": + return glob_file_tool( + arguments.get("pattern", ""), + arguments.get("path", "."), + ) + elif tool_name == "write_file": + return write_file_tool( + arguments.get("path", ""), + arguments.get("content", ""), + ) + else: + return f"未知工具: {tool_name}" + + +def call_qwen(messages: list, model: str, api_key: str, tools: list) -> dict: + """调用百炼 API（使用 OpenAI 兼容接口）""" + from openai import OpenAI + + client = OpenAI( + api_key=api_key, + base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", + ) + + response = client.chat.completions.create( + model=model, + messages=messages, + tools=tools, + ) + + choice = response.choices[0].message + result = { + "role": choice.role, + "content": choice.content, + "usage": { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + }, + } + if choice.tool_calls: + result["tool_calls"] = [ + { + "id": tc.id, + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments, + }, + } + for tc in choice.tool_calls + ] + + return result + + +def run_diagnosis( + job_url: str, + skill_dir: str, + api_key: str, + model: str, + max_turns: int, +): + """执行诊断流程""" + repo, job_id = parse_job_url(job_url) + print(f"仓库: {repo}") + print(f"Job ID: {job_id}") + print(f"Skill 目录: {skill_dir}") + print() + + skill_content = load_skill_content(skill_dir) + + gh_path = ( + subprocess.run( + "where gh.exe", + shell=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + .stdout.strip() + .split("\n")[0] + if True + else "" + ) + + env_info = f"""环境信息： +- 操作系统: Windows (PowerShell 5.1) +- gh CLI 路径: {gh_path or "C:\\Program Files\\GitHub CLI\\gh.exe"} +- 没有 bash、python3、curl 等 Unix 工具 +- 使用 cmd 命令或 PowerShell 命令 +- 使用 findstr 替代 grep +- 使用 type 替代 cat +- 使用 dir 替代 ls""" + + system_prompt = f"""你是一个 CI 故障诊断专家。请严格按照以下 SKILL 文档的指引，诊断用户提供的失败 Job。 + + +{skill_content} + + +{env_info} + +## 推荐诊断流程 + +**第一步（必须）：调用 fetch_run_script** +- `fetch_run_script(repo, job_id)` 会优先运行 `fetch-run.sh`（bash），失败则回退到 `fetch_run.py` +- 一次性输出： + - Job 基本信息、Runner、失败步骤 + - PR 上下文和变更文件 + - Annotations（通常直接揭示根因） + - 预过滤的错误日志（如需要） +- **这个工具的输出通常已足够完成诊断** + +**第二步（按需）：当 GitHub 日志不足以定位根因时，使用 MCP 查询集群日志** +- 场景：GitHub 日志只显示 exit code 1，但不知道具体原因 +- `mcp_get_runner_logs(runner_name, start, end, keywords)` - 一键获取 Runner Pod 日志 + - runner_name: 从 fetch_run_script 输出中获取（如 linux-aarch64-a3-0-ggwx6-runner-qbq72） + - start/end: 从 fetch_run_script 输出的时间范围获取，格式 YYYY-MM-DD HH:MM:SS + - **keywords: 必须传，根据错误类型选择过滤词**： + - 超时/网络: `"ETIMEDOUT|timeout|connection refused"` + - 内存: `"OOM|OutOfMemory|Killed"` + - 运行时错误: `"RuntimeError|AssertionError|EngineDeadError"` + - 编译错误: `"error:|failed|fatal"` + - 不确定: `"Error|Failed|exit code|Exception"` +- 或分步调用：`mcp_list_pods` → `mcp_export_logs` → `mcp_get_export` + +**重要规则：** +1. **必须先调用 fetch_run_script**，它的输出通常足以完成诊断 +2. **PR 号从 fetch_run_script 输出的 "PR #xxx" 行获取，不要猜测或使用分支名** +3. **fetch_run_script 输出后，如果信息足够，直接输出诊断报告，不要再调用其他工具** +4. 只有在 GitHub 日志不足以定位根因时，才使用 MCP 查询集群日志 +5. **如果某一步获取信息失败（如 PR 不存在），不要反复重试，基于已有信息输出诊断** +6. **最终诊断结果必须有内容，不能输出空报告** +7. 不粘贴大段原始日志，只引用最关键的一行错误标识 +8. 每个问题标注责任方（基础设施团队 / PR 作者） + +Skill 目录: {skill_dir} +当前工作目录: {os.getcwd()}""" + + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": f"请诊断这个失败的 Job: {job_url}\n\n仓库: {repo}\nJob ID: {job_id}", + }, + ] + + print("开始诊断...") + print() + + for turn in range(1, max_turns + 1): + print(f"--- 第 {turn} 轮 ---") + + message = call_qwen(messages, model, api_key, TOOLS) + + if message.get("tool_calls"): + tool_calls = message["tool_calls"] + tool_results = [] + + for tc in tool_calls: + tool_name = tc["function"]["name"] + tool_args = json.loads(tc["function"]["arguments"]) + + print(f" 调用工具: {tool_name}") + print(f" 参数: {json.dumps(tool_args, ensure_ascii=False)[:200]}") + + result = execute_tool(tool_name, tool_args) + + print(f" 结果长度: {len(result)} 字符") + if len(result) > 300: + print(f" 结果预览: {result[:300]}...") + else: + print(f" 结果: {result}") + print() + + tool_results.append( + { + "role": "tool", + "tool_call_id": tc["id"], + "content": result, + } + ) + + messages.append(message) + messages.extend(tool_results) + + else: + content = message.get("content", "").strip() + if not content: + # AI 输出了空内容，强制生成诊断 + print("⚠️ AI 输出为空，基于已有信息生成诊断...") + messages.append({ + "role": "user", + "content": "请基于以上收集的信息，立即输出诊断报告。即使信息不完整，也要给出当前最佳判断。格式：\n\n# CI 故障诊断报告\n\n- **定性**: 类型 A/B/C/D/E\n- **根因**: ...\n- **关键标识**: ...\n- **责任方**: ...\n- **建议**: ..." + }) + continue + print("诊断完成！") + print() + print("=" * 60) + print(content) + print("=" * 60) + return + + print(f"达到最大轮数限制 ({max_turns})，停止。") + + +def check_prerequisites(api_key: str): + """检查前置要求和依赖""" + errors = [] + warnings = [] + + # 1. Python 版本 + if sys.version_info < (3, 8): + errors.append(f"Python 版本过低 (当前 {sys.version.split()[0]})，需要 3.8+") + + # 2. 依赖包 + for pkg in ["openai", "requests"]: + try: + __import__(pkg) + except ImportError: + errors.append(f"缺少依赖: {pkg} (pip install {pkg})") + + # 3. gh CLI + gh_path = shutil.which("gh") or shutil.which("gh.exe") + gh_found = False + gh_logged_in = False + if gh_path: + gh_found = True + # 检查登录状态 + try: + result = subprocess.run( + [gh_path, "auth", "status"], + capture_output=True, text=True, timeout=10, + encoding="utf-8", errors="replace", + ) + if result.returncode == 0: + gh_logged_in = True + else: + errors.append("gh 未登录，请先运行: gh auth login") + except Exception: + warnings.append("无法检查 gh 登录状态") + else: + errors.append("未找到 gh CLI，请安装: https://cli.github.com/") + + # 4. API Key + if not api_key: + errors.append("未设置 API Key，请使用 --api-key 参数或设置环境变量 DASHSCOPE_API_KEY") + + # 5. 脚本文件存在性（警告级别） + for script in ["fetch_run.py", "mcp_client.py"]: + if not (SKILL_DIR / "scripts" / script).exists(): + warnings.append(f"脚本不存在: {script}") + + # 6. SKILL.md + if not (SKILL_DIR / "SKILL.md").exists(): + warnings.append("SKILL.md 不存在，AI 将缺少诊断指引") + + # 输出检查结果 + if errors: + print("\n❌ 前置检查失败:") + for e in errors: + print(f" - {e}") + print() + sys.exit(1) + + # 成功输出 + print("✅ 前置检查通过") + print(f" Python {sys.version.split()[0]}") + if gh_path: + status = "已登录" if gh_logged_in else "未登录" + print(f" gh CLI: {gh_path} ({status})") + print(f" 依赖包: openai, requests") + if api_key: + masked = api_key[:6] + "..." + api_key[-4:] if len(api_key) > 10 else "已设置" + print(f" API Key: {masked}") + if warnings: + print("\n⚠️ 警告:") + for w in warnings: + print(f" - {w}") + print() + + +def main(): + args = parse_args() + + check_prerequisites(args.api_key) + + run_diagnosis( + job_url=args.url, + skill_dir=args.skill_dir, + api_key=args.api_key, + model=args.model, + max_turns=args.max_turns, + ) + + +if __name__ == "__main__": + main() diff --git a/skills/infrastructure/github-action-diagnose/scripts/fetch-run.sh b/skills/infrastructure/github-action-diagnose/scripts/fetch-run.sh new file mode 100644 index 0000000..cbfcac7 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/scripts/fetch-run.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# scripts/fetch-run.sh [owner/repo] +# +# 用途：一次性抓取 CI Run 所有失败 Job 的 runner 信息和关键日志 +# 将每个 Job 原始日志（可达数百行）压缩至 ~20 行关键错误行 +# 在 LLM 进行 Step 2+ 推理之前完成所有数据收集，节省 token +# +# 用法： +# bash scripts/fetch-run.sh 22935910341 # Run ID +# bash scripts/fetch-run.sh 22935910341 owner/repo # Run ID + repo +# bash scripts/fetch-run.sh --job 67875153370 owner/repo # 单个 Job ID +# +# 输出： +# - Run 概览（Job 状态列表 + Annotations） +# - 每个失败 Job：runner 名称 + 预过滤的关键日志（~20 行） +# - cancelled Job：提示去看 Annotations（queue 抢占等无需读日志） +# +# 依赖：gh CLI（已登录） + +set -uo pipefail + +# 解析参数 +JOB_ID="" +RUN_ID="" +REPO="vllm-project/vllm-ascend" + +# 解析参数：支持 "--job [repo]" 或 " [repo]" +if [[ "$1" == "--job" ]]; then + JOB_ID="$2" + REPO="${3:-vllm-project/vllm-ascend}" +else + RUN_ID="${1:-}" + REPO="${2:-vllm-project/vllm-ascend}" +fi + +if [[ -n "$JOB_ID" ]]; then + # 单 Job 模式 + echo "════════════════════════════════════════════" + echo " Single Job Mode | $REPO" + echo "════════════════════════════════════════════" + + # ── Step A：基本信息（Runner / 时间 / 结论）────────────────────────────────── + JOB_META=$(gh api "repos/$REPO/actions/jobs/$JOB_ID" 2>/dev/null) + echo "$JOB_META" | \ + python3 -c " +import json,sys +d=json.load(sys.stdin) +print(f'── [{d[\"conclusion\"].upper()}] {d[\"name\"]}') +print(f' Runner : {d.get(\"runner_name\") or \"(unknown)\"}') +print(f' 时间 : {d.get(\"started_at\",\"?\")} ~ {d.get(\"completed_at\",\"?\")}') +steps=[s for s in d.get('steps',[]) if s.get('conclusion') not in ('success','skipped',None)] +if steps: + print(' ── 失败步骤') + for s in steps: + print(f' [{s[\"conclusion\"].upper():8}] Step {s[\"number\"]}: {s[\"name\"]}') +" + + # ── Step B：PR 上下文 ───────────────────────────────────────────────────────── + RUN_ID_FOR_JOB=$(echo "$JOB_META" | python3 -c "import json,sys; print(json.load(sys.stdin)['run_id'])" 2>/dev/null || echo "") + if [[ -n "$RUN_ID_FOR_JOB" ]]; then + RUN_META=$(gh api "repos/$REPO/actions/runs/$RUN_ID_FOR_JOB" 2>/dev/null) + PR_NUM=$(echo "$RUN_META" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['pull_requests'][0]['number'] if d.get('pull_requests') else '')" 2>/dev/null || echo "") + RUN_TITLE=$(echo "$RUN_META" | python3 -c "import json,sys; print(json.load(sys.stdin).get('display_title',''))" 2>/dev/null || echo "") + RUN_BRANCH=$(echo "$RUN_META" | python3 -c "import json,sys; print(json.load(sys.stdin).get('head_branch',''))" 2>/dev/null || echo "") + echo " PR : ${PR_NUM:+#$PR_NUM }${RUN_TITLE} (${RUN_BRANCH})" + if [[ -n "$PR_NUM" ]]; then + echo " ── PR 变更文件（分类参考）" + gh pr diff "$PR_NUM" --repo "$REPO" --name-only 2>/dev/null | head -20 | sed 's/^/ /' + fi + fi + + # ── Step C：Annotations（通常直接揭示根因，如 ETIMEDOUT / exit code）───────── + echo "" + echo " ── Annotations" + ANNOTATIONS=$(gh api "repos/$REPO/check-runs/$JOB_ID/annotations" 2>/dev/null) + ANN_COUNT=$(echo "$ANNOTATIONS" | python3 -c "import json,sys; print(len(json.load(sys.stdin)))" 2>/dev/null || echo "0") + if [[ "$ANN_COUNT" -gt 0 ]]; then + echo "$ANNOTATIONS" | python3 -c " +import json,sys +for a in json.load(sys.stdin): + print(f' [{a.get(\"annotation_level\",\"?\").upper()}] {a.get(\"message\",\"\").splitlines()[0]}') +" + else + echo " （无 Annotations）" + fi + + # ── Step D：判断是否需要拉完整日志 ─────────────────────────────────────────── + # Annotations 中包含明确错误信号时，跳过日志拉取（节省时间） + ANN_TEXT=$(echo "$ANNOTATIONS" | python3 -c "import json,sys; print(' '.join(a.get('message','') for a in json.load(sys.stdin)))" 2>/dev/null || echo "") + NEEDS_LOG=true + if echo "$ANN_TEXT" | grep -qiE 'ETIMEDOUT|timeout|failed to create artifact|network|connection refused'; then + echo "" + echo " ── Annotations 已明确根因（网络/超时类），跳过完整日志拉取" + NEEDS_LOG=false + fi + + if $NEEDS_LOG; then + FULL_LOG=$(gh api "repos/$REPO/actions/jobs/$JOB_ID/logs" 2>&1) + + echo "" + echo " ── 运行时错误（高优先级）" + echo "$FULL_LOG" \ + | grep -iE 'RuntimeError|OutOfMemoryError|NPU out of memory|AssertionError|EngineDeadError|EngineCore encountered|shm_broadcast.*60 seconds|No available shared memory broadcast|Process completed with exit code [1-9]|exit code 255|synchronized memcpy failed|NPU function error|error code is [0-9]|httpcore\.(ReadError|ConnectError)|httpx\.(ReadError|ConnectError)|RemoteProtocolError' \ + | grep -v -E 'ops_error\.h|error_check\.h' \ + | head -15 + + RUNTIME_COUNT=$(echo "$FULL_LOG" | grep -cE 'RuntimeError|OutOfMemoryError|NPU out of memory|AssertionError|EngineDeadError|shm_broadcast.*60 seconds' || true) + if [[ "$RUNTIME_COUNT" -eq 0 ]]; then + echo " ── 编译/安装错误（次优先级）" + echo "$FULL_LOG" \ + | grep -iE 'error:.*failed|error:.*exit|fatal error|ERROR.*install|No solution found|unsatisfiable|ERROR.*Build|dubious ownership' \ + | grep -v -E '##\[group\]|##\[endgroup\]|warning.*format|ops_error\.h|error_check\.h|tiling func|tiling failed' \ + | head -5 + fi + + echo " ── 最后输出（定位失败阶段）" + echo "$FULL_LOG" \ + | grep -E '^[^[:space:]]+[[:space:]]+(UNKNOWN STEP|Run python|bash|pytest)' \ + | tail -5 + fi + + exit 0 +fi + +if [[ -z "$RUN_ID" ]]; then + echo "用法: fetch-run.sh [owner/repo]" + echo " 或: fetch-run.sh --job [owner/repo]" + exit 1 +fi + +SEP="════════════════════════════════════════════" + +# ── 1. Run 概览（Job 列表 + Annotations）──────────────────────────────────── +echo "$SEP" +echo " Run $RUN_ID | $REPO" +echo "$SEP" +gh run view "$RUN_ID" --repo "$REPO" +echo "" + +# ── PR 上下文（用于分类时区分代码 Bug 与基础设施）────────────────────────── +PR_NUM=$(gh api "repos/$REPO/actions/runs/$RUN_ID" \ + --jq '.pull_requests[0].number // empty' 2>/dev/null || echo "") +if [[ -n "$PR_NUM" ]]; then + echo "$SEP" + echo " PR #$PR_NUM 变更文件（分类参考）" + echo "$SEP" + gh pr diff "$PR_NUM" --repo "$REPO" --name-only 2>/dev/null | head -20 + echo "" +fi + +# ── 2. 获取失败 / 取消的 Job ID ────────────────────────────────────────────── +FAILED_IDS=$(gh api "repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "cancelled") | .id') + +if [[ -z "$FAILED_IDS" ]]; then + echo "（未发现失败或取消的 Job）" + exit 0 +fi + +TOTAL=$(echo "$FAILED_IDS" | wc -l | tr -d ' ') +echo "$SEP" +echo " 失败 / 取消 Job 详情（共 ${TOTAL} 个）" +echo "$SEP" + +# ── 3. 逐 Job 输出 runner 信息 + 关键日志 ──────────────────────────────────── +for JOB_ID in $FAILED_IDS; do + echo "" + + # Runner 信息（一次 API 调用，获取名称 / 结论 / 时间） + gh api "repos/$REPO/actions/jobs/$JOB_ID" \ + --jq '"── [\(.conclusion | ascii_upcase)] \(.name)\n Runner : \(.runner_name // "(unknown)")\n 时间 : \(.started_at // "?") ~ \(.completed_at // "?")"' + + CONCLUSION=$(gh api "repos/$REPO/actions/jobs/$JOB_ID" --jq '.conclusion') + + if [[ "$CONCLUSION" == "failure" ]]; then + FULL_LOG=$(gh run view --job "$JOB_ID" --log --repo "$REPO" 2>&1) + + echo " ── 运行时错误（高优先级）" + # 同时捕获 CANN 底层错误（aclrtMemcpy/error code），便于区分硬件 vs 代码兼容性 + echo "$FULL_LOG" \ + | grep -iE 'RuntimeError|OutOfMemoryError|NPU out of memory|AssertionError|EngineDeadError|EngineCore encountered|shm_broadcast.*60 seconds|No available shared memory broadcast|Process completed with exit code [1-9]|exit code 255|synchronized memcpy failed|NPU function error|error code is [0-9]|httpcore\.(ReadError|ConnectError)|httpx\.(ReadError|ConnectError)|RemoteProtocolError' \ + | grep -v -E 'ops_error\.h|error_check\.h' \ + | head -15 + + RUNTIME_COUNT=$(echo "$FULL_LOG" | grep -cE 'RuntimeError|OutOfMemoryError|NPU out of memory|AssertionError|EngineDeadError|shm_broadcast.*60 seconds' || true) + if [[ "$RUNTIME_COUNT" -eq 0 ]]; then + echo " ── 编译/安装错误（次优先级）" + echo "$FULL_LOG" \ + | grep -iE 'error:.*failed|error:.*exit|fatal error|ERROR.*install|No solution found|unsatisfiable|ERROR.*Build|dubious ownership' \ + | grep -v -E '##\[group\]|##\[endgroup\]|warning.*format|ops_error\.h|error_check\.h|tiling func|tiling failed' \ + | head -5 + fi + + # 显示 Job 最后阶段的关键标识 + echo " ── 最后输出（定位失败阶段）" + echo "$FULL_LOG" \ + | grep -E '^[^[:space:]]+[[:space:]]+(UNKNOWN STEP|Run python|bash|pytest)' \ + | tail -5 + + else + echo " （cancelled — 根因见 Run 概览中的 ANNOTATIONS，通常为 queue 抢占）" + fi +done + +echo "" +echo "$SEP" +echo " 补充命令（按需使用）：" +echo " 完整日志 : gh run view --job --log --repo $REPO" +echo " PR diff : gh pr diff --repo $REPO --name-only" +echo " kubectl : kubectl get pods --all-namespaces | grep " +echo "$SEP" diff --git a/skills/infrastructure/github-action-diagnose/scripts/fetch_run.py b/skills/infrastructure/github-action-diagnose/scripts/fetch_run.py new file mode 100644 index 0000000..7f03926 --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/scripts/fetch_run.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +""" +fetch_run.py - 一次性抓取 CI Run/Job 的 runner 信息和关键日志 + +用途：将每个 Job 原始日志（可达数百行）压缩至 ~20 行关键错误行 + 在 LLM 进行 Step 2+ 推理之前完成所有数据收集，节省 token + +用法： + python fetch_run.py --run [owner/repo] + python fetch_run.py --job [owner/repo] + +依赖：gh CLI（已登录） +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import io +from pathlib import Path + +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") +sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8") + + +DEFAULT_REPO = "vllm-project/vllm-ascend" + +RUNTIME_PATTERNS = ( + r"RuntimeError|OutOfMemoryError|NPU out of memory|AssertionError|" + r"AttributeError|UnboundLocalError|TypeError|ValueError|ImportError|" + r"EngineDeadError|EngineCore encountered|shm_broadcast.*60 seconds|" + r"No available shared memory broadcast|" + r"Process completed with exit code [1-9]|exit code 255|" + r"synchronized memcpy failed|NPU function error|error code is [0-9]|" + r"httpcore\.(ReadError|ConnectError)|httpx\.(ReadError|ConnectError)|" + r"RemoteProtocolError" +) + +COMPILE_PATTERNS = ( + r"error:.*failed|error:.*exit|fatal error|ERROR.*install|" + r"No solution found|unsatisfiable|ERROR.*Build|dubious ownership" +) + +NETWORK_PATTERNS = ( + r"ETIMEDOUT|timeout|failed to create artifact|network|connection refused" +) + +EXCLUDE_PATTERNS = r"ops_error\.h|error_check\.h" + +COMPILE_EXCLUDE = ( + r"##\[group\]|##\[endgroup\]|warning.*format|" + r"ops_error\.h|error_check\.h|tiling func|tiling failed" +) + + +def find_gh() -> str: + """查找 gh.exe 路径""" + env_path = os.environ.get("GH_PATH", "") + if env_path and Path(env_path).exists(): + return env_path + + candidates = [ + r"C:\Program Files\GitHub CLI\gh.exe", + r"C:\Program Files (x86)\GitHub CLI\gh.exe", + ] + for c in candidates: + if Path(c).exists(): + return c + + try: + result = subprocess.run( + "where gh.exe", + shell=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if result.stdout.strip(): + return result.stdout.strip().split("\n")[0] + except Exception: + pass + + return "gh" + + +def run_gh(args: list, repo: str = "") -> str: + """执行 gh 命令，返回 stdout""" + gh = find_gh() + cmd = [gh] + args + if repo: + cmd.extend(["--repo", repo]) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=60, + encoding="utf-8", + errors="replace", + ) + return result.stdout + result.stderr + except subprocess.TimeoutExpired: + return "(命令超时)" + except Exception as e: + return f"(执行错误: {e})" + + +def gh_api(endpoint: str, repo: str = "") -> dict: + """调用 gh api，返回解析后的 JSON""" + output = run_gh(["api", endpoint], repo=repo) + try: + return json.loads(output) + except json.JSONDecodeError: + return {} + + +def grep_lines( + text: str, include_pattern: str, exclude_pattern: str = "", max_lines: int = 0 +) -> str: + """按正则过滤行""" + lines = text.splitlines() + matched = [] + inc_re = re.compile(include_pattern, re.IGNORECASE) + exc_re = re.compile(exclude_pattern, re.IGNORECASE) if exclude_pattern else None + + for line in lines: + if inc_re.search(line): + if exc_re and exc_re.search(line): + continue + matched.append(line) + if max_lines and len(matched) >= max_lines: + break + + return "\n".join(matched) + + +def fetch_job_meta(repo: str, job_id: str) -> dict: + """获取 Job 元数据""" + return gh_api(f"repos/{repo}/actions/jobs/{job_id}") + + +def fetch_run_meta(repo: str, run_id: str) -> dict: + """获取 Run 元数据""" + return gh_api(f"repos/{repo}/actions/runs/{run_id}") + + +def fetch_annotations(repo: str, job_id: str) -> list: + """获取 Annotations""" + data = gh_api(f"repos/{repo}/check-runs/{job_id}/annotations") + return data if isinstance(data, list) else [] + + +def fetch_pr_diff(repo: str, pr_number: str) -> list: + """获取 PR 变更文件列表""" + output = run_gh(["pr", "diff", pr_number, "--name-only"], repo=repo) + return [f for f in output.strip().splitlines() if f] + + +def fetch_job_logs(repo: str, job_id: str) -> str: + """获取 Job 完整日志""" + gh = find_gh() + log_file = f"_fetch_run_job_{job_id}.log" + try: + # Use powershell to redirect with UTF-8 encoding + subprocess.run( + f'cmd /c chcp 65001 >nul & "{gh}" run view --job {job_id} --log --repo {repo} > "{log_file}" 2>nul', + shell=True, + capture_output=True, + timeout=120, + ) + content = Path(log_file).read_text(encoding="utf-8", errors="replace") + Path(log_file).unlink(missing_ok=True) + return content + except Exception: + try: + Path(log_file).unlink(missing_ok=True) + except Exception: + pass + return "" + + +def has_network_root_cause(annotations: list) -> bool: + """判断 Annotations 是否已明确网络/超时类根因""" + for a in annotations: + msg = a.get("message", "") + if re.search(NETWORK_PATTERNS, msg, re.IGNORECASE): + return True + return False + + +def format_job_header(meta: dict) -> str: + """格式化 Job 头部信息""" + conclusion = (meta.get("conclusion") or "unknown").upper() + name = meta.get("name", "N/A") + runner = meta.get("runner_name") or "(unknown)" + started = meta.get("started_at", "?") + completed = meta.get("completed_at", "?") + + lines = [f"── [{conclusion}] {name}"] + lines.append(f" Runner : {runner}") + lines.append(f" 时间 : {started} ~ {completed}") + + steps = [ + s + for s in meta.get("steps", []) + if s.get("conclusion") not in ("success", "skipped", None) + ] + if steps: + lines.append(" ── 失败步骤") + for s in steps: + lines.append( + f" [{s['conclusion'].upper():8}] Step {s['number']}: {s['name']}" + ) + + return "\n".join(lines) + + +def format_annotations(annotations: list) -> str: + """格式化 Annotations""" + if not annotations: + return " （无 Annotations）" + lines = [] + for a in annotations: + level = a.get("annotation_level", "?").upper() + msg = a.get("message", "").splitlines()[0] + lines.append(f" [{level}] {msg}") + return "\n".join(lines) + + +def diagnose_single_job(repo: str, job_id: str) -> str: + """诊断单个 Job""" + output = [] + output.append("=" * 40) + output.append(f" Single Job Mode | {repo}") + output.append("=" * 40) + + # Step A: 基本信息 + job_meta = fetch_job_meta(repo, job_id) + if not job_meta: + output.append(f"错误: 无法获取 Job {job_id} 的信息") + return "\n".join(output) + + output.append(format_job_header(job_meta)) + + # Step B: PR 上下文 + run_id = job_meta.get("run_id", "") + if run_id: + run_meta = fetch_run_meta(repo, str(run_id)) + prs = run_meta.get("pull_requests", []) + pr_num = prs[0].get("number") if prs else None + title = run_meta.get("display_title", "") + branch = run_meta.get("head_branch", "") + + pr_str = f"#{pr_num} " if pr_num else "" + output.append(f" PR : {pr_str}{title} ({branch})") + + if pr_num: + output.append(" ── PR 变更文件（分类参考）") + diff_files = fetch_pr_diff(repo, str(pr_num)) + for f in diff_files[:20]: + output.append(f" {f}") + + # Step C: Annotations + output.append("") + output.append(" ── Annotations") + annotations = fetch_annotations(repo, job_id) + output.append(format_annotations(annotations)) + + # Step D: 判断是否需要拉日志 + if has_network_root_cause(annotations): + output.append("") + output.append(" ── Annotations 已明确根因（网络/超时类），跳过完整日志拉取") + return "\n".join(output) + + # Step E: 拉取并过滤日志 + full_log = fetch_job_logs(repo, job_id) + if not full_log: + output.append("") + output.append(" ── 无法获取日志") + return "\n".join(output) + + # 运行时错误 + runtime_lines = grep_lines( + full_log, RUNTIME_PATTERNS, EXCLUDE_PATTERNS, max_lines=15 + ) + output.append("") + output.append(" ── 运行时错误（高优先级）") + if runtime_lines: + output.append(runtime_lines) + else: + # 编译/安装错误 + compile_lines = grep_lines( + full_log, COMPILE_PATTERNS, COMPILE_EXCLUDE, max_lines=5 + ) + output.append(" ── 编译/安装错误（次优先级）") + if compile_lines: + output.append(compile_lines) + else: + output.append(" （未发现关键错误行）") + + # 最后输出 + last_lines = grep_lines( + full_log, r"^[^\s]+\s+(UNKNOWN STEP|Run python|bash|pytest)", max_lines=5 + ) + output.append(" ── 最后输出（定位失败阶段）") + if last_lines: + output.append(last_lines) + + return "\n".join(output) + + +def diagnose_run(repo: str, run_id: str) -> str: + """诊断整个 Run 的所有失败 Job""" + output = [] + sep = "=" * 40 + output.append(sep) + output.append(f" Run {run_id} | {repo}") + output.append(sep) + + # Run 概览 + run_view = run_gh(["run", "view", run_id], repo=repo) + output.append(run_view.strip()) + output.append("") + + # PR 上下文 + run_meta = fetch_run_meta(repo, run_id) + prs = run_meta.get("pull_requests", []) + pr_num = prs[0].get("number") if prs else None + if pr_num: + output.append(sep) + output.append(f" PR #{pr_num} 变更文件（分类参考）") + output.append(sep) + diff_files = fetch_pr_diff(repo, str(pr_num)) + for f in diff_files[:20]: + output.append(f) + output.append("") + + # 获取失败/取消的 Job + jobs_data = gh_api(f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100") + jobs = jobs_data.get("jobs", []) + failed_jobs = [j for j in jobs if j.get("conclusion") in ("failure", "cancelled")] + + if not failed_jobs: + output.append("（未发现失败或取消的 Job）") + return "\n".join(output) + + output.append(sep) + output.append(f" 失败 / 取消 Job 详情（共 {len(failed_jobs)} 个）") + output.append(sep) + + for job in failed_jobs: + job_id = job["id"] + output.append("") + output.append(format_job_header(job)) + + conclusion = job.get("conclusion", "") + if conclusion == "failure": + full_log = fetch_job_logs(repo, str(job_id)) + + runtime_lines = grep_lines( + full_log, RUNTIME_PATTERNS, EXCLUDE_PATTERNS, max_lines=15 + ) + output.append(" ── 运行时错误（高优先级）") + if runtime_lines: + output.append(runtime_lines) + else: + compile_lines = grep_lines( + full_log, COMPILE_PATTERNS, COMPILE_EXCLUDE, max_lines=5 + ) + output.append(" ── 编译/安装错误（次优先级）") + if compile_lines: + output.append(compile_lines) + else: + output.append(" （未发现关键错误行）") + + last_lines = grep_lines( + full_log, + r"^[^\s]+\s+(UNKNOWN STEP|Run python|bash|pytest)", + max_lines=5, + ) + output.append(" ── 最后输出（定位失败阶段）") + if last_lines: + output.append(last_lines) + else: + output.append( + " （cancelled — 根因见 Run 概览中的 ANNOTATIONS，通常为 queue 抢占）" + ) + + output.append("") + output.append(sep) + output.append(" 补充命令（按需使用）：") + output.append(f" 完整日志 : gh run view --job --log --repo {repo}") + output.append(f" PR diff : gh pr diff --repo {repo} --name-only") + output.append( + f" kubectl : kubectl get pods --all-namespaces | grep " + ) + output.append(sep) + + return "\n".join(output) + + +def main(): + parser = argparse.ArgumentParser(description="获取 CI Run/Job 的关键日志") + parser.add_argument("--run", help="Run ID") + parser.add_argument("--job", help="Job ID") + parser.add_argument("repo", nargs="?", default=DEFAULT_REPO, help="Owner/repo") + args = parser.parse_args() + + repo = args.repo + if args.job: + print(diagnose_single_job(repo, args.job)) + elif args.run: + print(diagnose_run(repo, args.run)) + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/skills/infrastructure/github-action-diagnose/scripts/mcp_client.py b/skills/infrastructure/github-action-diagnose/scripts/mcp_client.py new file mode 100644 index 0000000..d56eeaf --- /dev/null +++ b/skills/infrastructure/github-action-diagnose/scripts/mcp_client.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +""" +mcp_client.py - MCP 客户端，用于查询 LTS 集群日志 + +用法: + from mcp_client import MCPClient + client = MCPClient() + pods = client.list_pods("vllm-project", "2026-03-27 21:00:00", "2026-03-27 22:00:00") + export_id = client.export_logs("vllm-project", "runner-pod-name", "Error|Failed", ...) + status = client.get_export_status(export_id) +""" + +import json +import gzip +import time +import requests +from typing import Optional + + +class MCPClient: + """MCP 客户端，封装 LTS 日志查询操作""" + + DEFAULT_URL = "http://150.158.143.223:30089/mcp" + DEFAULT_SOURCE = "ascend-ci-log" + + def __init__(self, url: str = None): + self.url = url or self.DEFAULT_URL + self.session = requests.Session() + self.session.headers.update( + { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + ) + self._initialized = False + + def _ensure_initialized(self): + if self._initialized: + return + init_req = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "ci-diagnose", "version": "1.0"}, + }, + } + r = self.session.post(self.url, json=init_req, timeout=10) + session_id = r.headers.get("mcp-session-id", "") + if session_id: + self.session.headers["Mcp-Session-Id"] = session_id + self._initialized = True + + def _call_tool(self, tool_name: str, args: dict, timeout: int = 30) -> dict: + self._ensure_initialized() + req = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": tool_name, "arguments": args}, + } + r = self.session.post(self.url, json=req, timeout=timeout) + for line in r.text.split("\r\n"): + if line.startswith("data:"): + data = json.loads(line[5:]) + if data.get("isError"): + error_msg = data.get("content", [{}])[0].get( + "text", "Unknown error" + ) + return {"error": error_msg} + return data.get("result", {}) + return {"error": "No response from MCP server"} + + def list_sources(self) -> dict: + return self._call_tool("lts_list_sources", {}) + + def list_namespaces( + self, source: str = None, start: str = None, end: str = None + ) -> dict: + args = {"source": source or self.DEFAULT_SOURCE} + if start and end: + args["time"] = {"start": start, "end": end} + return self._call_tool("lts_list_namespaces", args) + + def list_pods( + self, namespace: str, start: str = None, end: str = None, source: str = None + ) -> dict: + args = {"source": source or self.DEFAULT_SOURCE, "namespace": namespace} + if start and end: + args["time"] = {"start": start, "end": end} + return self._call_tool("lts_list_pods", args, timeout=60) + + def export_logs( + self, + namespace: str, + pod_name: str = "", + keywords: str = "", + start: str = None, + end: str = None, + source: str = None, + ) -> dict: + args = { + "source": source or self.DEFAULT_SOURCE, + "namespace": namespace, + "pod_name": pod_name, + "keywords": keywords, + } + if start and end: + args["time"] = {"start": start, "end": end} + return self._call_tool("lts_export_logs", args, timeout=60) + + def get_export_status(self, export_id: str) -> dict: + return self._call_tool("lts_get_export_status", {"export_id": export_id}) + + def download_export(self, export_result: dict) -> bytes: + # 尝试多种路径获取 download_url + download_url = "" + # 路径 1: structuredContent.result.result.download_url + download_url = ( + export_result.get("structuredContent", {}) + .get("result", {}) + .get("result", {}) + .get("download_url", "") + ) + # 路径 2: result.result.download_url + if not download_url: + download_url = ( + export_result.get("result", {}) + .get("result", {}) + .get("download_url", "") + ) + # 路径 3: 从 content[0].text 解析 + if not download_url: + content = export_result.get("content", [{}])[0].get("text", "{}") + try: + parsed = json.loads(content) + download_url = parsed.get("result", {}).get("download_url", "") + except json.JSONDecodeError: + pass + if not download_url: + raise ValueError( + f"No download URL in export result: {str(export_result)[:200]}" + ) + r = requests.get(download_url, timeout=60) + r.raise_for_status() + return r.content + + def find_runner_pod( + self, + runner_name: str, + namespace: str = "vllm-project", + start: str = None, + end: str = None, + ) -> Optional[str]: + result = self.list_pods(namespace, start, end) + if "error" in result: + return None + # 解析 structuredContent.result.pods + pods = result.get("structuredContent", {}).get("result", {}).get("pods", []) + if not pods: + # 回退：从 content 字段解析 + content = result.get("content", [{}])[0].get("text", "{}") + try: + parsed = json.loads(content) + pods = parsed.get("pods", []) + except json.JSONDecodeError: + pass + for pod in pods: + if runner_name in pod: + return pod + return None + + def get_runner_logs( + self, + runner_name: str, + namespace: str = "vllm-project", + start: str = None, + end: str = None, + keywords: str = None, + ) -> Optional[str]: + # 方案 B：默认 keywords 过滤，避免返回全量日志 + if keywords is None or keywords == "": + keywords = "Error|Failed|Exception|exit code|Timeout|OOM|Killed|panic|fatal|WARNING" + pod_name = self.find_runner_pod(runner_name, namespace, start, end) + if not pod_name: + return f"未找到匹配的 Pod (runner: {runner_name}, namespace: {namespace})" + + export_result = self.export_logs(namespace, pod_name, keywords, start, end) + if "error" in export_result: + return f"导出失败: {export_result['error']}" + + # 提取 export_id（可能在顶层或 structuredContent.result 中） + export_id = export_result.get("export_id", "") + if not export_id: + export_id = ( + export_result.get("structuredContent", {}) + .get("result", {}) + .get("export_id", "") + ) + if not export_id: + # 回退：从 content 解析 + content = export_result.get("content", [{}])[0].get("text", "{}") + try: + export_id = json.loads(content).get("export_id", "") + except json.JSONDecodeError: + pass + if not export_id: + return "未返回 export_id" + + for _ in range(30): + time.sleep(3) + status = self.get_export_status(export_id) + phase = status.get("progress", {}).get("phase", "") + if phase == "completed": + break + if phase == "failed": + return f"导出任务失败: {status.get('error', '')}" + + try: + raw_data = self.download_export(status) + data = gzip.decompress(raw_data) + lines = data.decode("utf-8", errors="replace").strip().split("\n") + log_lines = [] + for line in lines: + try: + entry = json.loads(line) + content = entry.get("content", "") + if content: + log_lines.append(content) + except json.JSONDecodeError: + continue + result = "\n".join(log_lines) + if len(result) > 50000: + return ( + result[:50000] + f"\n... (内容过长，已截断，共 {len(log_lines)} 行)" + ) + return result + except Exception as e: + return f"下载/解析失败: {e}"