diff --git a/.github/scripts/collect-cache-health.sh b/.github/scripts/collect-cache-health.sh new file mode 100755 index 000000000000..e6c19a70e687 --- /dev/null +++ b/.github/scripts/collect-cache-health.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Scheduled collector: scans recent reviewer runs' LOGS for the CACHE_USAGE line +# emitted by emit-cache-usage.sh, computes the warm-hit trend, and fails (a red +# scheduled run = built-in alert) if caching looks regressed across a meaningful +# sample. +# +# Reads only run logs (token counts) - no message content, no artifacts. Runs +# entirely out of the review hot path. +# +# Tunables (env): +# LOOKBACK how many recent reviewer runs to scan (default 30) +# CACHE_CREATION_WARN_TOKENS a run is "cold" if creation exceeds this (default 15000) +# COLD_ALERT_PCT alert if more than this % of the sample is cold (default 40) +# MIN_SAMPLE need at least this many data-carrying runs to judge (default 10) +set -euo pipefail + +REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY not set}" +LOOKBACK="${LOOKBACK:-30}" +WARN_TOKENS="${CACHE_CREATION_WARN_TOKENS:-15000}" +COLD_ALERT_PCT="${COLD_ALERT_PCT:-40}" +MIN_SAMPLE="${MIN_SAMPLE:-10}" + +mapfile -t RUNS < <(gh run list --repo "$REPO" --workflow claude-review.yml \ + --status completed --limit "$LOOKBACK" --json databaseId --jq '.[].databaseId') + +scanned=0 +cold=0 +rows="" +for id in "${RUNS[@]}"; do + line=$(gh run view "$id" --repo "$REPO" --log 2>/dev/null | grep -m1 'CACHE_USAGE' || true) + [ -n "$line" ] || continue + c=$(sed -n 's/.*creation=\([0-9][0-9]*\).*/\1/p' <<<"$line") + r=$(sed -n 's/.*read=\([0-9][0-9]*\).*/\1/p' <<<"$line") + [ -n "$c" ] && [ -n "$r" ] || continue + # no cache activity at all = no data, not a warm sample + [ "$c" -eq 0 ] && [ "$r" -eq 0 ] && continue + t=$(( c + r )); ratio=0; [ "$t" -gt 0 ] && ratio=$(( r * 100 / t )) + scanned=$(( scanned + 1 )) + [ "$c" -gt "$WARN_TOKENS" ] && cold=$(( cold + 1 )) + rows="${rows}| ${id} | ${c} | ${r} | ${ratio}% |"$'\n' +done + +{ + echo "### Reviewer cache monitor" + echo "" + echo "Scanned **${scanned}** data-carrying runs of the last ${LOOKBACK}." + echo "" + echo "| run | creation | read | warm% |" + echo "| --- | --- | --- | --- |" + printf '%s' "$rows" +} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +if [ "$scanned" -lt "$MIN_SAMPLE" ]; then + echo "Only ${scanned} runs carried cache data (< ${MIN_SAMPLE}); not enough to judge — skipping." + exit 0 +fi + +coldpct=$(( cold * 100 / scanned )) +echo "cold=${cold}/${scanned} (${coldpct}%) alert threshold=${COLD_ALERT_PCT}%" + +if [ "$coldpct" -gt "$COLD_ALERT_PCT" ]; then + echo "::error title=Reviewer cache regressed::${coldpct}% of the last ${scanned} reviewer runs were cold (> ${COLD_ALERT_PCT}%). Likely cause: a rules edit, an action/CLI bump, a model change, or the 1h TTL being lost." + exit 1 +fi +echo "Reviewer cache healthy: ${coldpct}% cold over ${scanned} runs." diff --git a/.github/scripts/emit-cache-usage.sh b/.github/scripts/emit-cache-usage.sh new file mode 100755 index 000000000000..82a2f1866b1f --- /dev/null +++ b/.github/scripts/emit-cache-usage.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# In-run, minimal: emit the reviewer's prompt-cache token counts to the run log. +# +# Token COUNTS ONLY (no message content) -> safe in public logs. Source is the +# action's execution file (always written, independent of `show_full_output`). +# +# Analysis, thresholds and alerting live OUT of the review hot path: the +# scheduled collector (.github/workflows/reviewer-cache-monitor.yml) scans these +# lines across recent runs to detect caching regressions over time. +set -euo pipefail + +f="${EXEC_FILE:-}" +if [ -z "$f" ] || [ ! -s "$f" ]; then + echo "emit-cache-usage: no execution file, skipping" + exit 0 +fi + +jq -r ' + [.[] | select(.type=="assistant") | .message.usage // empty] as $u + | (($u | map(.cache_creation_input_tokens // 0) | add) // 0) as $c + | (($u | map(.cache_read_input_tokens // 0) | add) // 0) as $r + | (([.[] | select(.type=="result")] | last | .total_cost_usd) // 0) as $cost + | "CACHE_USAGE creation=\($c) read=\($r) cost=\($cost)"' "$f" diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index 4cfc0f46fcf2..efd3fd1b06e3 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -110,6 +110,15 @@ jobs: STRUCTURED_OUTPUT: ${{ steps.code-review.outputs.structured_output }} run: postCodeReviewResults.sh "$PR_NUMBER" + # Monitoring (emit only): print prompt-cache token counts to the log so the + # out-of-band collector (.github/workflows/reviewer-cache-monitor.yml) can + # track cache health across runs. Counts only - no message content. + - name: Emit reviewer cache usage + if: always() && steps.code-review.outputs.execution_file != '' + env: + EXEC_FILE: ${{ steps.code-review.outputs.execution_file }} + run: .github/scripts/emit-cache-usage.sh + - name: Run Claude Code (docs) if: steps.set-authorized.outputs.IS_AUTHORIZED == 'true' && steps.filter.outputs.docs == 'true' uses: anthropics/claude-code-action@4d7e1f0cd85743fdc93b1c8040ab54395da024e2 # v1.0.149 diff --git a/.github/workflows/reviewer-cache-monitor.yml b/.github/workflows/reviewer-cache-monitor.yml new file mode 100644 index 000000000000..2de9fe5886bc --- /dev/null +++ b/.github/workflows/reviewer-cache-monitor.yml @@ -0,0 +1,39 @@ +name: Reviewer cache monitor + +# Out-of-band collector for reviewer prompt-cache health. Scans recent +# "PR Reviews with Claude Code" runs' logs for the CACHE_USAGE line and fails +# (a red scheduled run = built-in alert) if caching has regressed across a +# meaningful sample. Reads logs only - no message content, no artifacts. + +on: + schedule: + - cron: "23 * * * *" # hourly + workflow_dispatch: + inputs: + lookback: + description: How many recent reviewer runs to scan + required: false + default: "30" + +permissions: + contents: read + actions: read + +concurrency: + group: reviewer-cache-monitor + cancel-in-progress: true + +jobs: + collect: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Scan reviewer runs for cache health + env: + GH_TOKEN: ${{ github.token }} + LOOKBACK: ${{ github.event.inputs.lookback || '30' }} + run: .github/scripts/collect-cache-health.sh