Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions .github/workflows/quality-bench-monthly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
name: "⚑ Quality: Benchmarks (monthly)"
run-name: >-
${{
github.event_name == 'schedule' && '⚑ Quality: Benchmarks β€” Monthly schedule' ||
format('⚑ Quality: Benchmarks β€” Monthly, manual by {0}', github.actor)
}}

# Monthly performance-regression signal. Runs every Go benchmark in the tree
# (auth middleware, Argon2id verify, client-IP extraction, the PHC / image-ref /
# label / trusted-proxy parsers, and the MCP dispatch path) five times so the
# numbers are stable enough to diff month over month. Deep fuzzing lives in
# `quality-fuzz-monthly.yml`; this workflow answers the separate question of
# whether the hot paths got slower or started allocating more. Mirrors
# sockguard's monthly benchmark tier; same rationale.
#
# Results upload as an artifact (90-day retention) and the top lines are echoed
# into the run summary so a regression is visible without downloading anything.

on:
workflow_dispatch:
schedule:
- cron: '45 7 1 * *' # Monthly on day 1 at 07:45 UTC (between mutation 06:30 and deep fuzz 08:30)

permissions:
contents: read

concurrency:
group: quality-bench-monthly-${{ github.workflow }}
cancel-in-progress: true

jobs:
benchmarks:
name: "⚑ Go benchmarks (hot paths)"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Harden Runner
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
with:
egress-policy: audit

- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Setup Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: "1.26"

- name: Run benchmarks
run: |
# -run='^$' skips the unit tests (CI already gates those) so the job
# spends its time only on benchmarks. -count=5 gives benchstat-ready
# samples; -benchmem tracks allocations alongside ns/op.
go test -run='^$' -bench=. -benchmem -count=5 -timeout=20m ./... \
| tee benchmark-results.txt

- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: benchmark-results-${{ github.run_id }}
path: benchmark-results.txt
retention-days: 90

- name: Summarize
if: always()
run: |
{
echo "### Go benchmarks (monthly)"
echo ""
echo '```'
grep -E '^Benchmark' benchmark-results.txt | head -60 || echo "No benchmark results captured."
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
172 changes: 172 additions & 0 deletions .github/workflows/quality-fuzz-monthly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
name: "πŸ”€ Quality: Deep Fuzz (monthly)"
run-name: >-
${{
github.event_name == 'schedule' && 'πŸ”€ Quality: Deep Fuzz β€” Monthly schedule' ||
format('πŸ”€ Quality: Deep Fuzz β€” Monthly, manual by {0}', github.actor)
}}

# Tier 3 of the fuzz strategy. Tier 1 lives in `ci.yml go-fuzz` (60-second
# smoke per PR/push), Tier 2 in `quality-fuzz-nightly.yml` (5 minutes per
# fuzzer daily). This workflow gives every fuzzer a 1-hour coverage budget on
# the first day of each month so we get one very deep pass that can reach paths
# the shorter tiers miss, plus on-demand longer runs via `workflow_dispatch`
# before a release. Mirrors sockguard's monthly tier; same rationale.
#
# 1 hour per fuzzer Γ— 5 fuzzers Γ— matrix-parallel = ~1 hour wall time. Crashes
# upload with 180-day retention so a failing input from an older monthly run
# stays recoverable long enough to turn into a committed regression test.

on:
workflow_dispatch:
inputs:
fuzztime:
description: "Per-fuzzer coverage budget (Go duration, e.g. 1h, 3h). Max β‰ˆ5h45m before the job timeout trims it."
required: false
default: "1h"
schedule:
- cron: '30 8 1 * *' # Monthly on day 1 at 08:30 UTC (after mutation at 06:30)

permissions:
contents: read

concurrency:
# Don't cancel an in-flight monthly when a manual dispatch fires β€” they
# answer different questions and either can confirm health.
group: quality-fuzz-monthly-${{ github.workflow }}-${{ github.event.inputs.fuzztime || 'scheduled' }}
cancel-in-progress: false

jobs:
monthly-fuzz:
name: "πŸ”€ Fuzz ${{ matrix.fuzzer.name }} (monthly)"
runs-on: ubuntu-latest
# 6-hour ceiling is the hard cap regardless of the fuzztime input.
# Anything longer should run on a dedicated box, not a shared runner.
timeout-minutes: 360

strategy:
fail-fast: false
matrix:
fuzzer:
- { name: FuzzParsePHC, pkg: ./internal/server/ }
- { name: FuzzParseTrustedProxies, pkg: ./internal/server/ }
- { name: FuzzParseImageRef, pkg: ./internal/adapter/ }
- { name: FuzzParseLabels, pkg: ./internal/adapter/drydock/ }
- { name: FuzzMCPHandler, pkg: ./internal/mcp/ }

steps:
- name: Harden Runner
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
with:
egress-policy: audit

- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Setup Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: "1.26"

- name: Resolve fuzz budget
id: budget
env:
INPUT_FUZZTIME: ${{ github.event.inputs.fuzztime }}
run: |
FUZZTIME="${INPUT_FUZZTIME:-1h}"
echo "fuzztime=${FUZZTIME}" >> "$GITHUB_OUTPUT"
# Parse the trailing unit (h, m, or s) and convert to seconds, then add
# a 120-second cushion so the go test -timeout always outlasts -fuzztime.
if [[ "${FUZZTIME}" =~ ^([0-9]+)h$ ]]; then
budget_s=$(( BASH_REMATCH[1] * 3600 ))
elif [[ "${FUZZTIME}" =~ ^([0-9]+)m$ ]]; then
budget_s=$(( BASH_REMATCH[1] * 60 ))
elif [[ "${FUZZTIME}" =~ ^([0-9]+)s$ ]]; then
budget_s=${BASH_REMATCH[1]}
else
budget_s=3600
fi
echo "test_timeout=$(( budget_s + 120 ))s" >> "$GITHUB_OUTPUT"

- name: Fuzz ${{ matrix.fuzzer.name }}
id: fuzz
env:
FUZZER: ${{ matrix.fuzzer.name }}
PKG: ${{ matrix.fuzzer.pkg }}
FUZZTIME: ${{ steps.budget.outputs.fuzztime }}
TEST_TIMEOUT: ${{ steps.budget.outputs.test_timeout }}
run: |
LOG="${RUNNER_TEMP}/fuzz-${FUZZER}.log"

run_fuzz() {
go test -run='^$' \
-fuzz="^${FUZZER}\$" \
-fuzztime="${FUZZTIME}" \
-timeout="${TEST_TIMEOUT}" \
"${PKG}" 2>&1 | tee "${LOG}"
return "${PIPESTATUS[0]}"
}

emit() { echo "kind=$1" >> "$GITHUB_OUTPUT"; }

for attempt in 1 2; do
rc=0
run_fuzz || rc=$?

if [ "${rc}" -eq 0 ]; then
emit pass
exit 0
fi

if grep -q "Failing input written to testdata" "${LOG}"; then
emit crash
echo "::error::${FUZZER} found a crashing input β€” commit it to the seed corpus and fix the regression."
exit "${rc}"
fi

if ! grep -q "context deadline exceeded" "${LOG}"; then
emit error
echo "::error::${FUZZER} failed for a non-flake reason (exit ${rc})."
exit "${rc}"
fi

echo "::warning::${FUZZER}: known -fuzztime boundary flake on attempt ${attempt}/2."
done

emit flake
echo "::error::${FUZZER} hit the boundary flake on both attempts."
exit 1

- name: Upload fuzz corpus on failure or cancel
if: failure() || cancelled()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: fuzz-corpus-${{ matrix.fuzzer.name }}-${{ github.run_id }}
path: "**/testdata/fuzz/${{ matrix.fuzzer.name }}/"
retention-days: 180
if-no-files-found: ignore

- name: Summarize
if: always()
env:
FUZZER: ${{ matrix.fuzzer.name }}
PKG: ${{ matrix.fuzzer.pkg }}
FUZZTIME: ${{ steps.budget.outputs.fuzztime }}
STATUS: ${{ job.status }}
KIND: ${{ steps.fuzz.outputs.kind }}
RUN_ID: ${{ github.run_id }}
run: |
{
echo "### ${FUZZER} (monthly deep)"
echo ""
echo "- Package: \`${PKG}\`"
echo "- Budget: ${FUZZTIME}"
echo "- Result: ${STATUS}"
if [ "${KIND}" = "crash" ]; then
echo ""
echo "A crashing input was saved to \`testdata/fuzz/${FUZZER}/\`."
echo "Download artifact \`fuzz-corpus-${FUZZER}-${RUN_ID}\`, commit the"
echo "minimized input to the seed corpus, fix the bug, and push."
fi
} >> "$GITHUB_STEP_SUMMARY"
136 changes: 136 additions & 0 deletions .github/workflows/quality-soak-weekly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
name: "⏱️ Quality: Soak"
run-name: >-
${{
github.event_name == 'schedule' && '⏱️ Quality: Soak β€” Weekly' ||
format('⏱️ Quality: Soak β€” Manual by {0}', github.actor)
}}

# RSS + thread-drift soak. Runs the Portwing agent (generic adapter) in front of
# a mock Docker daemon under a sustained loadgen mix β€” cached-inventory reads,
# version/info, a raw Docker proxy read, and a stream of SSE subscribers that
# connect/hold/disconnect (the leak-prone path) β€” and asserts the agent's
# working-set growth stays inside a configured threshold. This is the
# long-lived-agent leak signal the unit/integration/fuzz tiers can't give.
#
# GitHub-hosted runners cap a single job at 6h, so the scheduled run soaks for
# 4h β€” enough wall time that a per-request allocation/goroutine leak shows up as
# multiple-MiB RSS growth well above the 64 MiB threshold. The 24h target lives
# on once a self-hosted runner is wired up; the manual dispatch inputs let a
# maintainer override the duration for that, or shorten it for a one-off check.

on:
workflow_dispatch:
inputs:
duration:
description: "Soak duration (Go time.Duration; e.g. 30m, 4h)"
required: false
default: "4h"
concurrency:
description: "Concurrent loadgen workers for the inventory scenario"
required: false
default: "20"
rss_growth_threshold_bytes:
description: "Fail if VmRSS grows by more than this many bytes from the post-warmup baseline"
required: false
default: "67108864"
schedule:
- cron: '15 6 * * 0' # Sundays 06:15 UTC

permissions:
contents: read

concurrency:
group: quality-soak-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
soak:
name: "⏱️ Soak (portwing RSS + thread drift)"
runs-on: ubuntu-latest
# Allow 30 minutes for build + warmup + post-run reporting on top of the
# 4-hour soak; well under the 6-hour github-hosted ceiling.
timeout-minutes: 270

steps:
- name: Harden Runner
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
with:
egress-policy: audit

- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Setup Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: "1.26"

- name: Resolve soak parameters
id: params
env:
INPUT_DURATION: ${{ github.event.inputs.duration }}
INPUT_CONCURRENCY: ${{ github.event.inputs.concurrency }}
INPUT_RSS_THRESHOLD: ${{ github.event.inputs.rss_growth_threshold_bytes }}
run: |
# Scheduled runs see empty inputs and fall back to these defaults.
{
echo "duration=${INPUT_DURATION:-4h}"
echo "concurrency=${INPUT_CONCURRENCY:-20}"
echo "rss_threshold=${INPUT_RSS_THRESHOLD:-67108864}"
} >> "$GITHUB_OUTPUT"

- name: Validate the soak script accepts the resolved parameters
env:
# Pull inputs through env vars so the shell never interpolates raw
# `${{ … }}` β€” closes zizmor's template-injection audit even though
# soak.sh re-validates downstream.
SOAK_DURATION: ${{ steps.params.outputs.duration }}
SOAK_CONCURRENCY: ${{ steps.params.outputs.concurrency }}
SOAK_RSS_THRESHOLD: ${{ steps.params.outputs.rss_threshold }}
run: |
scripts/soak.sh --dry-run \
--duration "${SOAK_DURATION}" \
--concurrency "${SOAK_CONCURRENCY}" \
--rss-growth-threshold-bytes "${SOAK_RSS_THRESHOLD}"

- name: Run soak
id: soak
env:
SOAK_DURATION: ${{ steps.params.outputs.duration }}
SOAK_CONCURRENCY: ${{ steps.params.outputs.concurrency }}
SOAK_RSS_THRESHOLD: ${{ steps.params.outputs.rss_threshold }}
run: |
scripts/soak.sh \
--duration "${SOAK_DURATION}" \
--concurrency "${SOAK_CONCURRENCY}" \
--rss-growth-threshold-bytes "${SOAK_RSS_THRESHOLD}" \
| tee soak-output.txt

- name: Upload soak output
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: soak-output-${{ github.run_id }}
path: soak-output.txt
retention-days: 90

- name: Summarize
if: always()
env:
SOAK_DURATION: ${{ steps.params.outputs.duration }}
SOAK_CONCURRENCY: ${{ steps.params.outputs.concurrency }}
SOAK_RSS_THRESHOLD: ${{ steps.params.outputs.rss_threshold }}
run: |
{
echo "### Portwing soak"
echo "- Duration: ${SOAK_DURATION}"
echo "- Concurrency: ${SOAK_CONCURRENCY}"
echo "- RSS threshold: ${SOAK_RSS_THRESHOLD} bytes"
echo ""
echo "Last 20 lines of soak output:"
echo '```'
tail -n 20 soak-output.txt 2>/dev/null || echo "(no output captured)"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
Loading
Loading