From 7e8f8c83ac5ca7af1d242181ef4d8ac75b4ecfaf Mon Sep 17 00:00:00 2001 From: Ole Bueker Date: Tue, 7 Jul 2026 14:24:57 +0200 Subject: [PATCH] ci(fuzz): drop sccache from Fuzz Smoke (Windows server crashes on GNS TUs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fuzz Smoke has failed every scheduled run since 2026-07-03 with the identical failure signature the Windows ASan job hit before d93b5af5 dropped sccache there: the Windows sccache server crashes mid-build on GameNetworkingSockets' largest TUs (steamnetworkingsockets_shared/certs/certstore.cpp) with "An existing connection was forcibly closed by the remote host (os error 10054)". A local clang-cl build of GNS with no sccache completes fine, so sccache is the failing component, not our code. Mirror the proven fix: build both matrix legs (asan, ubsan) without the sccache launcher — direct compile, full cold build every run. Fixes #573. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/fuzz.yml | 60 ++++++++++++++------------------------ 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 6beef24fa..f66a2544f 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -22,19 +22,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} -env: - # Route sccache at its GitHub Actions cache backend (provisioned by the - # mozilla-actions/sccache-action step in each matrix leg below). Object files - # are reused across runs, so a nightly fuzz build only recompiles the TUs that - # actually changed instead of building the whole sanitised engine + harnesses - # cold every night. Mirrors the Windows.yml (#319) / asan.yml (#358) pattern. - # The clang-cl Ninja Multi-Config generator here honors - # CMAKE__COMPILER_LAUNCHER, so the launcher engages with no extra switch. - # The -fsanitize=fuzzer,address|undefined flags are part of each compile - # command line (and so part of sccache's hash), so the asan and ubsan matrix - # legs are distinct cache entries — a single shared backend cannot collide. - SCCACHE_GHA_ENABLED: "true" - jobs: fuzz: # Two parallel jobs — `asan` and `ubsan` — because compiler-rt's Windows @@ -103,16 +90,21 @@ jobs: version: "20.1.8" cached: true - # Provisions sccache and wires it to the GitHub Actions cache backend - # (SCCACHE_GHA_ENABLED above). Must run before Configure so the launcher - # resolves on PATH. Action v0.0.10 + sccache >= v0.10.0 are REQUIRED: - # GitHub decommissioned the legacy Actions cache API on 2025-04-15 and - # older sccache builds hit the dead endpoint and fail to start (and, as the - # compiler launcher, would break every compile). v0.15.0 matches Windows.yml. - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - with: - version: "v0.15.0" + # NOTE: this job deliberately does NOT use sccache (unlike Windows.yml). + # The Windows sccache server reliably crashes — "sccache: error: failed to + # execute compile … An existing connection was forcibly closed by the + # remote host. (os error 10054)" — while compiling the largest + # GameNetworkingSockets TUs (steamnetworkingsockets_shared.cpp, + # _certs.cpp, _certstore.cpp). This is the exact same failure signature + # hit by the Windows ASan job (asan.yml) before it dropped sccache in + # d93b5af5 — a local clang-cl build of GNS with no sccache completes + # fine, so sccache is the failing component. Building WITHOUT the + # sccache launcher (compiler runs directly) removes the crash entirely. + # The cost is a full cold compile every run, acceptable for a nightly + # job that otherwise never goes green. If a future sccache release + # fixes the Windows large-object crash, restore the Setup sccache step, + # the SCCACHE_GHA_ENABLED env var, -DOLO_ENABLE_COMPILER_CACHE=ON below, + # and the "Show sccache stats" step. - name: Configure CMake (standalone clang-cl via Ninja Multi-Config) # We replicate the `clangcl` CMake preset's settings inline instead @@ -152,17 +144,17 @@ jobs: -DOLO_ENABLE_FUZZING=ON ` -DOLO_FUZZ_SANITIZER=${{ matrix.sanitizer }} ` -DOLO_VIDEO_FFMPEG=OFF ` - -DOLO_ENABLE_COMPILER_CACHE=ON ` -DOLO_ENABLE_LTO=OFF - # -DOLO_ENABLE_COMPILER_CACHE=ON: wires sccache as the compiler - # launcher (cmake/CompilerCache.cmake). The Ninja generator honors - # CMAKE__COMPILER_LAUNCHER, so clang-cl compiles route through - # sccache; it also force-disables PCH and unity (both non-cacheable) - # and switches MSVC-style debug info to /Z7 so objects are cacheable. + # OLO_ENABLE_COMPILER_CACHE is intentionally OMITTED here — see the + # note above the (removed) sccache step: the Windows sccache server + # crashes on the largest GameNetworkingSockets TUs, so this job + # compiles directly. Debug info stays at its default, which is fine + # without a cache to key on. # -DOLO_ENABLE_LTO=OFF: LTO defaults ON, but a fuzz harness is never # shipped — its codegen quality is irrelevant, and ThinLTO would add a # slow whole-program link for no benefit. Dropping it also keeps every - # object a pure function of its TU (cacheable). Mirrors Windows.yml. + # object a pure function of its TU (cacheable, though nothing caches + # it here). Mirrors Windows.yml. - name: Build fuzzers # OloEngine + fuzz harnesses are sanitised (see cmake/Fuzzing.cmake); @@ -174,14 +166,6 @@ jobs: # step below handles the actual harness execution. run: cmake --build build-clang --target OloEngine-Fuzz --config Release --parallel - # Read the cache hit rate. A cold cache (first run / after a real source - # change) is mostly misses + writes; a warm cache shows a high hit rate, - # meaning the build step recompiled almost nothing. Two consecutive nightly - # runs of the same matrix leg are the cold-vs-warm proof. - - name: Show sccache stats - if: always() - run: sccache --show-stats - - name: Run fuzz smoke (per-target 30s) working-directory: OloEditor shell: pwsh