From d7e42e228f418066a27412e4b1b0578907627073 Mon Sep 17 00:00:00 2001 From: Christian Boos Date: Thu, 25 Jun 2026 10:17:15 +0200 Subject: [PATCH 1/3] test: shrink oversized transcripts in integration fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The realistic-integration tests exercise behavior (hierarchy processing, caching, CLI flags, regeneration), not raw data volume, yet rendering cost scales with transcript size. Two fixture session files are ~5 MB each and get re-rendered in every function-scoped copy, dominating suite wall-time — on Windows especially, where the per-message render work (not copytree, which measured at ~0.15s) is the bottleneck. Prefix-truncate oversized session files in the *copy* only, leaving the source tree pristine so volume-sensitive tests (test_performance, test_json_real_projects, test_dag*) keep the full data. A leading prefix is DAG-safe: an entry's parentUuid always refers to an earlier line, so dropping the tail removes only leaf descendants. agent-/ subagents files and the teammates/JSSoundRecorder fixtures are left intact (the 600 KB threshold sits above their largest files). Before/after (Windows, this file, serial): 150.3s -> 93.2s (-38%) Full unit suite (-n auto --dist=worksteal): 185.7s -> 146.9s (-21%) Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_integration_realistic.py | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test/test_integration_realistic.py b/test/test_integration_realistic.py index 0cb9738b..6848fb30 100644 --- a/test/test_integration_realistic.py +++ b/test/test_integration_realistic.py @@ -36,6 +36,46 @@ # Path to realistic test data REAL_PROJECTS_DIR = Path(__file__).parent / "test_data" / "real_projects" +# These integration tests exercise *behavior* (hierarchy processing, caching, +# CLI flags, regeneration) rather than raw data volume, yet rendering cost +# scales with transcript size. A couple of session files in the fixture are +# ~5 MB each, and re-rendering them in every function-scoped copy dominates the +# suite wall-time (especially on Windows, where the per-message render work is +# the bottleneck). We therefore prefix-truncate the oversized session files in +# the *copy* only — the source tree stays pristine, so the volume-sensitive +# tests (test_performance / test_json_real_projects / test_dag*) keep the full +# data. Truncating to a leading prefix is DAG-safe: a transcript entry's +# parentUuid always refers to an earlier line, so dropping the tail only removes +# leaf descendants and never dangles a parent reference. +# +# The 600 KB threshold sits above the largest content-sensitive fixture files +# (JSSoundRecorder's 506 KB session, the teammates trunk at 285 KB), so those +# projects are left intact without naming them. agent-*/subagents files are +# skipped outright — the teammate-linking tests depend on their full content. +_TRUNCATE_THRESHOLD = 600_000 +_TRUNCATE_KEEP = 100_000 + + +def _shrink_large_transcripts(projects_dir: Path) -> None: + """Prefix-truncate oversized standalone transcript files in a copied tree. + + See the module comment above for the rationale and safety argument. Mutates + files in place; only call on a throwaway copy, never the source fixture. + """ + for jsonl_file in projects_dir.rglob("*.jsonl"): + if jsonl_file.name.startswith("agent-") or "subagents" in jsonl_file.parts: + continue + if jsonl_file.stat().st_size <= _TRUNCATE_THRESHOLD: + continue + data = jsonl_file.read_bytes() + cut = data[:_TRUNCATE_KEEP] + # Trim back to the last complete line so we never leave a partial JSON + # record (which would be parsed as a malformed line, not a clean prefix). + last_newline = cut.rfind(b"\n") + if last_newline > 0: + cut = cut[: last_newline + 1] + jsonl_file.write_bytes(cut) + def make_valid_user_entry( content: str, @@ -105,6 +145,7 @@ def temp_projects_copy(real_projects_path: Path) -> Generator[Path, None, None]: with tempfile.TemporaryDirectory() as temp_dir: temp_projects = Path(temp_dir) / "projects" shutil.copytree(real_projects_path, temp_projects) + _shrink_large_transcripts(temp_projects) # Clean any existing cache/HTML to ensure fresh state for project_dir in temp_projects.iterdir(): @@ -130,6 +171,7 @@ def projects_with_cache(real_projects_path: Path) -> Generator[Path, None, None] with tempfile.TemporaryDirectory() as temp_dir: temp_projects = Path(temp_dir) / "projects" shutil.copytree(real_projects_path, temp_projects) + _shrink_large_transcripts(temp_projects) # Clean any existing cache/HTML first for project_dir in temp_projects.iterdir(): From f3bc27c3123374d04ca889c22a3853951b0022cd Mon Sep 17 00:00:00 2001 From: Christian Boos Date: Thu, 25 Jun 2026 10:17:48 +0200 Subject: [PATCH 2/3] test: skip symlink-based aggregate tests when symlinks unavailable TestIndexInlineAggregateLoopCharacterization creates a symlink to discover a synthetic project. On Windows without elevation/Developer Mode, os.symlink raises OSError [WinError 1314], failing all three tests. Probe symlink capability once at import and skipif the class, so the suite is green on stock Windows while still running the tests everywhere symlinks work (Linux/macOS/elevated Windows). Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_session_scan_characterization.py | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/test_session_scan_characterization.py b/test/test_session_scan_characterization.py index 41e64eee..fbebd77a 100644 --- a/test/test_session_scan_characterization.py +++ b/test/test_session_scan_characterization.py @@ -51,6 +51,27 @@ ) +def _symlinks_supported() -> bool: + """Whether the OS lets this process create symlinks. + + On Windows, ``os.symlink`` raises ``OSError [WinError 1314]`` unless the + process is elevated or Developer Mode is on, so the symlink-based tests + below can't run there. Probe once at import time. + """ + import tempfile + + with tempfile.TemporaryDirectory() as td: + link = Path(td) / "probe-link" + try: + link.symlink_to(Path(td)) + except (OSError, NotImplementedError): + return False + return True + + +_SYMLINKS_SUPPORTED = _symlinks_supported() + + # ----- fixture builders ---------------------------------------------------- @@ -457,6 +478,10 @@ def test_d1_d2_fixture_cache_equals_fallback( # ----- characterization: index inline-aggregate loop ----------------------- +@pytest.mark.skipif( + not _SYMLINKS_SUPPORTED, + reason="symlink creation requires privilege/Developer Mode on Windows", +) class TestIndexInlineAggregateLoopCharacterization: """Pin the project-aggregate totals produced by the inline loop inside `process_projects_hierarchy` (the cache-unavailable From 9014816cb20b3253139c668128c57badd1c67679 Mon Sep 17 00:00:00 2001 From: Christian Boos Date: Mon, 29 Jun 2026 23:10:08 +0200 Subject: [PATCH 3/3] test: leave file untruncated when prefix has no newline In _shrink_large_transcripts, if the kept prefix contains no newline (the first record alone exceeds the keep size), skip truncating that file rather than writing a partial JSON record that would parse as malformed JSONL. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_integration_realistic.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_integration_realistic.py b/test/test_integration_realistic.py index 6848fb30..6cafb45c 100644 --- a/test/test_integration_realistic.py +++ b/test/test_integration_realistic.py @@ -71,9 +71,13 @@ def _shrink_large_transcripts(projects_dir: Path) -> None: cut = data[:_TRUNCATE_KEEP] # Trim back to the last complete line so we never leave a partial JSON # record (which would be parsed as a malformed line, not a clean prefix). + # If the kept prefix has no newline (the first record alone exceeds + # _TRUNCATE_KEEP), leave the file untruncated rather than writing a + # partial record. last_newline = cut.rfind(b"\n") - if last_newline > 0: - cut = cut[: last_newline + 1] + if last_newline <= 0: + continue + cut = cut[: last_newline + 1] jsonl_file.write_bytes(cut)