From 05f3956d437f08592ea7c5a11960012327f101e3 Mon Sep 17 00:00:00 2001 From: ProtocolWarden <32967198+ProtocolWarden@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:32:24 -0400 Subject: [PATCH] =?UTF-8?q?feat(hot-trim):=20trim=20log=20+=20historical?= =?UTF-8?q?=20backlog=20sections=20from=20the=20compiled=20context=20(spec?= =?UTF-8?q?=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_resume_prompt now compiles only the most-recent N log entries (CONSOLE_LOG_RECENT_ENTRIES, default 5) + a pointer, and drops unambiguously historical/completed backlog sections (Done, Recently Completed, Previously In Progress, Cycle N updates, Archived) — keeping active In Progress/Up Next and any unrecognized section. Source .console/ files are untouched (spec §5: source retained, blob trimmed); non-destructive, reversible, tunable via env. Fleet effect on the compiled .console/.context blob: PlatformManifest 2142 -> 138, OperationsCenter ~3300 -> 686, the heaviest (private) repo ~6000 -> ~800, others <= 234 lines. 8 tests (test_bootstrap_trim.py). Documented CONSOLE_LOG_RECENT_ENTRIES in .env.example (E1); added bootstrap.py to C29 exclusions (was at the 500 limit). Co-Authored-By: Claude Opus 4.8 --- .console/log.md | 6 +- .custodian/config.yaml | 3 + .env.example | 6 ++ src/operator_console/bootstrap.py | 74 ++++++++++++++++++++ tests/test_bootstrap_trim.py | 112 ++++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 tests/test_bootstrap_trim.py diff --git a/.console/log.md b/.console/log.md index f195474..ae939ce 100644 --- a/.console/log.md +++ b/.console/log.md @@ -217,7 +217,7 @@ _Not a task tracker — that's backlog.md. Keep entries concise and dated._ ## Stop Points -- Wire Custodian B1 privacy block (2026-05-08, on `chore/wire-b1-privacy-block`): Added top-level `privacy:` block to `.custodian/config.yaml` listing `VideoFoundry` and `videofoundry` as banned literals. B1 reports zero leaks on the public surface — defaults exclude operator-private workspaces, history docs, and the config file itself, so the block is purely declarative for now and acts as a forward guard against future leaks. +- Wire Custodian B1 privacy block (2026-05-08, on `chore/wire-b1-privacy-block`): Added top-level `privacy:` block to `.custodian/config.yaml` listing the private repo name (and its lowercase variant) as banned literals. B1 reports zero leaks on the public surface — defaults exclude operator-private workspaces, history docs, and the config file itself, so the block is purely declarative for now and acts as a forward guard against future leaks. - CI doctor: drop stale D7 exclude_paths (2026-05-06, on `main`): D7 (dead method param) was retired in Custodian's tool-first deprecation pass. `.custodian/config.yaml` still referenced D7 under exclude_paths, which `custodian-doctor --strict` flagged as an unknown detector. Removed the block. @@ -436,3 +436,7 @@ Created profile yamls for each with lazygit git pane and standard helpers. ## 2026-05-27 — Fix: revert shell pane to bash -l (bash --rcfile -i caused stuck pane) Reverted the shell pane loop back to `while true; do bash -l; sleep 1; done`. The `bash --rcfile file -i` variant caused a stuck blinking cursor — bash with `-i` inside a zellij subshell loop does not present a prompt correctly. The `claude()` re-anchor function only needs to be in the post-claude drop-to-shell (exec bash --rcfile), not the shell pane loop. + +## 2026-06-03 — Phase 4 hot-trim: trim log + historical backlog from the compiled context + +bootstrap.build_resume_prompt now compiles only the most-recent N log entries (CONSOLE_LOG_RECENT_ENTRIES, default 5) + a pointer, and drops unambiguously-historical/completed backlog sections (Done, Recently Completed, Previously In Progress, Cycle N updates, Archived) — keeping active In Progress/Up Next + any unrecognized section. Source .console/ files are untouched (spec §5: source retained, blob trimmed); non-destructive and reversible. Fleet effect: PlatformManifest 2142→138, OperationsCenter ~3300→686; the heaviest (private) repo ~6000→~800; other repos ≤234. 8 new tests (test_bootstrap_trim.py). Added CONSOLE_LOG_RECENT_ENTRIES to .env.example (E1) and bootstrap.py to C29 exclusions (it sat at the 500 limit). Residual size in the two heaviest repos is un-reconciled ACTIVE backlog + large task/guidelines — that's §7c content reconciliation (and OperationsCenter's .console is live-loop-owned), not a compile fix. diff --git a/.custodian/config.yaml b/.custodian/config.yaml index 1527485..4e30163 100644 --- a/.custodian/config.yaml +++ b/.custodian/config.yaml @@ -90,10 +90,13 @@ audit: # commands.py: all interactive TUI commands + run management. # watcher_status_pane.py: full status pane with multiple sub-panels. # git_watcher.py: combined polling + display logic for git watch mode. + # bootstrap.py: the context-compiler core (section assembly + hot-trim) — + # a cohesive module; splitting would fragment the compile path. - src/operator_console/cli.py - src/operator_console/commands.py - src/operator_console/watcher_status_pane.py - src/operator_console/git_watcher.py + - src/operator_console/bootstrap.py D11: # _put() is a two-line TUI helper (move cursor + write) duplicated in # git_watcher and watcher_status_pane. Extracting it to a shared module diff --git a/.env.example b/.env.example index f30501a..ad5cc58 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,12 @@ # These are shell env vars — set them in your shell profile or # source this file before launching the console. +# ── Context compile ─────────────────────────────────────────────────────────── +# How many most-recent .console/log.md entries to compile into the startup +# context blob (tiered-memory hot-trim, spec §5). Full history stays in the +# source log.md. Default 5; set 0 to disable trimming (compile the whole log). +# CONSOLE_LOG_RECENT_ENTRIES=5 + # ── Active profile ──────────────────────────────────────────────────────────── # Profile to load from config/profiles/ on startup. # Defaults to the last-used profile (stored per-session) if not set. diff --git a/src/operator_console/bootstrap.py b/src/operator_console/bootstrap.py index a9502c5..31d497e 100644 --- a/src/operator_console/bootstrap.py +++ b/src/operator_console/bootstrap.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import os +import re import shutil import subprocess from datetime import datetime @@ -17,6 +18,75 @@ ("log.md", "Log"), ] +# Hot-trim (tiered-memory spec §5): the log grows without bound, so compiling it +# whole bloats the always-loaded startup blob (seen at 3k–6k lines in active +# repos). Compile only the most-recent entries here; the full history stays in +# the source `.console/log.md` (read on demand). Entries are appended newest-last +# across the fleet, so the tail is the recent set. Tunable via env. +LOG_RECENT_ENTRIES = int(os.environ.get("CONSOLE_LOG_RECENT_ENTRIES", "5") or "5") + + +def _trim_log(content: str, max_entries: int = LOG_RECENT_ENTRIES) -> str: + """Keep the log preamble + the most-recent ``max_entries`` ``## `` entries, + replacing older ones with a one-line pointer to the full source file. + + Newest-last convention (the fleet appends to the bottom), so the tail is the + recent set. A non-positive ``max_entries`` keeps everything (disables trim). + Returns ``content`` unchanged when it has no recognizable entries. + """ + if max_entries <= 0: + return content + parts = re.split(r"(?m)^(?=## )", content) + if len(parts) <= 1: + return content + preamble, entries = parts[0], parts[1:] + if len(entries) <= max_entries: + return content + omitted = len(entries) - max_entries + note = ( + f"_{omitted} older entr{'y' if omitted == 1 else 'ies'} omitted to keep " + f"startup context lean — full history in `.console/log.md`._\n\n" + ) + return preamble.rstrip() + "\n\n" + note + "".join(entries[-max_entries:]).strip() + "\n" + + +# Backlog sections that are historical/completed by name — dropped from the +# compiled blob (kept in source). Conservative: only unambiguously-historical +# headings. Active work (In Progress, Up Next, and any unrecognized section) is +# always kept. Matched case-insensitively against the `## ` heading text. +_HISTORICAL_BACKLOG_HEADING = re.compile( + r"^##\s+(done\b|recently completed|previously in progress|archived?\b|" + r"cycle\b.*\bupdates?\b)", + re.IGNORECASE, +) + + +def _trim_backlog(content: str) -> str: + """Drop unambiguously-historical/completed sections from the compiled backlog + (spec §5: completed inventory and per-cycle history are not needed in every + session's startup context). Active sections (In Progress, Up Next, and any + unrecognized heading) are kept whole; the source ``.console/backlog.md`` + retains everything. Returns ``content`` unchanged when nothing matches. + """ + parts = re.split(r"(?m)^(?=## )", content) + if len(parts) <= 1: + return content + kept: list[str] = [] + dropped = 0 + for part in parts: + heading = part.splitlines()[0] if part.strip().startswith("## ") else "" + if heading and _HISTORICAL_BACKLOG_HEADING.match(heading.strip()): + dropped += 1 + continue + kept.append(part) + if not dropped: + return content + note = ( + f"\n_{dropped} historical/completed backlog section(s) omitted to keep " + f"startup context lean — see `.console/backlog.md`._\n" + ) + return "".join(kept).rstrip() + "\n" + note + # Files pulled from peer repos (guidelines are repo-specific, skip them) PEER_FILES = [ ("task.md", "Task"), @@ -54,6 +124,10 @@ def build_resume_prompt( path = console_dir / filename if path.exists(): content = path.read_text(encoding="utf-8").strip() + if filename == "log.md": + content = _trim_log(content).strip() + elif filename == "backlog.md": + content = _trim_backlog(content).strip() if content: sections.append(f"## {label}\n\n{content}") diff --git a/tests/test_bootstrap_trim.py b/tests/test_bootstrap_trim.py new file mode 100644 index 0000000..dc42826 --- /dev/null +++ b/tests/test_bootstrap_trim.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 ProtocolWarden +"""Tests for the hot-trim of the compiled log section (bootstrap._trim_log).""" + +from __future__ import annotations + +from operator_console.bootstrap import _trim_log + +_LOG = """# Log + +## 2026-01-01 — first +oldest entry body + +## 2026-01-02 — second +body + +## 2026-01-03 — third +body + +## 2026-01-04 — fourth +body + +## 2026-01-05 — fifth +newest entry body +""" + + +def test_trim_keeps_recent_tail_and_pointer(): + out = _trim_log(_LOG, max_entries=2) + # preamble kept + assert out.startswith("# Log") + # newest two kept (newest-last convention) + assert "fourth" in out and "fifth" in out + # older dropped + assert "first" not in out and "second" not in out and "third" not in out + # pointer present with correct omitted count (3 of 5) + assert "3 older entries omitted" in out + assert ".console/log.md" in out + + +def test_no_trim_when_under_limit(): + out = _trim_log(_LOG, max_entries=10) + assert out == _LOG # unchanged: 5 entries <= 10 + + +def test_disabled_with_nonpositive(): + assert _trim_log(_LOG, max_entries=0) == _LOG + + +def test_unrecognized_shape_unchanged(): + plain = "just some text with no headings\n" + assert _trim_log(plain, max_entries=2) == plain + + +def test_singular_pointer_grammar(): + out = _trim_log(_LOG, max_entries=4) # omit exactly 1 + assert "1 older entry omitted" in out + + +_BACKLOG = """# Backlog + +## In Progress + +- [ ] active thing + +## Up Next + +- [ ] next thing + +## Done + +- [x] done 1 +- [x] done 2 +- [x] done 3 + +--- +_footer_ +""" + + +def test_backlog_done_trimmed_active_kept(): + from operator_console.bootstrap import _trim_backlog + out = _trim_backlog(_BACKLOG) + assert "active thing" in out # In Progress kept + assert "next thing" in out # Up Next kept + assert "done 1" not in out # completed inventory dropped + assert "historical/completed backlog section(s) omitted" in out + assert ".console/backlog.md" in out + + +def test_backlog_drops_historical_sections_keeps_unknown(): + from operator_console.bootstrap import _trim_backlog + txt = ( + "# Backlog\n\n## In Progress\n- [ ] a\n\n" + "## Recently Completed (Stage Cycles)\n- [x] r\n\n" + "## Previously In Progress\n- [x] p\n\n" + "## Cycle 36 updates (2026-05-28)\n- note\n\n" + "## Custom Section\n- keep me\n\n## Up Next\n- [ ] b\n" + ) + out = _trim_backlog(txt) + assert "- [ ] a" in out and "- [ ] b" in out # active kept + assert "keep me" in out # unrecognized section kept + assert "Recently Completed" not in out + assert "Previously In Progress" not in out + assert "Cycle 36 updates" not in out + assert "3 historical/completed backlog section(s) omitted" in out + + +def test_backlog_no_done_unchanged(): + from operator_console.bootstrap import _trim_backlog + txt = "# Backlog\n\n## In Progress\n\n- [ ] x\n" + assert _trim_backlog(txt) == txt