fix(mates): delta-based DM digests to prevent content loss past 6k chars

The original digest system collected the full conversation from index 0
on every turn, capped at 6000 chars. After ~5 turns, new content was
truncated and never reached the memory summary.

- Delta collection: only digest new turns since last successful checkpoint
- Per-session checkpoint persistence via digest_checkpoint history entries
  so the index survives server restarts
- Prior memory summary injected into digest worker prompt so Haiku has
  context for small delta fragments
- onDone(err) convention: checkpoint only advances on success (null err),
  preventing permanent data loss on Haiku failures
- Skip updateMemorySummary on parse_failed digests
- Reset digest index when history is trimmed (rewind)
- Filter digest_checkpoint entries from replayHistory

Author: leiyangyou

lib/project.js

@@ -2272,6 +2272,10 @@ function createProjectContext(opts) {
               }
               session.history = session.history.slice(0, trimTo);
               session.messageUUIDs = session.messageUUIDs.slice(0, targetIdx);
+              // Reset digest checkpoint if it points past the trimmed history
+              if (typeof session._dmLastDigestedIndex === "number" && session._dmLastDigestedIndex > trimTo) {
+                session._dmLastDigestedIndex = trimTo;
+              }
             }
 
             var kept = session.messageUUIDs;
@@ -3983,12 +3987,19 @@ function createProjectContext(opts) {
     } catch (e) {}
 
     // Combined gate + digest in one prompt (saves a full round-trip vs separate gate)
-    var prompt = [
+    var promptParts = [
       "[SYSTEM: Memory Gate + Digest]",
       "You are a memory system for an AI Mate (role: " + (mateRole || "assistant") + ").",
       "",
-      "Conversation (" + job.type + "):",
-      job.conversationContent,
+    ];
+    if (job.priorSummary) {
+      promptParts.push("Prior conversation context (memory summary so far):");
+      promptParts.push(job.priorSummary);
+      promptParts.push("");
+    }
+    promptParts.push("Conversation (" + job.type + "):");
+    promptParts.push(job.conversationContent);
+    var prompt = promptParts.concat([
       "",
       "STEP 1: Should this be saved to memory?",
       'Answer "no" ONLY if the entire conversation is trivial (e.g. just "hi"/"hello").',
@@ -4034,7 +4045,7 @@ function createProjectContext(opts) {
 
       if (digestObj && digestObj.skip) {
         console.log("[digest-worker] Gate declined for " + job.mateId);
-        if (job.onDone) job.onDone();
+        if (job.onDone) job.onDone(null);
         processDigestQueue();
         return;
       }
@@ -4072,9 +4083,14 @@ function createProjectContext(opts) {
         }
       }
 
-      updateMemorySummary(job.mateCtx, job.mateId, digestObj);
-      maybeSynthesizeUserProfile(job.mateCtx, job.mateId);
-      if (job.onDone) job.onDone();
+      // Skip summary update for failed parses — the fallback digest would degrade quality
+      if (digestObj.topic !== "parse_failed") {
+        updateMemorySummary(job.mateCtx, job.mateId, digestObj);
+        maybeSynthesizeUserProfile(job.mateCtx, job.mateId);
+        if (job.onDone) job.onDone(null);
+      } else {
+        if (job.onDone) job.onDone(new Error("parse_failed"));
+      }
       processDigestQueue();
     }
 
@@ -4096,7 +4112,7 @@ function createProjectContext(opts) {
           console.error("[digest-worker] Error:", err);
           _digestWorker = null;
           _digestWorkerTurns = 0;
-          if (job.onDone) job.onDone();
+          if (job.onDone) job.onDone(err);
           processDigestQueue();
         },
       });
@@ -4112,11 +4128,11 @@ function createProjectContext(opts) {
         onError: function (err) {
           console.error("[digest-worker] Create error:", err);
           _digestWorker = null;
-          if (job.onDone) job.onDone();
+          if (job.onDone) job.onDone(err);
           processDigestQueue();
         },
-      }).then(function (ws) { _digestWorker = ws; _digestWorkerTurns = 1; }).catch(function () {
-        if (job.onDone) job.onDone();
+      }).then(function (ws) { _digestWorker = ws; _digestWorkerTurns = 1; }).catch(function (err) {
+        if (job.onDone) job.onDone(err || new Error("digest worker creation failed"));
         processDigestQueue();
       });
     }
@@ -4153,19 +4169,37 @@ function createProjectContext(opts) {
     });
   }
 
-  // Digest DM turn for mate projects - uses shared digest worker
+  // Digest DM turn for mate projects - uses shared digest worker.
+  // Delta-based: only collects new turns since the last successful digest.
+  // Concurrency debounce: turns that arrive while a digest is in-flight
+  // are naturally batched into the next flush.
   var _dmDigestPending = false;
   function digestDmTurn(session, responsePreview) {
     if (!isMate || _dmDigestPending) return;
     var mateId = path.basename(cwd);
     var mateCtx = matesModule.buildMateCtx(projectOwnerId);
     if (!matesModule.isMate(mateCtx, mateId)) return;
 
-    // Collect full conversation from session history (all user + mate turns)
+    // Track digest index per session so switching sessions doesn't misalign.
+    // On resumed sessions (after restart), recover index from the last
+    // digest_checkpoint entry in history so undigested turns aren't lost.
+    if (typeof session._dmLastDigestedIndex !== "number") {
+      session._dmLastDigestedIndex = 0;
+      for (var ci = session.history.length - 1; ci >= 0; ci--) {
+        if (session.history[ci].type === "digest_checkpoint") {
+          session._dmLastDigestedIndex = session.history[ci].digestedIndex;
+          break;
+        }
+      }
+    }
+
+    // Collect only new turns since the last successful digest
     var conversationParts = [];
     var totalLen = 0;
     var CONV_CAP = 6000;
-    for (var hi = 0; hi < session.history.length; hi++) {
+    var startIndex = session._dmLastDigestedIndex;
+    for (var hi = startIndex; hi < session.history.length; hi++) {
+      if (totalLen >= CONV_CAP) break;
       var entry = session.history[hi];
       if (entry.type === "user_message" && entry.text) {
         var uText = entry.text;
@@ -4182,8 +4216,8 @@ function createProjectContext(opts) {
         conversationParts.push("Mate: " + aText);
         totalLen += aText.length;
       }
-      if (totalLen >= CONV_CAP) break;
     }
+    // If the latest response hasn't landed in history yet, append it
     var lastResponseText = responsePreview || "";
     if (lastResponseText && conversationParts.length > 0) {
       var lastPart = conversationParts[conversationParts.length - 1];
@@ -4209,14 +4243,33 @@ function createProjectContext(opts) {
       });
     }
 
+    // Read existing summary to give the digest worker context for delta content
+    var priorSummary = "";
+    try {
+      if (fs.existsSync(summaryFile)) {
+        priorSummary = fs.readFileSync(summaryFile, "utf8").trim();
+      }
+    } catch (e) {}
+
     _dmDigestPending = true;
+    var snapshotIndex = session.history.length;
 
     enqueueDigest({
       mateCtx: mateCtx,
       mateId: mateId,
       type: "dm",
+      priorSummary: priorSummary || "",
       conversationContent: conversationParts.join("\n"),
-      onDone: function () { _dmDigestPending = false; },
+      onDone: function (err) {
+        if (!err) {
+          session._dmLastDigestedIndex = snapshotIndex;
+          // Persist checkpoint so resumed sessions know where to continue
+          var checkpoint = { type: "digest_checkpoint", digestedIndex: snapshotIndex };
+          session.history.push(checkpoint);
+          sm.appendToSessionFile(session, checkpoint);
+        }
+        _dmDigestPending = false;
+      },
     });
   }
 

lib/sessions.js

@@ -344,6 +344,8 @@ function createSessionManager(opts) {
 
     for (var i = fromIndex; i < total; i++) {
       var _item = session.history[i];
+      // Skip internal bookkeeping entries not meant for the UI
+      if (_item && _item.type === "digest_checkpoint") continue;
       if (_item && (_item.type === "mention_user" || _item.type === "mention_response")) {
         console.log("[DEBUG replayHistory] sending mention at index=" + i + " from=" + fromIndex + " total=" + total + " type=" + _item.type + " mate=" + (_item.mateName || ""));
       }