feat: add passive learning daemon — queue, auto-compile, multi-folder watch, service install

packages/cli/src/index.ts

@@ -132,10 +132,15 @@ program
 
 program
 	.command("watch")
-	.description("Watch inbox/ and auto-ingest")
-	.action(async () => {
+	.description("Watch inbox/ and auto-ingest (passive learning daemon)")
+	.option("--daemon", "run in background as a daemon")
+	.option("--stop", "stop the running daemon")
+	.option("--status", "check if the daemon is running")
+	.option("--install", "install as a system service (launchd/systemd)")
+	.option("--uninstall", "remove the system service")
+	.action(async (opts) => {
 		const { watch } = await import("./commands/watch.js");
-		await watch();
+		await watch(opts);
 	});
 
 program

packages/core/src/constants.ts

@@ -32,6 +32,9 @@ export const DEFAULTS = {
 	cacheMaxSizeMb: 500,
 	watchPollIntervalMs: 2000,
 	maxFileSizeMb: 50,
+	autoCompileThreshold: 5, // compile after N new sources
+	autoCompileDelayMs: 30 * 60 * 1000, // 30 min idle → auto-compile
+	watchLogMaxMb: 10,
 	compileArticleMinWords: 200,
 	compileArticleMaxWords: 1000,
 	contextWindow: 200_000, // tokens — conservative default (Sonnet 4.6 supports 1M)

packages/core/src/daemon/folder-watcher.test.ts

@@ -0,0 +1,175 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { matchGlob, scanFolder, startFolderWatchers } from "./folder-watcher.js";
+
+let tempDirs: string[] = [];
+
+afterEach(async () => {
+	for (const dir of tempDirs) {
+		await rm(dir, { recursive: true, force: true });
+	}
+	tempDirs = [];
+});
+
+async function makeTempDir(name: string) {
+	const dir = await mkdtemp(join(tmpdir(), `kib-fw-${name}-`));
+	tempDirs.push(dir);
+	return dir;
+}
+
+describe("matchGlob", () => {
+	test("matches *.pdf", () => {
+		expect(matchGlob("paper.pdf", "*.pdf")).toBe(true);
+		expect(matchGlob("paper.PDF", "*.pdf")).toBe(true); // case insensitive
+		expect(matchGlob("paper.txt", "*.pdf")).toBe(false);
+	});
+
+	test("matches *.{ext1,ext2}", () => {
+		expect(matchGlob("file.md", "*.{md,txt}")).toBe(true);
+		expect(matchGlob("file.txt", "*.{md,txt}")).toBe(true);
+		expect(matchGlob("file.pdf", "*.{md,txt}")).toBe(false);
+	});
+
+	test("matches * (wildcard all)", () => {
+		expect(matchGlob("anything.xyz", "*")).toBe(true);
+		expect(matchGlob("file", "*")).toBe(true);
+	});
+
+	test("matches exact filename", () => {
+		expect(matchGlob("notes.md", "notes.md")).toBe(true);
+		expect(matchGlob("other.md", "notes.md")).toBe(false);
+	});
+});
+
+describe("scanFolder", () => {
+	test("finds matching files in a directory", async () => {
+		const dir = await makeTempDir("scan");
+		await writeFile(join(dir, "a.pdf"), "");
+		await writeFile(join(dir, "b.pdf"), "");
+		await writeFile(join(dir, "c.txt"), "");
+		await writeFile(join(dir, ".hidden.pdf"), "");
+
+		const matches = await scanFolder({ path: dir, glob: "*.pdf", recursive: false });
+		expect(matches.length).toBe(2);
+		expect(matches.every((m) => m.endsWith(".pdf"))).toBe(true);
+	});
+
+	test("scans recursively when enabled", async () => {
+		const dir = await makeTempDir("scan-rec");
+		const sub = join(dir, "sub");
+		await mkdir(sub);
+		await writeFile(join(dir, "a.md"), "");
+		await writeFile(join(sub, "b.md"), "");
+
+		const matches = await scanFolder({ path: dir, glob: "*.md", recursive: true });
+		expect(matches.length).toBe(2);
+	});
+
+	test("returns empty for non-existent directory", async () => {
+		const matches = await scanFolder({ path: "/nonexistent/path", glob: "*", recursive: false });
+		expect(matches).toEqual([]);
+	});
+});
+
+describe("startFolderWatchers", () => {
+	test("detects new files matching glob", async () => {
+		const dir = await makeTempDir("watch");
+		const detected: string[] = [];
+
+		const watcher = startFolderWatchers({
+			folders: [{ path: dir, glob: "*.md", recursive: false }],
+			onFile: (path) => detected.push(path),
+			debounceMs: 50,
+		});
+
+		// Write a matching file
+		await writeFile(join(dir, "test.md"), "content");
+		await new Promise((r) => setTimeout(r, 200));
+
+		// Write a non-matching file
+		await writeFile(join(dir, "test.pdf"), "content");
+		await new Promise((r) => setTimeout(r, 200));
+
+		watcher.stop();
+		expect(detected.length).toBe(1);
+		expect(detected[0]).toContain("test.md");
+	});
+
+	test("ignores dotfiles", async () => {
+		const dir = await makeTempDir("dotfiles");
+		const detected: string[] = [];
+
+		const watcher = startFolderWatchers({
+			folders: [{ path: dir, glob: "*", recursive: false }],
+			onFile: (path) => detected.push(path),
+			debounceMs: 50,
+		});
+
+		await writeFile(join(dir, ".hidden"), "content");
+		await new Promise((r) => setTimeout(r, 200));
+
+		watcher.stop();
+		expect(detected.length).toBe(0);
+	});
+
+	test("does not duplicate events for same file", async () => {
+		const dir = await makeTempDir("dedup");
+		const detected: string[] = [];
+
+		const watcher = startFolderWatchers({
+			folders: [{ path: dir, glob: "*", recursive: false }],
+			onFile: (path) => detected.push(path),
+			debounceMs: 50,
+		});
+
+		await writeFile(join(dir, "test.txt"), "v1");
+		await new Promise((r) => setTimeout(r, 200));
+		// Re-write same file
+		await writeFile(join(dir, "test.txt"), "v2");
+		await new Promise((r) => setTimeout(r, 200));
+
+		watcher.stop();
+		expect(detected.length).toBe(1);
+	});
+
+	test("watches multiple folders", async () => {
+		const dir1 = await makeTempDir("multi1");
+		const dir2 = await makeTempDir("multi2");
+		const detected: string[] = [];
+
+		const watcher = startFolderWatchers({
+			folders: [
+				{ path: dir1, glob: "*.md", recursive: false },
+				{ path: dir2, glob: "*.pdf", recursive: false },
+			],
+			onFile: (path) => detected.push(path),
+			debounceMs: 50,
+		});
+
+		await writeFile(join(dir1, "notes.md"), "content");
+		await writeFile(join(dir2, "paper.pdf"), "content");
+		await new Promise((r) => setTimeout(r, 300));
+
+		watcher.stop();
+		expect(detected.length).toBe(2);
+	});
+
+	test("stop() cleans up all watchers", async () => {
+		const dir = await makeTempDir("stop");
+		const detected: string[] = [];
+
+		const watcher = startFolderWatchers({
+			folders: [{ path: dir, glob: "*", recursive: false }],
+			onFile: (path) => detected.push(path),
+			debounceMs: 50,
+		});
+
+		watcher.stop();
+
+		await writeFile(join(dir, "after-stop.txt"), "content");
+		await new Promise((r) => setTimeout(r, 200));
+		expect(detected.length).toBe(0);
+	});
+});

packages/core/src/daemon/folder-watcher.ts

@@ -0,0 +1,119 @@
+import { type FSWatcher, watch as fsWatch } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
+import { extname, join, resolve } from "node:path";
+
+export interface WatchFolder {
+	path: string;
+	glob: string;
+	recursive: boolean;
+}
+
+export interface FolderWatcherOptions {
+	folders: WatchFolder[];
+	/** Called when a matching file is detected */
+	onFile: (filePath: string) => void;
+	/** Debounce in ms before processing a new file (default: 500) */
+	debounceMs?: number;
+}
+
+/**
+ * Match a filename against a simple glob pattern.
+ * Supports: *.ext, *.{ext1,ext2}, * (match all)
+ */
+export function matchGlob(filename: string, pattern: string): boolean {
+	if (pattern === "*") return true;
+
+	// Handle *.{ext1,ext2} pattern
+	const braceMatch = pattern.match(/^\*\.\{(.+)\}$/);
+	if (braceMatch) {
+		const extensions = braceMatch[1].split(",").map((e) => `.${e.trim()}`);
+		return extensions.includes(extname(filename).toLowerCase());
+	}
+
+	// Handle *.ext pattern
+	const extMatch = pattern.match(/^\*(\..+)$/);
+	if (extMatch) {
+		return extname(filename).toLowerCase() === extMatch[1].toLowerCase();
+	}
+
+	return filename === pattern;
+}
+
+/**
+ * Watch multiple folders for new files matching glob patterns.
+ * Returns a cleanup function to stop all watchers.
+ */
+export function startFolderWatchers(options: FolderWatcherOptions): { stop: () => void } {
+	const { folders, onFile, debounceMs = 500 } = options;
+	const watchers: FSWatcher[] = [];
+	const seen = new Set<string>();
+	const pending = new Map<string, ReturnType<typeof setTimeout>>();
+
+	for (const folder of folders) {
+		const absPath = resolve(folder.path.replace(/^~/, process.env.HOME ?? ""));
+
+		try {
+			const watcher = fsWatch(absPath, { recursive: folder.recursive }, (_event, filename) => {
+				if (!filename) return;
+				if (filename.startsWith(".")) return;
+
+				// Extract just the basename for glob matching
+				const base = filename.includes("/") ? filename.split("/").pop()! : filename;
+				if (!matchGlob(base, folder.glob)) return;
+
+				const fullPath = join(absPath, filename);
+				if (seen.has(fullPath)) return;
+
+				// Debounce: wait for file to finish writing
+				if (pending.has(fullPath)) {
+					clearTimeout(pending.get(fullPath)!);
+				}
+				pending.set(
+					fullPath,
+					setTimeout(async () => {
+						pending.delete(fullPath);
+						try {
+							await stat(fullPath); // verify file still exists
+							seen.add(fullPath);
+							onFile(fullPath);
+						} catch {
+							// File deleted before we could process
+						}
+					}, debounceMs),
+				);
+			});
+			watchers.push(watcher);
+		} catch {
+			// Folder doesn't exist or isn't watchable — skip
+		}
+	}
+
+	return {
+		stop: () => {
+			for (const w of watchers) w.close();
+			for (const t of pending.values()) clearTimeout(t);
+			pending.clear();
+		},
+	};
+}
+
+/** Scan a folder for existing files that match the glob. Used for initial seeding. */
+export async function scanFolder(folder: WatchFolder): Promise<string[]> {
+	const absPath = resolve(folder.path.replace(/^~/, process.env.HOME ?? ""));
+	const matches: string[] = [];
+
+	try {
+		const files = await readdir(absPath, { recursive: folder.recursive });
+		for (const file of files) {
+			const name = typeof file === "string" ? file : file.toString();
+			const base = name.includes("/") ? name.split("/").pop()! : name;
+			if (!base.startsWith(".") && matchGlob(base, folder.glob)) {
+				matches.push(join(absPath, name));
+			}
+		}
+	} catch {
+		// Folder doesn't exist
+	}
+
+	return matches;
+}

packages/core/src/daemon/index.ts

@@ -0,0 +1,30 @@
+export {
+	type FolderWatcherOptions,
+	matchGlob,
+	scanFolder,
+	startFolderWatchers,
+	type WatchFolder,
+} from "./folder-watcher.js";
+export { appendWatchLog } from "./log.js";
+export { getDaemonStatus, type PidInfo, readPid, removePid, stopDaemon, writePid } from "./pid.js";
+export {
+	clearFailed,
+	dequeue,
+	enqueue,
+	ensureQueueDirs,
+	listFailed,
+	listPending,
+	markFailed,
+	type QueueItem,
+	queueDepth,
+	readItem,
+} from "./queue.js";
+export { CompileScheduler, type SchedulerOptions } from "./scheduler.js";
+export {
+	detectPlatform,
+	type InstallResult,
+	installService,
+	isServiceInstalled,
+	type ServicePlatform,
+	uninstallService,
+} from "./service.js";