diff --git a/.gitignore b/.gitignore index ee364a1..ec84622 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ __pycache__/ *.pyc *.egg-info/ +build/ +dist/ .venv/ .env .coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b62292..9e57996 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,30 @@ # Unreleased -## Installer -- `scripts/install.sh` now skips starting services on a fresh install when `/etc/sacred-brain/*` still contains `CHANGE_ME` placeholders. Edit configs, then run `sudo ./scripts/install.sh --update` to start. -- Systemd units are discovered by globbing `ops/systemd/` instead of being listed in the script. New units added to the repo are picked up automatically. -- Added `--uninstall` (removes units, keeps state) and `--uninstall --purge` (also removes `/etc/sacred-brain`, `/var/lib/sacred-brain`, and the `sacred` user). -- Python venv is now `chown`'d to `sacred:sacred` so post-install pip operations from `just` recipes or hooks work without sudo. -- Added a `shellcheck` GitHub Actions workflow for `scripts/`. +## Installer — conventional pipx + Makefile + +Sacred Brain is now installed as a proper Python package via `pipx`, driven by +a Makefile that honors `DESTDIR` / `PREFIX` / `SYSCONFDIR`. + +- `pyproject.toml` ships console-script entry points: `hippocampus` (runs the + FastAPI app via uvicorn) and `memory-governor` (runs `memory_governor.app:run`). + Packages cover `brain.*`, `memory_governor.*`, and `sacred_brain.*`, including + `sacred_brain/prompts/sam_system.txt` as package data. +- New `Makefile` with `install`, `install-update`, `uninstall`, and + `uninstall-purge` targets. `make install` runs `pipx install --force .` into + `/opt/pipx/venvs/sacred-brain-hippocampus/` and symlinks the entry-point + binaries into `$(PREFIX)/bin/`. +- Systemd units now point at `/usr/local/bin/hippocampus` and + `/usr/local/bin/memory-governor` instead of an in-tree `.venv`. Timer-target + scripts use `/opt/pipx/venvs/sacred-brain-hippocampus/bin/python`. +- Migration: `make install` auto-detects and removes any pre-existing + `/opt/sacred-brain/.venv/` from the old in-tree-venv layout. Idempotent. +- `scripts/install.sh` is now a thin shim that exec's the Makefile, so existing + automation that calls `./scripts/install.sh [--update|--uninstall]` keeps + working. +- `sacred-search` is now installed by the Makefile to `$(PREFIX)/bin/`. +- Earlier in this cycle: skipped service start when configs contain `CHANGE_ME`, + switched to globbed unit discovery in `ops/systemd/`, added `--uninstall` / + `--uninstall --purge`, and added a `shellcheck` GitHub Actions workflow. # v0.3.0 (2026-02-15) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fb62fd7 --- /dev/null +++ b/Makefile @@ -0,0 +1,149 @@ +# Sacred Brain installer +# +# Common targets: +# sudo make install Full install (user, dirs, package, units, configs) +# sudo make install-update Reinstall package + units, restart services +# sudo make uninstall Stop + disable + remove units (keeps state) +# sudo make uninstall-purge Also remove configs, state, and 'sacred' user +# +# Honors DESTDIR, PREFIX, SYSCONFDIR, PIPX_HOME for packagers and non-default +# installs: +# make install PREFIX=/usr SYSCONFDIR=/etc DESTDIR=/tmp/stage + +PREFIX ?= /usr/local +SYSCONFDIR ?= /etc +DESTDIR ?= + +ETC_DIR := $(DESTDIR)$(SYSCONFDIR)/sacred-brain +STATE_DIR := $(DESTDIR)/var/lib/sacred-brain +SYSTEMD_DIR := $(DESTDIR)/etc/systemd/system +BIN_DIR := $(DESTDIR)$(PREFIX)/bin + +# pipx layout: a system-wide venv under /opt/pipx, with bins symlinked into +# $(PREFIX)/bin. This works on every pipx version (no --global needed) and +# keeps service binaries on the standard PATH. +PIPX_HOME ?= /opt/pipx +PIPX_BIN_DIR ?= $(PREFIX)/bin + +REPO_DIR := $(shell pwd) + +.PHONY: install install-update install-deps migrate-legacy install-package \ + install-bin install-systemd install-config uninstall uninstall-purge help + +help: + @sed -n '1,12p' Makefile + +# ── Composite targets ─────────────────────────────────────────────── + +install: install-deps migrate-legacy install-package install-bin install-systemd install-config + @./scripts/post-install-message.sh "$(ETC_DIR)" || true + +# Auto-cleanup of pre-pipx layout. Older installs put the venv at +# /opt/sacred-brain/.venv and the systemd ExecStarts pointed there. After +# this install, those paths are dead weight (and could confuse a future +# `pip install -e` if anyone runs it). Idempotent — silent if already clean. +migrate-legacy: + @if [ -d /opt/sacred-brain/.venv ]; then \ + echo " [+] Removing legacy in-tree venv: /opt/sacred-brain/.venv"; \ + rm -rf /opt/sacred-brain/.venv; \ + fi + @if [ -e /opt/sacred-brain/.venv ]; then \ + echo " [!] /opt/sacred-brain/.venv still present after cleanup — investigate"; \ + fi + +install-update: install-package install-systemd + systemctl daemon-reload + @for unit in $$(ls ops/systemd/*.service | xargs -n1 basename); do \ + systemctl is-enabled $$unit >/dev/null 2>&1 && systemctl restart $$unit || true; \ + done + +# ── Phase: service user + state dirs ──────────────────────────────── + +install-deps: + @if ! id sacred >/dev/null 2>&1; then \ + echo " [+] Creating service user 'sacred'"; \ + useradd --system --shell /usr/sbin/nologin --home-dir /opt/sacred-brain sacred; \ + else echo " [+] User 'sacred' already exists"; fi + @echo " [+] Creating state directories" + install -d -m 0755 -o sacred -g sacred $(STATE_DIR)/hippocampus + install -d -m 0755 -o sacred -g sacred $(STATE_DIR)/governor + install -d -m 0755 -o sacred -g sacred $(STATE_DIR)/cache + install -d -m 0755 -o sacred -g sacred $(ETC_DIR) + +# ── Phase: Python package via pipx ────────────────────────────────── + +install-package: + @command -v pipx >/dev/null || { echo "ERROR: pipx not installed (apt install pipx)"; exit 1; } + @echo " [+] Installing Python package via pipx into $(PIPX_HOME)" + PIPX_HOME=$(PIPX_HOME) PIPX_BIN_DIR=$(PIPX_BIN_DIR) \ + pipx install --force "$(REPO_DIR)" + +# ── Phase: shell scripts on PATH ──────────────────────────────────── + +install-bin: + @echo " [+] Installing shell utilities to $(BIN_DIR)" + install -d -m 0755 $(BIN_DIR) + install -m 0755 scripts/sacred-search $(BIN_DIR)/sacred-search + +# ── Phase: systemd units ──────────────────────────────────────────── + +install-systemd: + @echo " [+] Installing systemd units to $(SYSTEMD_DIR)" + install -d -m 0755 $(SYSTEMD_DIR) + @for f in ops/systemd/*.service ops/systemd/*.timer; do \ + [ -e "$$f" ] || continue; \ + install -m 0644 "$$f" $(SYSTEMD_DIR)/; \ + echo " $$(basename $$f)"; \ + done + @if [ -z "$(DESTDIR)" ]; then \ + systemctl daemon-reload; \ + for f in ops/systemd/*.service ops/systemd/*.timer; do \ + [ -e "$$f" ] || continue; \ + if grep -q '^\[Install\]' "$$f"; then \ + systemctl enable "$$(basename $$f)"; \ + fi; \ + done; \ + fi + +# ── Phase: config templates ───────────────────────────────────────── + +install-config: + @echo " [+] Installing config templates to $(ETC_DIR)" + @for tmpl in ops/config/*.example; do \ + [ -e "$$tmpl" ] || continue; \ + target="$(ETC_DIR)/$$(basename $${tmpl%.example})"; \ + if [ -f "$$target" ]; then \ + echo " exists, skipping: $$target"; \ + else \ + install -m 0640 -o sacred -g sacred "$$tmpl" "$$target"; \ + echo " created: $$target (edit CHANGE_ME values!)"; \ + fi; \ + done + +# ── Uninstall ─────────────────────────────────────────────────────── + +uninstall: + @echo " [+] Stopping and disabling units" + @for f in ops/systemd/*.service ops/systemd/*.timer; do \ + [ -e "$$f" ] || continue; \ + unit=$$(basename $$f); \ + if [ -f "$(SYSTEMD_DIR)/$$unit" ]; then \ + systemctl disable --now "$$unit" 2>/dev/null || true; \ + rm -f "$(SYSTEMD_DIR)/$$unit"; \ + fi; \ + done + @if [ -z "$(DESTDIR)" ]; then systemctl daemon-reload; fi + @echo " [+] Removing $(BIN_DIR)/sacred-search" + @rm -f $(BIN_DIR)/sacred-search + @echo " [+] Uninstalling Python package" + @PIPX_HOME=$(PIPX_HOME) PIPX_BIN_DIR=$(PIPX_BIN_DIR) \ + pipx uninstall sacred-brain-hippocampus 2>/dev/null || true + @echo " [+] Kept $(ETC_DIR) and $(STATE_DIR) (use 'make uninstall-purge' to remove)" + +uninstall-purge: uninstall + @echo " [+] Removing $(ETC_DIR) and $(STATE_DIR)" + rm -rf $(ETC_DIR) $(STATE_DIR) + @if id sacred >/dev/null 2>&1; then \ + echo " [+] Removing service user 'sacred'"; \ + userdel sacred 2>/dev/null || echo " [!] could not remove user 'sacred'"; \ + fi diff --git a/brain/hippocampus/__main__.py b/brain/hippocampus/__main__.py new file mode 100644 index 0000000..8f8ac6b --- /dev/null +++ b/brain/hippocampus/__main__.py @@ -0,0 +1,25 @@ +"""Console entry point for the Hippocampus service. + +Installed by pyproject's [project.scripts] as `hippocampus`. Reads +HIPPOCAMPUS_HOST / HIPPOCAMPUS_PORT from the environment so the systemd +unit and dev runs share configuration. +""" + +from __future__ import annotations + +import os + +import uvicorn + + +def main() -> None: + uvicorn.run( + "brain.hippocampus.app:app", + host=os.environ.get("HIPPOCAMPUS_HOST", "0.0.0.0"), + port=int(os.environ.get("HIPPOCAMPUS_PORT", "54321")), + reload=False, + ) + + +if __name__ == "__main__": + main() diff --git a/brain/hippocampus/app.py b/brain/hippocampus/app.py index 7c47570..3127115 100644 --- a/brain/hippocampus/app.py +++ b/brain/hippocampus/app.py @@ -131,12 +131,17 @@ async def delete_memory( @application.get("/memories/{user_id}", response_model=MemoryQueryResponse) async def query_memories( user_id: str, - query: str = Query(..., min_length=1), - limit: int | None = Query(None, ge=1, le=100), + query: str | None = Query(None), + limit: int | None = Query(None, ge=1, le=1000), adapter: Mem0Adapter = Depends(get_adapter), _: None = Depends(auth_dependency), ) -> MemoryQueryResponse: - records = adapter.query_memories(user_id=user_id, query=query, limit=limit) + # query present -> semantic search; absent -> list all (dreaming / + # governor recall-fallback need list-all, which used to 422). + if query and query.strip(): + records = adapter.query_memories(user_id=user_id, query=query, limit=limit) + else: + records = adapter.list_memories(user_id=user_id, limit=limit) return MemoryQueryResponse(memories=records) @application.post("/summaries", response_model=SummaryResponse) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 5962c68..2e801ef 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -4,7 +4,8 @@ - Linux with systemd (Debian/Ubuntu/Raspberry Pi OS) - Python 3.11+ -- `curl` (for health checks and consolidation) +- `pipx` (`sudo apt install pipx`) +- `make`, `curl` - `just` (optional, for ops recipes): `sudo apt install just` - A LiteLLM-compatible gateway on port 4000 (optional, for LLM features) @@ -12,45 +13,43 @@ ```bash # 1. Clone the repo -git clone git@github.com:davidj4tech/sacred-brain.git /opt/sacred-brain +sudo git clone https://github.com/davidj4tech/sacred-brain.git /opt/sacred-brain cd /opt/sacred-brain -# 2. Copy and edit config templates -cp ops/config/hippocampus.toml.example ops/config/hippocampus.toml.local -cp ops/config/hippocampus.env.example ops/config/hippocampus.env.local -cp ops/config/memory-governor.env.example ops/config/memory-governor.env.local +# 2. Install +sudo make install -# Edit each file — replace CHANGE_ME values with real API keys -# The installer will copy these to /etc/sacred-brain/ if no config exists yet - -# 3. Run the installer -sudo ./scripts/install.sh - -# 4. Edit /etc/sacred-brain/*.env and hippocampus.toml (replace CHANGE_ME values) +# 3. Edit /etc/sacred-brain/* — replace CHANGE_ME values sudoedit /etc/sacred-brain/hippocampus.toml sudoedit /etc/sacred-brain/hippocampus.env sudoedit /etc/sacred-brain/memory-governor.env -# 5. Start the services (the installer skips this on a fresh install while CHANGE_ME values are present) -sudo ./scripts/install.sh --update +# 4. Start the services +sudo systemctl start hippocampus memory-governor -# 6. Verify +# 5. Verify just health just timers ``` -## What the Installer Does +The repo lives at `/opt/sacred-brain/` because the timer-target scripts +(in `scripts/`) run from there. The Python services themselves run from a +pipx-managed venv at `/opt/pipx/venvs/sacred-brain-hippocampus/`, with +`hippocampus` and `memory-governor` symlinked into `/usr/local/bin/`. + +## What `make install` Does -| Phase | Action | -|-------|--------| -| 1. User | Creates `sacred` system user (nologin shell) | -| 2. Directories | Creates `/var/lib/sacred-brain/{hippocampus,governor,cache}` and `/etc/sacred-brain/` | -| 3. Config | Copies `.example` templates to `/etc/sacred-brain/` (skips if files already exist) | -| 4. Venv | Creates Python venv (owned by `sacred`) and installs dependencies | -| 5. Systemd | Copies all unit files from `ops/systemd/` to `/etc/systemd/system/` | -| 6. Enable | Enables every unit that has an `[Install]` section | -| 7. Start | Starts services + timers — **skipped on fresh install if any config still contains `CHANGE_ME`**; re-run with `--update` after editing | -| 8. Verify | Health checks both services | +| Target | Action | +|--------|--------| +| `install-deps` | Creates `sacred` system user (nologin) and `/var/lib/sacred-brain/{hippocampus,governor,cache}` and `/etc/sacred-brain/` | +| `migrate-legacy` | Removes any old `/opt/sacred-brain/.venv/` from a pre-pipx install (idempotent) | +| `install-package` | `pipx install --force .` into `/opt/pipx/venvs/sacred-brain-hippocampus/`, with `hippocampus` and `memory-governor` symlinks in `$(PREFIX)/bin/` | +| `install-bin` | Installs `sacred-search` to `$(PREFIX)/bin/sacred-search` | +| `install-systemd` | Copies all unit files from `ops/systemd/` to `/etc/systemd/system/`, runs `daemon-reload`, enables every unit with an `[Install]` section | +| `install-config` | Copies `.example` templates to `/etc/sacred-brain/` (skips files that already exist) | + +`make install` does **not** start services automatically — the operator +edits `CHANGE_ME` values, then runs `systemctl start` manually. ## Updating @@ -58,34 +57,54 @@ After pulling new code: ```bash cd /opt/sacred-brain -git pull - -# Update systemd units only (doesn't touch config or recreate dirs) -just update-units - -# Or manually: -sudo ./scripts/install.sh --update +sudo git pull +sudo make install-update ``` +`install-update` reinstalls the Python package, refreshes systemd unit +files, runs `daemon-reload`, and restarts any enabled services. + ## Uninstalling ```bash -# Stop, disable, and remove all systemd units (keeps configs and state) -sudo ./scripts/install.sh --uninstall +# Stop, disable, and remove all systemd units; uninstall the pipx package; +# remove sacred-search from $(PREFIX)/bin. Keeps configs and state. +sudo make uninstall # Also remove /etc/sacred-brain, /var/lib/sacred-brain, and the 'sacred' user -sudo ./scripts/install.sh --uninstall --purge +sudo make uninstall-purge ``` +## Compatibility shim + +`scripts/install.sh` still exists as a thin wrapper that delegates to the +Makefile, so `sudo ./scripts/install.sh [--update|--uninstall|--uninstall --purge]` +continues to work for existing automation. + +## Customization + +The Makefile honors standard variables for non-default installs and packagers: + +| Variable | Default | Purpose | +|----------|---------|---------| +| `PREFIX` | `/usr/local` | Where `hippocampus`, `memory-governor`, `sacred-search` live | +| `SYSCONFDIR` | `/etc` | Parent of `sacred-brain/` config dir | +| `DESTDIR` | (empty) | Stage all paths under this prefix (skips `systemctl` calls) | +| `PIPX_HOME` | `/opt/pipx` | Where pipx puts the package's venv | + ## Directory Layout ``` -/opt/sacred-brain/ ← repo (code, scripts, ops) -/etc/sacred-brain/ ← configuration (not in repo) - hippocampus.toml ← Hippocampus app config - hippocampus.env ← Hippocampus environment vars - memory-governor.env ← Governor environment vars -/var/lib/sacred-brain/ ← state (not in repo) +/opt/sacred-brain/ ← repo clone (timer scripts run from here) +/opt/pipx/venvs/sacred-brain-hippocampus/ ← installed Python package (services run from here) +/usr/local/bin/hippocampus +/usr/local/bin/memory-governor +/usr/local/bin/sacred-search +/etc/sacred-brain/ ← configuration (not in repo) + hippocampus.toml + hippocampus.env + memory-governor.env +/var/lib/sacred-brain/ ← state (not in repo) hippocampus/ hippocampus_memories.sqlite memories-denote/ @@ -125,6 +144,7 @@ Core settings: auth keys, Mem0 backend, Agno model, notes directory. See `ops/co ### Hippocampus Environment (`hippocampus.env`) SAM pipeline settings (LLM gateway URL, model alias, timeout) and the Hippocampus API key. +Optional: `HIPPOCAMPUS_HOST` (default `0.0.0.0`) and `HIPPOCAMPUS_PORT` (default `54321`). ### Governor Environment (`memory-governor.env`) @@ -152,7 +172,7 @@ All services run with: - `ProtectHome=true` - `PrivateTmp=true` - `ReadWritePaths` limited to `/var/lib/sacred-brain` -- `ReadOnlyPaths` for code and config +- `ReadOnlyPaths` for code (`/opt/pipx`, plus `/opt/sacred-brain` for timer scripts) and config (`/etc/sacred-brain`) Check security scores: `just security-audit` diff --git a/memory_governor/app.py b/memory_governor/app.py index 16483f8..64c8675 100644 --- a/memory_governor/app.py +++ b/memory_governor/app.py @@ -337,10 +337,16 @@ async def outcome(payload: OutcomeRequest) -> OutcomeResponse: @app.post("/recall", response_model=RecallResponse) async def recall(payload: RecallRequest) -> RecallResponse: + # Fetch a wider pool than k when reranking, so the reranker can pick the best k. + fetch_limit = ( + max(payload.k, runtime.hippo.rerank_max) + if runtime.hippo.rerank_enabled + else payload.k + ) memories = await runtime.hippo.query_memories( user_id=payload.user_id, query=payload.query, - limit=payload.k, + limit=fetch_limit, ) filter_scope_path: str | None = None if payload.filters.scope is not None: @@ -432,21 +438,26 @@ def _score(item: dict[str, Any]) -> float: scored = [(item, _score(item)) for item in filtered] scored.sort(key=lambda pair: pair[1], reverse=True) - top = scored[: payload.k] + score_by_id = {item.get("memory_id"): s for item, s in scored} + + # Optional LLM rerank over the deterministically-sorted pool, then take k. + # _rerank returns the original order unchanged when disabled or on error. + ordered_items = await runtime.hippo._rerank(payload.query, [item for item, _s in scored]) + top_items = ordered_items[: payload.k] query_hash = ( hashlib.sha1(payload.query.strip().lower().encode("utf-8")).hexdigest()[:16] if payload.query.strip() else None ) - for item, item_score in top: + for item in top_items: mid = item.get("memory_id") if mid: - runtime.enqueue_recall_hit(mid, query_hash=query_hash, rerank_score=item_score) + runtime.enqueue_recall_hit(mid, query_hash=query_hash, rerank_score=score_by_id.get(mid)) results = [ {k: v for k, v in item.items() if k not in ("memory_id", "scope_path", "last_outcome")} - for item, _s in top + for item in top_items ] return RecallResponse(results=results) diff --git a/memory_governor/clients.py b/memory_governor/clients.py index cf09447..1ae6928 100644 --- a/memory_governor/clients.py +++ b/memory_governor/clients.py @@ -36,19 +36,29 @@ def _headers(self) -> dict[str, str]: return headers async def _rerank(self, query: str, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Reorder candidates by LLM relevance. Robust: asks the model for an + index ORDER and reorders the ORIGINAL objects (never trusts the model to + echo objects back). Returns the original order on any failure / when + disabled, so callers can use it unconditionally.""" if not (self.rerank_enabled and self.litellm_base_url and self.rerank_model and candidates): return candidates import json as _json + import re as _re + pool = candidates[: self.rerank_max] + listing = "\n".join( + f"[{i}] {(c.get('text') or c.get('memory') or '')[:300]}" + for i, c in enumerate(pool) + ) prompt = ( - "Reorder the following memories by relevance to the query. " - "Return JSON array of the memory objects, unchanged, just reordered.\n" - f"Query: {query}\n" - f"Memories: {_json.dumps(candidates[: self.rerank_max], ensure_ascii=False)}" + "Rank these memories by relevance to the query, most relevant first. " + "Return ONLY a JSON array of their integer indices, each exactly once.\n" + f"Query: {query}\nMemories:\n{listing}" ) payload = { "model": self.rerank_model, "messages": [{"role": "user", "content": prompt}], + "temperature": 0, } headers = {"Content-Type": "application/json"} if self.litellm_api_key: @@ -62,11 +72,22 @@ async def _rerank(self, query: str, candidates: list[dict[str, Any]]) -> list[di headers=headers, ) resp.raise_for_status() - data = resp.json() - content = data["choices"][0]["message"]["content"] - parsed = _json.loads(content) - if isinstance(parsed, list): - return parsed + content = resp.json()["choices"][0]["message"]["content"] + match = _re.search(r"\[[\d,\s]*\]", content) + order = _json.loads(match.group(0) if match else content) + seen: set[int] = set() + ranked: list[dict[str, Any]] = [] + for idx in order: + if isinstance(idx, int) and 0 <= idx < len(pool) and idx not in seen: + ranked.append(pool[idx]) + seen.add(idx) + # Safety: append any pool items the model omitted, then the tail + # beyond rerank_max, so no candidate is ever dropped. + for i, cand in enumerate(pool): + if i not in seen: + ranked.append(cand) + ranked.extend(candidates[self.rerank_max:]) + return ranked except Exception as exc: LOGGER.warning("Rerank failed, using original order: %s", exc) return candidates diff --git a/ops/systemd/governor-digest.service b/ops/systemd/governor-digest.service index bdb0389..67162c0 100644 --- a/ops/systemd/governor-digest.service +++ b/ops/systemd/governor-digest.service @@ -10,7 +10,7 @@ WorkingDirectory=/opt/sacred-brain Environment=MG_USER_ID=sam Environment=DIGEST_OUTPUT_DIR=/opt/sam/memory/memory Environment=MG_TZ_OFFSET_HOURS=11 -ExecStart=/opt/sacred-brain/.venv/bin/python /opt/sacred-brain/scripts/governor_digest.py +ExecStart=/opt/pipx/venvs/sacred-brain-hippocampus/bin/python /opt/sacred-brain/scripts/governor_digest.py # Hardening (lighter for oneshot) NoNewPrivileges=true @@ -18,7 +18,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/opt/sam/memory -ReadOnlyPaths=/opt/sacred-brain +ReadOnlyPaths=/opt/sacred-brain /opt/pipx ProtectKernelTunables=true ProtectKernelModules=true diff --git a/ops/systemd/hippocampus-auto-prune.service b/ops/systemd/hippocampus-auto-prune.service index 99c3a5c..7fefe9d 100644 --- a/ops/systemd/hippocampus-auto-prune.service +++ b/ops/systemd/hippocampus-auto-prune.service @@ -9,7 +9,7 @@ User=sacred Group=sacred EnvironmentFile=/etc/sacred-brain/hippocampus.env Environment=HIPPOCAMPUS_SQLITE_PATH=/var/lib/sacred-brain/hippocampus/hippocampus_memories.sqlite -ExecStart=/opt/sacred-brain/.venv/bin/python scripts/prune_auto_memories.py +ExecStart=/opt/pipx/venvs/sacred-brain-hippocampus/bin/python scripts/prune_auto_memories.py # Hardening (lighter for oneshot) NoNewPrivileges=true @@ -17,7 +17,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain /etc/sacred-brain +ReadOnlyPaths=/opt/sacred-brain /opt/pipx /etc/sacred-brain ProtectKernelTunables=true ProtectKernelModules=true diff --git a/ops/systemd/hippocampus-auto-tune.service b/ops/systemd/hippocampus-auto-tune.service index 582c472..3419898 100644 --- a/ops/systemd/hippocampus-auto-tune.service +++ b/ops/systemd/hippocampus-auto-tune.service @@ -9,7 +9,7 @@ User=sacred Group=sacred Environment=HIPPOCAMPUS_SQLITE_PATH=/var/lib/sacred-brain/hippocampus/hippocampus_memories.sqlite Environment=AUTO_TUNE_PATH=/var/lib/sacred-brain/auto_memory_tuning.json -ExecStart=/opt/sacred-brain/.venv/bin/python scripts/auto_memory_tuner.py +ExecStart=/opt/pipx/venvs/sacred-brain-hippocampus/bin/python scripts/auto_memory_tuner.py # Hardening NoNewPrivileges=true @@ -17,7 +17,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain +ReadOnlyPaths=/opt/sacred-brain /opt/pipx ProtectKernelTunables=true ProtectKernelModules=true diff --git a/ops/systemd/hippocampus-memory-sync.service b/ops/systemd/hippocampus-memory-sync.service index dc3f1ef..e9adc1a 100644 --- a/ops/systemd/hippocampus-memory-sync.service +++ b/ops/systemd/hippocampus-memory-sync.service @@ -10,7 +10,7 @@ EnvironmentFile=/etc/sacred-brain/hippocampus.env Environment=MEMORY_SYNC_ROOT=/opt/sam Environment=HIPPOCAMPUS_USER_ID=sam Environment=HIPPOCAMPUS_SQLITE_PATH=/var/lib/sacred-brain/hippocampus/hippocampus_memories.sqlite -ExecStart=/opt/sacred-brain/.venv/bin/python /opt/sacred-brain/scripts/memory_sync.py push +ExecStart=/opt/pipx/venvs/sacred-brain-hippocampus/bin/python /opt/sacred-brain/scripts/memory_sync.py push # Hardening NoNewPrivileges=true @@ -18,7 +18,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain /opt/sam /etc/sacred-brain +ReadOnlyPaths=/opt/sacred-brain /opt/pipx /opt/sam /etc/sacred-brain ProtectKernelTunables=true ProtectKernelModules=true diff --git a/ops/systemd/hippocampus-notes-export.service b/ops/systemd/hippocampus-notes-export.service index 627e6ad..055ab3b 100644 --- a/ops/systemd/hippocampus-notes-export.service +++ b/ops/systemd/hippocampus-notes-export.service @@ -8,7 +8,7 @@ WorkingDirectory=/opt/sacred-brain User=sacred Group=sacred Environment=PYTHONPATH=/opt/sacred-brain -ExecStart=/opt/sacred-brain/.venv/bin/python scripts/mem0_org_sync.py export --dir /var/lib/sacred-brain/hippocampus/memories-denote +ExecStart=/opt/pipx/venvs/sacred-brain-hippocampus/bin/python scripts/mem0_org_sync.py export --dir /var/lib/sacred-brain/hippocampus/memories-denote # Hardening NoNewPrivileges=true @@ -16,7 +16,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain +ReadOnlyPaths=/opt/sacred-brain /opt/pipx ProtectKernelTunables=true ProtectKernelModules=true diff --git a/ops/systemd/hippocampus-notes-import.service b/ops/systemd/hippocampus-notes-import.service index f67e021..29828e7 100644 --- a/ops/systemd/hippocampus-notes-import.service +++ b/ops/systemd/hippocampus-notes-import.service @@ -8,7 +8,7 @@ WorkingDirectory=/opt/sacred-brain User=sacred Group=sacred Environment=PYTHONPATH=/opt/sacred-brain -ExecStart=/opt/sacred-brain/.venv/bin/python scripts/mem0_org_sync.py import --dir /var/lib/sacred-brain/hippocampus/memories-denote +ExecStart=/opt/pipx/venvs/sacred-brain-hippocampus/bin/python scripts/mem0_org_sync.py import --dir /var/lib/sacred-brain/hippocampus/memories-denote # Hardening NoNewPrivileges=true @@ -16,7 +16,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain +ReadOnlyPaths=/opt/sacred-brain /opt/pipx ProtectKernelTunables=true ProtectKernelModules=true diff --git a/ops/systemd/hippocampus.service b/ops/systemd/hippocampus.service index f67b5a1..ede44fb 100644 --- a/ops/systemd/hippocampus.service +++ b/ops/systemd/hippocampus.service @@ -4,9 +4,11 @@ After=network.target [Service] Type=simple -WorkingDirectory=/opt/sacred-brain EnvironmentFile=/etc/sacred-brain/hippocampus.env -ExecStart=/opt/sacred-brain/.venv/bin/uvicorn brain.hippocampus.app:app --host 0.0.0.0 --port 54321 +# Shared API keys rendered from the sops secrets store; optional (-), loaded +# second so it wins where keys overlap. No key literals in the file above. +EnvironmentFile=-/etc/sacred-brain/secrets.env +ExecStart=/usr/local/bin/hippocampus Restart=on-failure User=sacred Group=sacred @@ -17,7 +19,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain /etc/sacred-brain +ReadOnlyPaths=/opt/pipx /etc/sacred-brain ProtectKernelTunables=true ProtectKernelModules=true ProtectControlGroups=true diff --git a/ops/systemd/memory-governor.service b/ops/systemd/memory-governor.service index b2e4533..c4bdd02 100644 --- a/ops/systemd/memory-governor.service +++ b/ops/systemd/memory-governor.service @@ -7,9 +7,11 @@ Wants=network-online.target Type=simple User=sacred Group=sacred -WorkingDirectory=/opt/sacred-brain EnvironmentFile=/etc/sacred-brain/memory-governor.env -ExecStart=/opt/sacred-brain/.venv/bin/python -m memory_governor.app +# Shared API keys (LITELLM_API_KEY, HIPPOCAMPUS_API_KEY) rendered from the sops +# secrets store; optional (-), loaded second so it wins. No key literals above. +EnvironmentFile=-/etc/sacred-brain/secrets.env +ExecStart=/usr/local/bin/memory-governor Restart=always RestartSec=3 @@ -19,7 +21,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true ReadWritePaths=/var/lib/sacred-brain -ReadOnlyPaths=/opt/sacred-brain /etc/sacred-brain +ReadOnlyPaths=/opt/pipx /etc/sacred-brain ProtectKernelTunables=true ProtectKernelModules=true ProtectControlGroups=true diff --git a/pyproject.toml b/pyproject.toml index a1a2a14..412a8f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,13 +17,25 @@ dependencies = [ "mem0ai>=1.0.1", "agno>=2.3.0", "mcp>=1.27.0", - "kerykeion>=4.11", ] -[tool.setuptools] -packages = ["memory_governor"] +[project.scripts] +hippocampus = "brain.hippocampus.__main__:main" +memory-governor = "memory_governor.app:run" + +[tool.setuptools.packages.find] +include = ["brain*", "memory_governor*", "sacred_brain*"] +namespaces = true + +[tool.setuptools.package-data] +"sacred_brain.prompts" = ["*.txt"] [project.optional-dependencies] +# NB: kerykeion (Swiss Ephemeris / pyswisseph C build) is intentionally NOT a +# dependency here. Astrology/natal computation belongs to the *astrotunes* +# project, which owns kerykeion; sacred-brain's astrology code (memory_governor +# oracle, scripts/natal_to_sacred_brain) should delegate there. The hippocampus +# core needs none of it (the sqlite backend is lexical — no embeddings/LLM). dev = [ "pytest>=7.4", "ruff>=0.3.0", diff --git a/scripts/install.sh b/scripts/install.sh index eb7d175..48b5335 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,241 +1,36 @@ #!/usr/bin/env bash -set -euo pipefail - -# Sacred Brain installer -# Clone the repo, run this, edit /etc/sacred-brain/*, then re-run with --update. +# Backwards-compatible shim around the Makefile. +# +# Sacred Brain installation is driven by `make` targets — see Makefile and +# docs/INSTALL.md. This script is kept so existing muscle memory and any +# external docs that reference `./scripts/install.sh` continue to work. # # Usage: -# sudo ./scripts/install.sh # full install (won't start until configs are edited) -# sudo ./scripts/install.sh --update # refresh units + restart services -# sudo ./scripts/install.sh --uninstall # stop + disable + remove units (keeps state by default) -# sudo ./scripts/install.sh --uninstall --purge # also remove /etc and /var/lib state - -REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" -SYSTEMD_DIR="$REPO_DIR/ops/systemd" -CONFIG_DIR="$REPO_DIR/ops/config" -ETC_DIR="/etc/sacred-brain" -STATE_DIR="/var/lib/sacred-brain" - -# ── Helpers ────────────────────────────────────────────────────────── - -info() { echo " [+] $*"; } -warn() { echo " [!] $*" >&2; } -die() { echo " [ERROR] $*" >&2; exit 1; } - -# List unit files available in the repo (basenames). Every unit in -# ops/systemd/ is expected to have an [Install] section; if one doesn't, -# `systemctl enable` skips it (which is fine for compose-style units that -# are pulled in by another unit's Wants=). -list_units() { - local f - for f in "$SYSTEMD_DIR"/*.service "$SYSTEMD_DIR"/*.timer; do - [[ -e "$f" ]] && basename "$f" - done -} - -# True if /etc/sacred-brain/*.env or *.toml still contains CHANGE_ME placeholders. -configs_have_placeholders() { - [[ -d "$ETC_DIR" ]] || return 1 - grep -rlE 'CHANGE[_-]ME' "$ETC_DIR" >/dev/null 2>&1 -} - -# ── Argument parsing ──────────────────────────────────────────────── - -MODE=install -PURGE=false -for arg in "$@"; do - case "$arg" in - --update) MODE=update ;; - --uninstall) MODE=uninstall ;; - --purge) PURGE=true ;; - -h|--help) - sed -n '4,11p' "$0" - exit 0 - ;; - *) die "Unknown argument: $arg" ;; - esac -done - -# ── Pre-flight checks ─────────────────────────────────────────────── - -[[ $EUID -eq 0 ]] || die "Must run as root (sudo)" -[[ -d "$SYSTEMD_DIR" ]] || die "ops/systemd/ not found — run from repo root" - -# ── Uninstall ─────────────────────────────────────────────────────── - -if [[ "$MODE" == "uninstall" ]]; then - info "Stopping and disabling units" - for unit in $(list_units); do - if [[ -f "/etc/systemd/system/$unit" ]]; then - systemctl disable --now "$unit" || true - rm -f "/etc/systemd/system/$unit" - fi - done - systemctl daemon-reload - - if $PURGE; then - info "Removing $ETC_DIR and $STATE_DIR" - rm -rf "$ETC_DIR" "$STATE_DIR" - if id sacred &>/dev/null; then - info "Removing service user 'sacred'" - userdel sacred || warn "Could not remove user 'sacred'" - fi - else - info "Kept $ETC_DIR and $STATE_DIR (use --purge to remove)" - fi - info "Uninstall complete." - exit 0 -fi - -# ── Phase 1: Service user ─────────────────────────────────────────── - -if [[ "$MODE" == "install" ]]; then - if id sacred &>/dev/null; then - info "User 'sacred' already exists" - else - info "Creating service user 'sacred'" - useradd --system --shell /usr/sbin/nologin --home-dir /opt/sacred-brain sacred - fi -fi - -# ── Phase 2: FHS directories ──────────────────────────────────────── +# sudo ./scripts/install.sh → make install +# sudo ./scripts/install.sh --update → make install-update +# sudo ./scripts/install.sh --uninstall → make uninstall +# sudo ./scripts/install.sh --uninstall --purge → make uninstall-purge -if [[ "$MODE" == "install" ]]; then - info "Creating state directories" - mkdir -p "$STATE_DIR"/{hippocampus,governor,cache} - chown -R sacred:sacred "$STATE_DIR" - - info "Creating config directory" - mkdir -p "$ETC_DIR" - chown sacred:sacred "$ETC_DIR" -fi - -# ── Phase 3: Configuration ────────────────────────────────────────── +set -euo pipefail -if [[ "$MODE" == "install" ]]; then - for tmpl in "$CONFIG_DIR"/*.example; do - [[ -e "$tmpl" ]] || continue - target="$ETC_DIR/$(basename "${tmpl%.example}")" - if [[ -f "$target" ]]; then - info "Config exists, skipping: $target" +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_DIR" + +case "${1:-}" in + "") target=install ;; + --update) target=install-update ;; + --uninstall) + if [[ "${2:-}" == "--purge" ]]; then + target=uninstall-purge else - cp "$tmpl" "$target" - chown sacred:sacred "$target" - chmod 640 "$target" - warn "Created $target from template — edit CHANGE_ME values!" + target=uninstall fi - done -fi - -# ── Phase 4: Python venv ──────────────────────────────────────────── - -if [[ "$MODE" == "install" ]]; then - if [[ ! -d "$REPO_DIR/.venv" ]]; then - info "Creating Python venv" - python3 -m venv "$REPO_DIR/.venv" - "$REPO_DIR/.venv/bin/pip" install --upgrade pip - fi - - if [[ -f "$REPO_DIR/pyproject.toml" ]]; then - info "Installing Python package (editable)" - "$REPO_DIR/.venv/bin/pip" install -e "$REPO_DIR" - elif [[ -f "$REPO_DIR/requirements.txt" ]]; then - info "Installing Python dependencies from requirements.txt" - "$REPO_DIR/.venv/bin/pip" install -r "$REPO_DIR/requirements.txt" - else - warn "No pyproject.toml or requirements.txt — skipping Python deps" - fi - - # Services run as 'sacred', so the venv must be owned by 'sacred' for any - # post-install pip operations (just recipes, hooks) to work without sudo. - chown -R sacred:sacred "$REPO_DIR/.venv" -fi - -# ── Phase 5: Systemd units ────────────────────────────────────────── - -info "Installing systemd units" -installed_units=() -for unit in $(list_units); do - cp "$SYSTEMD_DIR/$unit" "/etc/systemd/system/$unit" - info " $unit" - installed_units+=("$unit") -done - -systemctl daemon-reload - -# ── Phase 6: Enable units ─────────────────────────────────────────── - -info "Enabling units" -# `systemctl enable` errors on units without [Install]; skip those quietly. -for unit in "${installed_units[@]}"; do - if grep -q '^\[Install\]' "$SYSTEMD_DIR/$unit"; then - systemctl enable "$unit" - fi -done - -# ── Phase 7: Start ────────────────────────────────────────────────── - -start_all() { - info "Starting services" - for unit in "${installed_units[@]}"; do - [[ "$unit" == *.service ]] || continue - systemctl start "$unit" || warn "Failed to start $unit" - done - info "Starting timers" - for unit in "${installed_units[@]}"; do - [[ "$unit" == *.timer ]] || continue - systemctl start "$unit" || warn "Failed to start $unit" - done -} - -if [[ "$MODE" == "install" ]]; then - if configs_have_placeholders; then - warn "Configs in $ETC_DIR still contain CHANGE_ME placeholders." - warn "Skipping service start. Edit them, then run:" - warn " sudo $0 --update" + ;; + -h|--help) + sed -n '4,12p' "$0" exit 0 - fi - start_all -elif [[ "$MODE" == "update" ]]; then - info "Restarting services" - for unit in "${installed_units[@]}"; do - [[ "$unit" == *.service ]] || continue - systemctl restart "$unit" || warn "Failed to restart $unit" - done -fi - -# ── Phase 8: Verify ───────────────────────────────────────────────── - -echo "" -info "Verifying..." -sleep 2 - -hippo_ok=false -gov_ok=false - -if curl -sf http://127.0.0.1:54321/health >/dev/null 2>&1; then - info "Hippocampus: OK" - hippo_ok=true -else - warn "Hippocampus: NOT RESPONDING" -fi - -if curl -sf http://127.0.0.1:54323/health >/dev/null 2>&1; then - info "Governor: OK" - gov_ok=true -else - warn "Governor: NOT RESPONDING" -fi + ;; + *) echo "ERROR: unknown argument: $1" >&2; exit 2 ;; +esac -echo "" -if $hippo_ok && $gov_ok; then - info "Sacred Brain is running!" - echo "" - echo " Hippocampus: http://127.0.0.1:54321" - echo " Governor: http://127.0.0.1:54323" - echo " Config: $ETC_DIR/" - echo " State: $STATE_DIR/" - echo " Ops: cd $REPO_DIR && just --list" -else - warn "Some services failed to start. Check: journalctl -u hippocampus -u memory-governor" -fi +exec make "$target" diff --git a/scripts/post-install-message.sh b/scripts/post-install-message.sh new file mode 100755 index 0000000..93820b7 --- /dev/null +++ b/scripts/post-install-message.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Print a friendly post-install summary. If any /etc/sacred-brain/* config +# still contains CHANGE_ME placeholders, tell the operator to edit them +# before starting services. +set -euo pipefail + +ETC_DIR="${1:-/etc/sacred-brain}" + +echo "" +echo " Sacred Brain installed." +echo "" +echo " Config: $ETC_DIR/" +echo " State: /var/lib/sacred-brain/" +echo " Code: /opt/pipx/venvs/sacred-brain-hippocampus/" +echo "" + +if [[ -d "$ETC_DIR" ]] && grep -rlE 'CHANGE[_-]ME' "$ETC_DIR" >/dev/null 2>&1; then + echo " [!] Configs still contain CHANGE_ME placeholders. Edit them, then:" + echo " sudo systemctl start hippocampus memory-governor" + echo " sudo make install-update # to also restart on code change" +else + echo " Start services:" + echo " sudo systemctl start hippocampus memory-governor" +fi +echo ""