feat(detect): index Jupyter notebooks with kernel-language fenced code blocks

graphify/detect.py

@@ -32,6 +32,9 @@ class FileType(str, Enum):
 PAPER_EXTENSIONS = {'.pdf'}
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
 OFFICE_EXTENSIONS = {'.docx', '.xlsx'}
+# Notebooks are converted to markdown sidecars before indexing — do NOT add .ipynb
+# to CODE_EXTENSIONS or DOC_EXTENSIONS.
+NOTEBOOK_EXTENSIONS = {'.ipynb'}
 VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'}
 
 CORPUS_WARN_THRESHOLD = 50_000    # words - below this, warn "you may not need a graph"
@@ -413,6 +416,8 @@ def classify_file(path: Path) -> FileType | None:
         return FileType.DOCUMENT
     if ext in OFFICE_EXTENSIONS:
         return FileType.DOCUMENT
+    if ext in NOTEBOOK_EXTENSIONS:
+        return FileType.DOCUMENT
     if ext in GOOGLE_WORKSPACE_EXTENSIONS:
         return FileType.DOCUMENT
     if ext in VIDEO_EXTENSIONS:
@@ -642,6 +647,68 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None:
     return out_path
 
 
+def ipynb_to_markdown(path: Path) -> str:
+    """Convert a Jupyter notebook to markdown, stripping outputs.
+
+    Uses the notebook's kernel language from metadata for fenced code blocks,
+    falling back to ``code`` when the metadata is absent.
+    """
+    if not _file_within_size_cap(path):
+        return ""
+    try:
+        nb = json.loads(path.read_text(encoding="utf-8", errors="ignore"))
+        # Resolve the kernel language from notebook metadata so fenced code
+        # blocks use the correct language identifier (e.g. ```python) rather
+        # than the generic ```code fallback.
+        meta = nb.get("metadata", {})
+        lang = (
+            meta.get("language_info", {}).get("name")
+            or meta.get("kernelspec", {}).get("language")
+            or "code"
+        )
+        lines = []
+        for cell in nb.get("cells", []):
+            ct = cell.get("cell_type")
+            raw_src = cell.get("source", [])
+            src = raw_src if isinstance(raw_src, str) else "".join(raw_src)
+            if not src.strip():
+                continue
+            if ct == "markdown":
+                lines.append(src)
+            elif ct == "code":
+                lines.append(f"```{lang}\n{src}\n```")
+        return "\n\n".join(lines)
+    except Exception:
+        return ""
+
+
+def convert_notebook_file(path: Path, out_dir: Path) -> Path | None:
+    """Convert a .ipynb to a markdown sidecar in out_dir.
+
+    Mirrors convert_office_file() sidecar naming. Unlike office files, an
+    existing sidecar is refreshed when cell sources change but left untouched
+    when only outputs/metadata changed — notebook re-runs must not re-extract.
+    """
+    if path.suffix.lower() not in NOTEBOOK_EXTENSIONS:
+        return None
+
+    text = ipynb_to_markdown(path)
+    if not text.strip():
+        return None
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    import hashlib
+    import unicodedata
+    normalized_path = unicodedata.normalize("NFC", str(path.resolve()))
+    name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8]
+    out_path = out_dir / f"{path.stem}_{name_hash}.md"
+    payload = f"<!-- converted from {path.name} -->\n\n{text}"
+    if out_path.exists() and out_path.read_text(encoding="utf-8") == payload:
+        return out_path
+    out_path.write_text(payload, encoding="utf-8")
+    return out_path
+
+
 def count_words(path: Path) -> int:
     try:
         ext = path.suffix.lower()
@@ -1131,6 +1198,17 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace:
                     # Conversion failed (library not installed) - skip with note
                     skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]")
                 continue
+            # Notebooks: convert to markdown sidecar so subagents can read them
+            if p.suffix.lower() in NOTEBOOK_EXTENSIONS:
+                md_path = convert_notebook_file(p, converted_dir)
+                if md_path:
+                    if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache):
+                        continue
+                    files[ftype].append(str(md_path))
+                    total_words += count_words(md_path)
+                else:
+                    skipped_sensitive.append(str(p) + " [notebook conversion failed]")
+                continue
             files[ftype].append(str(p))
             if ftype != FileType.VIDEO:
                 total_words += count_words(p)

tests/test_detect.py

@@ -1515,3 +1515,244 @@ def test_convert_office_file_does_not_rewrite_existing_sidecar(tmp_path, monkeyp
     second = detect_mod.convert_office_file(src, out_dir)
     assert second == first
     assert second.stat().st_mtime_ns == mtime_before
+
+
+def _minimal_ipynb(cells, metadata=None):
+    import json
+    nb = {"cells": cells, "nbformat": 4, "nbformat_minor": 5}
+    if metadata is not None:
+        nb["metadata"] = metadata
+    return json.dumps(nb)
+
+
+def test_classify_ipynb():
+    assert classify_file(Path("analysis.ipynb")) == FileType.DOCUMENT
+
+
+def test_ipynb_to_markdown_uses_language_info(tmp_path):
+    nb_path = tmp_path / "nb.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb(
+            [{"cell_type": "code", "source": "import pandas as pd\nprint('hi')"}],
+            metadata={"language_info": {"name": "python"}},
+        ),
+        encoding="utf-8",
+    )
+    md = detect_mod.ipynb_to_markdown(nb_path)
+    assert "```python\nimport pandas as pd" in md
+
+
+def test_ipynb_to_markdown_uses_kernelspec_fallback(tmp_path):
+    nb_path = tmp_path / "nb.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb(
+            [{"cell_type": "code", "source": "x = 1"}],
+            metadata={"kernelspec": {"language": "python"}},
+        ),
+        encoding="utf-8",
+    )
+    md = detect_mod.ipynb_to_markdown(nb_path)
+    assert "```python\nx = 1" in md
+
+
+def test_ipynb_to_markdown_falls_back_to_code_when_no_metadata(tmp_path):
+    nb_path = tmp_path / "nb.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "code", "source": "x = 1"}]),
+        encoding="utf-8",
+    )
+    md = detect_mod.ipynb_to_markdown(nb_path)
+    assert "```code\nx = 1" in md
+
+
+def test_ipynb_to_markdown_mixed_cells(tmp_path):
+    nb_path = tmp_path / "nb.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb(
+            [
+                {"cell_type": "markdown", "source": "# Title\n\nIntro text."},
+                {
+                    "cell_type": "code",
+                    "source": "import pandas as pd\nprint('hi')",
+                    "outputs": [{"output_type": "stream", "text": "hi\n"}],
+                },
+                {"cell_type": "markdown", "source": "## Section"},
+            ],
+            metadata={"language_info": {"name": "python"}},
+        ),
+        encoding="utf-8",
+    )
+    md = detect_mod.ipynb_to_markdown(nb_path)
+    assert "# Title" in md
+    assert "```python\nimport pandas as pd" in md
+    assert "hi\n" not in md  # outputs stripped
+    assert "## Section" in md
+    assert md.index("# Title") < md.index("```python") < md.index("## Section")
+
+
+def test_ipynb_to_markdown_empty_notebook(tmp_path):
+    nb_path = tmp_path / "empty.ipynb"
+    nb_path.write_text(_minimal_ipynb([]), encoding="utf-8")
+    assert detect_mod.ipynb_to_markdown(nb_path) == ""
+
+
+def test_ipynb_to_markdown_malformed_json(tmp_path):
+    nb_path = tmp_path / "bad.ipynb"
+    nb_path.write_text("{not valid json", encoding="utf-8")
+    assert detect_mod.ipynb_to_markdown(nb_path) == ""
+
+
+def test_detect_converts_notebook_to_sidecar(tmp_path):
+    nb_path = tmp_path / "analysis.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb(
+            [
+                {"cell_type": "markdown", "source": "# Analysis"},
+                {"cell_type": "code", "source": "x = 1"},
+            ],
+            metadata={"language_info": {"name": "python"}},
+        ),
+        encoding="utf-8",
+    )
+    result = detect(tmp_path)
+    assert len(result["files"]["document"]) == 1
+    sidecar = Path(result["files"]["document"][0])
+    assert sidecar.suffix == ".md"
+    assert sidecar.exists()
+    text = sidecar.read_text(encoding="utf-8")
+    assert "converted from analysis.ipynb" in text
+    assert "# Analysis" in text
+    assert "x = 1" in text
+    assert result["total_words"] > 0
+
+
+def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path):
+    nb_path = tmp_path / "empty.ipynb"
+    nb_path.write_text(_minimal_ipynb([]), encoding="utf-8")
+    out_dir = tmp_path / "converted"
+    assert detect_mod.convert_notebook_file(nb_path, out_dir) is None
+    assert not list(out_dir.glob("*.md"))
+
+
+def test_convert_notebook_file_output_change_preserves_sidecar_mtime(tmp_path):
+    """Re-running a notebook updates outputs in the .ipynb but not the sidecar."""
+    nb_path = tmp_path / "analysis.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "code", "source": "print(1)", "outputs": []}]),
+        encoding="utf-8",
+    )
+    out_dir = tmp_path / "converted"
+    sidecar = detect_mod.convert_notebook_file(nb_path, out_dir)
+    assert sidecar is not None
+    mtime_before = sidecar.stat().st_mtime_ns
+
+    nb_path.write_text(
+        _minimal_ipynb([
+            {
+                "cell_type": "code",
+                "source": "print(1)",
+                "outputs": [{"output_type": "stream", "text": "1\n"}],
+                "execution_count": 1,
+            },
+        ]),
+        encoding="utf-8",
+    )
+    again = detect_mod.convert_notebook_file(nb_path, out_dir)
+    assert again == sidecar
+    assert again.stat().st_mtime_ns == mtime_before
+
+
+def test_convert_notebook_file_source_change_rewrites_sidecar(tmp_path):
+    nb_path = tmp_path / "analysis.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "code", "source": "x = 1", "outputs": []}]),
+        encoding="utf-8",
+    )
+    out_dir = tmp_path / "converted"
+    sidecar = detect_mod.convert_notebook_file(nb_path, out_dir)
+    mtime_before = sidecar.stat().st_mtime_ns
+
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "code", "source": "x = 2", "outputs": []}]),
+        encoding="utf-8",
+    )
+    updated = detect_mod.convert_notebook_file(nb_path, out_dir)
+    assert updated == sidecar
+    assert "x = 2" in updated.read_text(encoding="utf-8")
+    assert updated.stat().st_mtime_ns >= mtime_before
+
+
+def test_detect_refreshes_notebook_sidecar_on_source_change(tmp_path):
+    """Cell source edits must update the sidecar so a later extract sees new content."""
+    nb_path = tmp_path / "analysis.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "markdown", "source": "v1"}]),
+        encoding="utf-8",
+    )
+    detect(tmp_path)
+    converted_dir = tmp_path / "graphify-out" / "converted"
+    sidecar = next(converted_dir.glob("analysis_*.md"))
+
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]),
+        encoding="utf-8",
+    )
+    detect(tmp_path)
+    assert "v2 updated" in sidecar.read_text(encoding="utf-8")
+
+
+def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path):
+    import json
+
+    nb_path = tmp_path / "analysis.ipynb"
+    nb_path.write_text(
+        _minimal_ipynb([{"cell_type": "code", "source": "print(1)", "outputs": []}]),
+        encoding="utf-8",
+    )
+    first = detect(tmp_path)
+    sidecar = Path(first["files"]["document"][0])
+    mtime_before = sidecar.stat().st_mtime_ns
+    manifest_path = tmp_path / "graphify-out" / "manifest.json"
+    Path(manifest_path).write_text(
+        json.dumps({
+            str(sidecar): {
+                "mtime": sidecar.stat().st_mtime,
+                "ast_hash": "a" * 32,
+                "semantic_hash": "b" * 32,
+            }
+        }),
+        encoding="utf-8",
+    )
+
+    nb_path.write_text(
+        _minimal_ipynb([
+            {
+                "cell_type": "code",
+                "source": "print(1)",
+                "outputs": [{"output_type": "stream", "text": "1\n"}],
+                "execution_count": 1,
+            },
+        ]),
+        encoding="utf-8",
+    )
+    inc = detect_incremental(tmp_path, manifest_path=str(manifest_path))
+    assert sidecar.stat().st_mtime_ns == mtime_before
+    assert not inc["new_files"]["document"]
+    assert str(sidecar) in inc["unchanged_files"]["document"]
+
+
+def test_ipynb_to_markdown_source_as_string(tmp_path):
+    """Notebooks may store cell source as a plain string rather than a list."""
+    import json
+    nb_path = tmp_path / "nb.ipynb"
+    nb_path.write_text(
+        json.dumps({
+            "cells": [{"cell_type": "code", "source": "x = 1"}],
+            "metadata": {"language_info": {"name": "python"}},
+            "nbformat": 4,
+            "nbformat_minor": 5,
+        }),
+        encoding="utf-8",
+    )
+    md = detect_mod.ipynb_to_markdown(nb_path)
+    assert "```python\nx = 1" in md