From c47e82b490dd9314d3b53520ff7e1d88bc450ba2 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sun, 28 Jun 2026 00:23:59 +0800 Subject: [PATCH 1/5] feat(detect): index Jupyter notebooks via markdown sidecar extraction Convert .ipynb files to markdown sidecars (code cells as fenced blocks, markdown cells verbatim, outputs stripped) and classify them as documents so notebook-heavy ML corpora are no longer silently dropped during scan. Fixes #1497 Co-authored-by: Cursor --- graphify/detect.py | 72 +++++++++++++++++++++++++----- tests/test_detect.py | 103 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 12 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 92b773f7b..98b8dceb0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -32,6 +32,9 @@ class FileType(str, Enum): PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} +# Notebooks are converted to markdown sidecars before indexing — do NOT add .ipynb +# to CODE_EXTENSIONS or DOC_EXTENSIONS. +NOTEBOOK_EXTENSIONS = {'.ipynb'} VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} CORPUS_WARN_THRESHOLD = 50_000 # words - below this, warn "you may not need a graph" @@ -413,6 +416,8 @@ def classify_file(path: Path) -> FileType | None: return FileType.DOCUMENT if ext in OFFICE_EXTENSIONS: return FileType.DOCUMENT + if ext in NOTEBOOK_EXTENSIONS: + return FileType.DOCUMENT if ext in GOOGLE_WORKSPACE_EXTENSIONS: return FileType.DOCUMENT if ext in VIDEO_EXTENSIONS: @@ -601,20 +606,29 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} -def convert_office_file(path: Path, out_dir: Path) -> Path | None: - """Convert a .docx or .xlsx to a markdown sidecar in out_dir. +def ipynb_to_markdown(path: Path) -> str: + """Convert a Jupyter notebook to markdown, stripping outputs.""" + if not _file_within_size_cap(path): + return "" + try: + nb = json.loads(path.read_text(encoding="utf-8", errors="ignore")) + lines = [] + for cell in nb.get("cells", []): + ct = cell.get("cell_type") + src = "".join(cell.get("source", [])) + if not src.strip(): + continue + if ct == "markdown": + lines.append(src) + elif ct == "code": + lines.append(f"```python\n{src}\n```") + return "\n\n".join(lines) + except Exception: + return "" - Returns the path of the converted .md file, or None if conversion failed - or the required library is not installed. - """ - ext = path.suffix.lower() - if ext == ".docx": - text = docx_to_markdown(path) - elif ext == ".xlsx": - text = xlsx_to_markdown(path) - else: - return None +def _write_markdown_sidecar(path: Path, out_dir: Path, text: str) -> Path | None: + """Write a markdown sidecar for a converted source file.""" if not text.strip(): return None @@ -642,6 +656,30 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None: return out_path +def convert_office_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .docx or .xlsx to a markdown sidecar in out_dir. + + Returns the path of the converted .md file, or None if conversion failed + or the required library is not installed. + """ + ext = path.suffix.lower() + if ext == ".docx": + text = docx_to_markdown(path) + elif ext == ".xlsx": + text = xlsx_to_markdown(path) + else: + return None + + return _write_markdown_sidecar(path, out_dir, text) + + +def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .ipynb to a markdown sidecar in out_dir.""" + if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: + return None + return _write_markdown_sidecar(path, out_dir, ipynb_to_markdown(path)) + + def count_words(path: Path) -> int: try: ext = path.suffix.lower() @@ -1131,6 +1169,16 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: # Conversion failed (library not installed) - skip with note skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue + if p.suffix.lower() in NOTEBOOK_EXTENSIONS: + md_path = convert_notebook_file(p, converted_dir) + if md_path: + if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): + continue + files[ftype].append(str(md_path)) + total_words += count_words(md_path) + else: + skipped_sensitive.append(str(p) + " [notebook conversion failed]") + continue files[ftype].append(str(p)) if ftype != FileType.VIDEO: total_words += count_words(p) diff --git a/tests/test_detect.py b/tests/test_detect.py index 76282f295..fb1580573 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1515,3 +1515,106 @@ def test_convert_office_file_does_not_rewrite_existing_sidecar(tmp_path, monkeyp second = detect_mod.convert_office_file(src, out_dir) assert second == first assert second.stat().st_mtime_ns == mtime_before + + +def _minimal_ipynb(cells): + import json + return json.dumps({"cells": cells, "nbformat": 4, "nbformat_minor": 5}) + + +def test_classify_ipynb(): + assert classify_file(Path("analysis.ipynb")) == FileType.DOCUMENT + + +def test_ipynb_to_markdown_mixed_cells(tmp_path): + nb_path = tmp_path / "nb.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "markdown", "source": "# Title\n\nIntro text."}, + { + "cell_type": "code", + "source": "import pandas as pd\nprint('hi')", + "outputs": [{"output_type": "stream", "text": "hi\n"}], + }, + {"cell_type": "markdown", "source": "## Section"}, + ]), + encoding="utf-8", + ) + md = detect_mod.ipynb_to_markdown(nb_path) + assert "# Title" in md + assert "```python\nimport pandas as pd" in md + assert "hi\n" not in md # outputs stripped + assert "## Section" in md + assert md.index("# Title") < md.index("```python") < md.index("## Section") + + +def test_ipynb_to_markdown_empty_notebook(tmp_path): + nb_path = tmp_path / "empty.ipynb" + nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") + assert detect_mod.ipynb_to_markdown(nb_path) == "" + + +def test_ipynb_to_markdown_malformed_json(tmp_path): + nb_path = tmp_path / "bad.ipynb" + nb_path.write_text("{not valid json", encoding="utf-8") + assert detect_mod.ipynb_to_markdown(nb_path) == "" + + +def test_detect_converts_notebook_to_sidecar(tmp_path): + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "markdown", "source": "# Analysis"}, + {"cell_type": "code", "source": "x = 1"}, + ]), + encoding="utf-8", + ) + result = detect(tmp_path) + assert len(result["files"]["document"]) == 1 + sidecar = Path(result["files"]["document"][0]) + assert sidecar.suffix == ".md" + assert sidecar.exists() + text = sidecar.read_text(encoding="utf-8") + assert "converted from analysis.ipynb" in text + assert "# Analysis" in text + assert "x = 1" in text + assert result["total_words"] > 0 + + +def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path): + nb_path = tmp_path / "empty.ipynb" + nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") + out_dir = tmp_path / "converted" + assert detect_mod.convert_notebook_file(nb_path, out_dir) is None + assert not list(out_dir.glob("*.md")) + + +def test_detect_incremental_notebook_sidecar_tracks_changes(tmp_path): + """When a notebook is edited, re-converting after removing the stale sidecar + produces updated content that detect_incremental picks up as changed.""" + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v1"}]), + encoding="utf-8", + ) + first = detect(tmp_path) + sidecar = Path(first["files"]["document"][0]) + manifest_path = tmp_path / "graphify-out" / "manifest.json" + save_manifest( + {sidecar: {"mtime": sidecar.stat().st_mtime, "ast_hash": "x", "semantic_hash": "y"}}, + str(manifest_path), + root=tmp_path, + ) + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + encoding="utf-8", + ) + sidecar.unlink() + converted_dir = tmp_path / "graphify-out" / "converted" + new_sidecar = detect_mod.convert_notebook_file(nb_path, converted_dir) + assert new_sidecar is not None + assert "v2 updated" in new_sidecar.read_text(encoding="utf-8") + + inc = detect_incremental(tmp_path, manifest_path=str(manifest_path)) + assert any(str(new_sidecar) == f for f in inc["new_files"]["document"]) From 208378aff17a09a4176c3082ab136955845b4b1a Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sun, 28 Jun 2026 00:33:05 +0800 Subject: [PATCH 2/5] refactor(detect): keep convert_office_file unchanged, add notebook path separately Restore convert_office_file inline and place ipynb_to_markdown/convert_notebook_file after it so the PR diff clearly adds notebook support without touching office conversion. Co-authored-by: Cursor --- graphify/detect.py | 85 +++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 98b8dceb0..b9e824098 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -606,6 +606,47 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} +def convert_office_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .docx or .xlsx to a markdown sidecar in out_dir. + + Returns the path of the converted .md file, or None if conversion failed + or the required library is not installed. + """ + ext = path.suffix.lower() + if ext == ".docx": + text = docx_to_markdown(path) + elif ext == ".xlsx": + text = xlsx_to_markdown(path) + else: + return None + + if not text.strip(): + return None + + out_dir.mkdir(parents=True, exist_ok=True) + # Use a stable name derived from the original path to avoid collisions. + # Normalize the resolved path to NFC before hashing: on macOS (HFS+/APFS) + # os.walk/rglob return filenames in NFD, while Python string literals and + # directly-constructed Path objects are NFC, so the same source file would + # otherwise hash to different sidecar names across runs — causing --update + # to treat every Office file as new and re-extract it (#1226). + import hashlib + import unicodedata + normalized_path = unicodedata.normalize("NFC", str(path.resolve())) + name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] + out_path = out_dir / f"{path.stem}_{name_hash}.md" + # Once the hash is stable the sidecar name is deterministic; skip re-writing + # an existing sidecar so an unchanged source never churns its mtime (which + # would still flag it as changed in detect_incremental). + if out_path.exists(): + return out_path + out_path.write_text( + f"\n\n{text}", + encoding="utf-8", + ) + return out_path + + def ipynb_to_markdown(path: Path) -> str: """Convert a Jupyter notebook to markdown, stripping outputs.""" if not _file_within_size_cap(path): @@ -627,26 +668,24 @@ def ipynb_to_markdown(path: Path) -> str: return "" -def _write_markdown_sidecar(path: Path, out_dir: Path, text: str) -> Path | None: - """Write a markdown sidecar for a converted source file.""" +def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .ipynb to a markdown sidecar in out_dir. + + Mirrors convert_office_file(): same sidecar naming and mtime semantics. + """ + if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: + return None + + text = ipynb_to_markdown(path) if not text.strip(): return None out_dir.mkdir(parents=True, exist_ok=True) - # Use a stable name derived from the original path to avoid collisions. - # Normalize the resolved path to NFC before hashing: on macOS (HFS+/APFS) - # os.walk/rglob return filenames in NFD, while Python string literals and - # directly-constructed Path objects are NFC, so the same source file would - # otherwise hash to different sidecar names across runs — causing --update - # to treat every Office file as new and re-extract it (#1226). import hashlib import unicodedata normalized_path = unicodedata.normalize("NFC", str(path.resolve())) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" - # Once the hash is stable the sidecar name is deterministic; skip re-writing - # an existing sidecar so an unchanged source never churns its mtime (which - # would still flag it as changed in detect_incremental). if out_path.exists(): return out_path out_path.write_text( @@ -656,30 +695,6 @@ def _write_markdown_sidecar(path: Path, out_dir: Path, text: str) -> Path | None return out_path -def convert_office_file(path: Path, out_dir: Path) -> Path | None: - """Convert a .docx or .xlsx to a markdown sidecar in out_dir. - - Returns the path of the converted .md file, or None if conversion failed - or the required library is not installed. - """ - ext = path.suffix.lower() - if ext == ".docx": - text = docx_to_markdown(path) - elif ext == ".xlsx": - text = xlsx_to_markdown(path) - else: - return None - - return _write_markdown_sidecar(path, out_dir, text) - - -def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: - """Convert a .ipynb to a markdown sidecar in out_dir.""" - if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: - return None - return _write_markdown_sidecar(path, out_dir, ipynb_to_markdown(path)) - - def count_words(path: Path) -> int: try: ext = path.suffix.lower() From df6515103e60f778b6f119d136364924e56948c2 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sun, 28 Jun 2026 00:37:38 +0800 Subject: [PATCH 3/5] fix(detect): ignore notebook output-only changes during conversion Compare extracted sidecar payload before rewriting so re-running a notebook updates the .ipynb on disk without bumping sidecar mtime or triggering re-extraction. Cell source edits still refresh the sidecar. Co-authored-by: Cursor --- graphify/detect.py | 12 ++--- tests/test_detect.py | 108 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index b9e824098..aff86a387 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -671,7 +671,9 @@ def ipynb_to_markdown(path: Path) -> str: def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir. - Mirrors convert_office_file(): same sidecar naming and mtime semantics. + Mirrors convert_office_file() sidecar naming. Unlike office files, an + existing sidecar is refreshed when cell sources change but left untouched + when only outputs/metadata changed — notebook re-runs must not re-extract. """ if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: return None @@ -686,12 +688,10 @@ def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: normalized_path = unicodedata.normalize("NFC", str(path.resolve())) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" - if out_path.exists(): + payload = f"\n\n{text}" + if out_path.exists() and out_path.read_text(encoding="utf-8") == payload: return out_path - out_path.write_text( - f"\n\n{text}", - encoding="utf-8", - ) + out_path.write_text(payload, encoding="utf-8") return out_path diff --git a/tests/test_detect.py b/tests/test_detect.py index fb1580573..e183dabad 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1589,32 +1589,110 @@ def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path): assert not list(out_dir.glob("*.md")) -def test_detect_incremental_notebook_sidecar_tracks_changes(tmp_path): - """When a notebook is edited, re-converting after removing the stale sidecar - produces updated content that detect_incremental picks up as changed.""" +def test_convert_notebook_file_output_change_preserves_sidecar_mtime(tmp_path): + """Re-running a notebook updates outputs in the .ipynb but not the sidecar.""" + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "code", "source": "print(1)", "outputs": []}, + ]), + encoding="utf-8", + ) + out_dir = tmp_path / "converted" + sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) + assert sidecar is not None + mtime_before = sidecar.stat().st_mtime_ns + + nb_path.write_text( + _minimal_ipynb([ + { + "cell_type": "code", + "source": "print(1)", + "outputs": [{"output_type": "stream", "text": "1\n"}], + "execution_count": 1, + }, + ]), + encoding="utf-8", + ) + again = detect_mod.convert_notebook_file(nb_path, out_dir) + assert again == sidecar + assert again.stat().st_mtime_ns == mtime_before + + +def test_convert_notebook_file_source_change_rewrites_sidecar(tmp_path): + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 1", "outputs": []}]), + encoding="utf-8", + ) + out_dir = tmp_path / "converted" + sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) + mtime_before = sidecar.stat().st_mtime_ns + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 2", "outputs": []}]), + encoding="utf-8", + ) + updated = detect_mod.convert_notebook_file(nb_path, out_dir) + assert updated == sidecar + assert "x = 2" in updated.read_text(encoding="utf-8") + assert updated.stat().st_mtime_ns >= mtime_before + + +def test_detect_refreshes_notebook_sidecar_on_source_change(tmp_path): + """Cell source edits must update the sidecar so a later extract sees new content.""" nb_path = tmp_path / "analysis.ipynb" nb_path.write_text( _minimal_ipynb([{"cell_type": "markdown", "source": "v1"}]), encoding="utf-8", ) + detect(tmp_path) + converted_dir = tmp_path / "graphify-out" / "converted" + sidecar = next(converted_dir.glob("analysis_*.md")) + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + encoding="utf-8", + ) + detect(tmp_path) + assert "v2 updated" in sidecar.read_text(encoding="utf-8") + + +def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path): + import json + + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "print(1)", "outputs": []}]), + encoding="utf-8", + ) first = detect(tmp_path) sidecar = Path(first["files"]["document"][0]) + mtime_before = sidecar.stat().st_mtime_ns manifest_path = tmp_path / "graphify-out" / "manifest.json" - save_manifest( - {sidecar: {"mtime": sidecar.stat().st_mtime, "ast_hash": "x", "semantic_hash": "y"}}, - str(manifest_path), - root=tmp_path, + Path(manifest_path).write_text( + json.dumps({ + str(sidecar): { + "mtime": sidecar.stat().st_mtime, + "ast_hash": "a" * 32, + "semantic_hash": "b" * 32, + } + }), + encoding="utf-8", ) nb_path.write_text( - _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + _minimal_ipynb([ + { + "cell_type": "code", + "source": "print(1)", + "outputs": [{"output_type": "stream", "text": "1\n"}], + "execution_count": 1, + }, + ]), encoding="utf-8", ) - sidecar.unlink() - converted_dir = tmp_path / "graphify-out" / "converted" - new_sidecar = detect_mod.convert_notebook_file(nb_path, converted_dir) - assert new_sidecar is not None - assert "v2 updated" in new_sidecar.read_text(encoding="utf-8") - inc = detect_incremental(tmp_path, manifest_path=str(manifest_path)) - assert any(str(new_sidecar) == f for f in inc["new_files"]["document"]) + assert sidecar.stat().st_mtime_ns == mtime_before + assert not inc["new_files"]["document"] + assert str(sidecar) in inc["unchanged_files"]["document"] From 992a41d77cfc459bedeb13ca6272c79e06bb5946 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Mon, 29 Jun 2026 01:41:19 +0800 Subject: [PATCH 4/5] Change code block syntax from 'python' to 'code' --- graphify/detect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphify/detect.py b/graphify/detect.py index aff86a387..ba0fc88e5 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -662,7 +662,7 @@ def ipynb_to_markdown(path: Path) -> str: if ct == "markdown": lines.append(src) elif ct == "code": - lines.append(f"```python\n{src}\n```") + lines.append(f"```code\n{src}\n```") return "\n\n".join(lines) except Exception: return "" From fd2ec4ec445961a19b226d7530b59637a092673d Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Mon, 29 Jun 2026 03:06:21 +0800 Subject: [PATCH 5/5] Enhance ipynb_to_markdown for kernel language support Updated the ipynb_to_markdown function to use the notebook's kernel language for fenced code blocks instead of a generic fallback. Improved handling of source code extraction from notebook cells. --- graphify/detect.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index ba0fc88e5..105d8f100 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -648,26 +648,40 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None: def ipynb_to_markdown(path: Path) -> str: - """Convert a Jupyter notebook to markdown, stripping outputs.""" + """Convert a Jupyter notebook to markdown, stripping outputs. + + Uses the notebook's kernel language from metadata for fenced code blocks, + falling back to ``code`` when the metadata is absent. + """ if not _file_within_size_cap(path): return "" try: nb = json.loads(path.read_text(encoding="utf-8", errors="ignore")) + # Resolve the kernel language from notebook metadata so fenced code + # blocks use the correct language identifier (e.g. ```python) rather + # than the generic ```code fallback. + meta = nb.get("metadata", {}) + lang = ( + meta.get("language_info", {}).get("name") + or meta.get("kernelspec", {}).get("language") + or "code" + ) lines = [] for cell in nb.get("cells", []): ct = cell.get("cell_type") - src = "".join(cell.get("source", [])) + raw_src = cell.get("source", []) + src = raw_src if isinstance(raw_src, str) else "".join(raw_src) if not src.strip(): continue if ct == "markdown": lines.append(src) elif ct == "code": - lines.append(f"```code\n{src}\n```") + lines.append(f"```{lang}\n{src}\n```") return "\n\n".join(lines) except Exception: return "" - + def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir.