Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,7 @@ GRAPHIFY_TRIAGE_BACKEND=kimi graphify prs --triage # use a specific backend fo
graphify clone https://github.com/karpathy/nanoGPT
graphify merge-graphs a.json b.json --out merged.json
graphify merge-graphs a.json b.json --repo-tag api --repo-tag web --out merged.json
graphify --version # print installed version
graphify watch ./src
graphify check-update ./src
Expand Down
57 changes: 33 additions & 24 deletions graphify/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2282,6 +2282,7 @@ def main() -> None:
print(" merge-driver <base> <current> <other> git merge driver: union-merge two graph.json files (set up via hook install)")
print(" merge-graphs <g1> <g2> merge two or more graph.json files into one cross-repo graph")
print(" --out <path> output path (default: graphify-out/merged-graph.json)")
print(" --repo-tag <tag> repository tag for each input, in input order (repeatable)")
print(" --branch <branch> checkout a specific branch (default: repo default)")
print(" --out <dir> clone to a custom directory (default: ~/.graphify/repos/<owner>/<repo>)")
print(" add <url> fetch a URL and save it to ./raw, then update the graph")
Expand Down Expand Up @@ -3890,24 +3891,37 @@ def _load_graph(p: str):
# graphify merge-graphs graph1.json graph2.json ... --out merged.json
args = sys.argv[2:]
graph_paths: list[Path] = []
repo_tags: list[str] = []
out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json"
i = 0
while i < len(args):
if args[i] == "--out" and i + 1 < len(args):
out_path = Path(args[i + 1])
i += 2
elif args[i] == "--repo-tag" and i + 1 < len(args):
repo_tags.append(args[i + 1])
i += 2
else:
graph_paths.append(Path(args[i]))
i += 1
if len(graph_paths) < 2:
print(
"Usage: graphify merge-graphs <graph1.json> <graph2.json> [...] [--out merged.json]",
"Usage: graphify merge-graphs <graph1.json> <graph2.json> [...] "
"[--repo-tag TAG ...] [--out merged.json]",
file=sys.stderr,
)
sys.exit(1)
if repo_tags and len(repo_tags) != len(graph_paths):
print(
"error: --repo-tag must be repeated once for every input graph",
file=sys.stderr,
)
sys.exit(1)
import networkx as _nx
from networkx.readwrite import json_graph as _jg
from graphify.build import prefix_graph_for_global as _prefix
from graphify.build import (
compose_repository_graphs as _compose_repository_graphs,
derive_repository_tag as _derive_repository_tag,
)
graphs = []
for gp in graph_paths:
if not gp.exists():
Expand All @@ -3923,34 +3937,29 @@ def _load_graph(p: str):
G = _jg.node_link_graph(data, edges="links")
except TypeError:
G = _jg.node_link_graph(data)
G.graph["hyperedges"] = data.get(
"hyperedges", G.graph.get("hyperedges", [])
)
graphs.append(G)
# nx.compose requires all graphs to be the same type. When input graphs
# come from different sources (e.g. an AST-only run vs a full LLM run) one
# may be a MultiGraph and another a Graph. Normalise everything to Graph
# (the graphify default) by converting MultiGraphs with nx.Graph().
def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
# nx.compose requires every graph to be the same type. Inputs may
# disagree on BOTH axes — directed vs undirected, and multi vs simple
# — because per-repo graph.json files are written by different extract
# paths at different times. Normalise everything to a plain undirected
# Graph (the merged cross-repo view is undirected anyway), which covers
# DiGraph / MultiGraph / MultiDiGraph. Without this a directed input
# crashed compose with "All graphs must be directed or undirected" (#1606).
if type(g) is not _nx.Graph:
return _nx.Graph(g)
return g
merged = _nx.Graph()
for G, gp in zip(graphs, graph_paths):
repo_tag = gp.parent.parent.name # graphify-out/../ → repo dir name
prefixed = _to_simple(_prefix(G, repo_tag))
merged = _nx.compose(merged, prefixed)
if not repo_tags:
repo_tags = [_derive_repository_tag(path) for path in graph_paths]
try:
merged = _compose_repository_graphs(list(zip(repo_tags, graphs)))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
sys.exit(1)
try:
out_data = _jg.node_link_data(merged, edges="links")
except TypeError:
out_data = _jg.node_link_data(merged)
out_data["hyperedges"] = merged.graph.get("hyperedges", [])
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8")
print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges")
print(
f"Merged {len(graphs)} graphs [{', '.join(repo_tags)}] -> "
f"{merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges, "
f"{len(merged.graph.get('hyperedges', []))} hyperedges"
)
print(f"Written to: {out_path}")

elif cmd == "clone":
Expand Down
80 changes: 80 additions & 0 deletions graphify/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
# before any graph construction happens.
#
from __future__ import annotations
import copy
import json
import os
import re
Expand Down Expand Up @@ -925,6 +926,85 @@ def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph:
return H


def derive_repository_tag(graph_path: Path) -> str:
"""Derive a readable repository tag from a graph path.

Standard ``<repo>/graphify-out/graph.json`` inputs use the repository
directory. Flat JSON inputs use the filename stem instead of the empty tag
produced by ``path.parent.parent.name``.
"""
if graph_path.name == "graph.json" and graph_path.parent.name == "graphify-out":
return graph_path.parent.parent.resolve().name
return graph_path.stem or graph_path.parent.name


def _community_identity(value: object) -> str:
"""Return a stable, sortable identity for a repository-local community id."""
try:
return json.dumps(value, ensure_ascii=False, sort_keys=True)
except TypeError:
return repr(value)


def compose_repository_graphs(tagged_graphs: list[tuple[str, nx.Graph]]) -> nx.Graph:
"""Compose repository graphs without conflating repository-local identity.

Node and hyperedge ids are namespaced by repository tag. Community ids are
remapped deterministically into one non-overlapping integer range while
preserving each repository's existing partition.
"""
tags = [tag for tag, _graph in tagged_graphs]
invalid_tags = [tag for tag in tags if not tag or "::" in tag]
if invalid_tags:
raise ValueError("repository tags must be non-empty and cannot contain '::'")
duplicate_tags = sorted({tag for tag in tags if tags.count(tag) > 1})
if duplicate_tags:
raise ValueError(
"duplicate repository tag(s): " + ", ".join(repr(tag) for tag in duplicate_tags)
)

community_keys = {
(tag, _community_identity(data["community"]))
for tag, graph in tagged_graphs
for _node, data in graph.nodes(data=True)
if data.get("community") is not None
}
community_ids = {
key: community_id for community_id, key in enumerate(sorted(community_keys))
}

merged = nx.Graph()
merged_hyperedges: list = []
for tag, source_graph in tagged_graphs:
graph = source_graph if type(source_graph) is nx.Graph else nx.Graph(source_graph)
prefixed = prefix_graph_for_global(graph, tag)

for _node, data in prefixed.nodes(data=True):
local_community = data.get("community")
if local_community is not None:
merged_community = community_ids[(tag, _community_identity(local_community))]
data["community"] = merged_community
if data.get("community_name") == f"Community {local_community}":
data["community_name"] = f"Community {merged_community}"

for raw_hyperedge in graph.graph.get("hyperedges", []) or []:
hyperedge = copy.deepcopy(raw_hyperedge)
if not isinstance(hyperedge, dict):
merged_hyperedges.append(hyperedge)
continue
_normalize_hyperedge_members(hyperedge)
if hyperedge.get("id") is not None:
hyperedge["id"] = f"{tag}::{hyperedge['id']}"
if isinstance(hyperedge.get("nodes"), list):
hyperedge["nodes"] = [f"{tag}::{node}" for node in hyperedge["nodes"]]
merged_hyperedges.append(hyperedge)

merged = nx.compose(merged, prefixed)

merged.graph["hyperedges"] = merged_hyperedges
return merged


def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int:
"""Remove all nodes tagged with repo_tag from G in-place. Returns count removed."""
to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag]
Expand Down
151 changes: 150 additions & 1 deletion tests/test_merge_graphs_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,35 @@ def _write(p: Path, directed: bool, multigraph: bool, node_id: str):
}))


def _write_repository_graph(p: Path, label: str):
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(json.dumps({
"directed": True,
"multigraph": False,
"graph": {},
"nodes": [
{
"id": "entry",
"label": f"{label} entry",
"community": 0,
"community_name": "Community 0",
},
{
"id": "worker",
"label": f"{label} worker",
"community": 1,
"community_name": f"{label} workers",
},
],
"links": [
{"source": "entry", "target": "worker", "relation": "calls"},
],
"hyperedges": [
{"id": "request_flow", "nodes": ["entry", "worker"], "relation": "flow"},
],
}))


def test_merge_graphs_mixed_directed_and_multigraph(tmp_path):
a = tmp_path / "r1" / "graphify-out" / "graph.json"
b = tmp_path / "r2" / "graphify-out" / "graph.json"
Expand All @@ -43,6 +72,126 @@ def test_merge_graphs_mixed_directed_and_multigraph(tmp_path):
data = json.loads(out.read_text())
ids = {n["id"] for n in data["nodes"]}
# every input's node survives, normalized into one undirected simple graph
assert {"r1::x", "r2::y", "r3::z"} <= ids or len(ids) == 3
assert ids == {"r1::x", "r2::y", "r3::z"}
assert data.get("directed") is False
assert data.get("multigraph") is False


def test_merge_graphs_preserves_repository_local_identity(tmp_path):
a = tmp_path / "a.json"
b = tmp_path / "b.json"
_write_repository_graph(a, "A")
_write_repository_graph(b, "B")
out = tmp_path / "merged.json"

r = _run(["merge-graphs", str(a), str(b), "--out", str(out)], tmp_path)

assert r.returncode == 0, r.stderr
data = json.loads(out.read_text())
nodes = {node["id"]: node for node in data["nodes"]}
assert set(nodes) == {"a::entry", "a::worker", "b::entry", "b::worker"}
assert {nodes["a::entry"]["community"], nodes["a::worker"]["community"]}.isdisjoint(
{nodes["b::entry"]["community"], nodes["b::worker"]["community"]}
)
assert nodes["b::entry"]["community_name"] == (
f"Community {nodes['b::entry']['community']}"
)
assert nodes["b::worker"]["community_name"] == "B workers"
assert {
(edge["source"], edge["target"], edge["relation"])
for edge in data["links"]
} == {
("a::entry", "a::worker", "calls"),
("b::entry", "b::worker", "calls"),
}
assert {
hyperedge["id"]: hyperedge["nodes"] for hyperedge in data["hyperedges"]
} == {
"a::request_flow": ["a::entry", "a::worker"],
"b::request_flow": ["b::entry", "b::worker"],
}
assert "4 nodes, 2 edges, 2 hyperedges" in r.stdout


def test_merge_graphs_derives_tag_from_relative_canonical_path(tmp_path):
a = tmp_path / "repo-a" / "graphify-out" / "graph.json"
b = tmp_path / "repo-b" / "graphify-out" / "graph.json"
_write_repository_graph(a, "A")
_write_repository_graph(b, "B")

r = _run([
"merge-graphs", "graphify-out/graph.json",
"../repo-b/graphify-out/graph.json", "--out", "merged.json",
], tmp_path / "repo-a")

assert r.returncode == 0, r.stderr
ids = {node["id"] for node in json.loads((tmp_path / "repo-a/merged.json").read_text())["nodes"]}
assert {"repo-a::entry", "repo-b::entry"} <= ids


def test_merge_graphs_community_remap_is_input_order_independent(tmp_path):
a = tmp_path / "a.json"
b = tmp_path / "b.json"
_write_repository_graph(a, "A")
_write_repository_graph(b, "B")
out_ab = tmp_path / "merged-ab.json"
out_ba = tmp_path / "merged-ba.json"

first = _run(["merge-graphs", str(a), str(b), "--out", str(out_ab)], tmp_path)
second = _run(["merge-graphs", str(b), str(a), "--out", str(out_ba)], tmp_path)

assert first.returncode == second.returncode == 0
communities_ab = {
node["id"]: node.get("community")
for node in json.loads(out_ab.read_text())["nodes"]
}
communities_ba = {
node["id"]: node.get("community")
for node in json.loads(out_ba.read_text())["nodes"]
}
assert communities_ab == communities_ba


def test_merge_graphs_rejects_duplicate_derived_tags(tmp_path):
a = tmp_path / "owner-a" / "service" / "graphify-out" / "graph.json"
b = tmp_path / "owner-b" / "service" / "graphify-out" / "graph.json"
_write_repository_graph(a, "A")
_write_repository_graph(b, "B")
out = tmp_path / "merged.json"

r = _run(["merge-graphs", str(a), str(b), "--out", str(out)], tmp_path)

assert r.returncode != 0
assert "duplicate repository tag(s): 'service'" in r.stderr
assert not out.exists()


def test_merge_graphs_accepts_explicit_repository_tags(tmp_path):
a = tmp_path / "owner-a" / "service" / "graphify-out" / "graph.json"
b = tmp_path / "owner-b" / "service" / "graphify-out" / "graph.json"
_write_repository_graph(a, "A")
_write_repository_graph(b, "B")
out = tmp_path / "merged.json"

r = _run([
"merge-graphs", str(a), str(b),
"--repo-tag", "owner-a-service", "--repo-tag", "owner-b-service",
"--out", str(out),
], tmp_path)

assert r.returncode == 0, r.stderr
ids = {node["id"] for node in json.loads(out.read_text())["nodes"]}
assert "owner-a-service::entry" in ids
assert "owner-b-service::entry" in ids


def test_merge_graphs_requires_one_explicit_tag_per_input(tmp_path):
a = tmp_path / "a.json"
b = tmp_path / "b.json"
_write_repository_graph(a, "A")
_write_repository_graph(b, "B")

r = _run(["merge-graphs", str(a), str(b), "--repo-tag", "a"], tmp_path)

assert r.returncode != 0
assert "--repo-tag must be repeated once for every input graph" in r.stderr
Loading