Codex
82890606aa
Selectively port source hashing, locked manifest/memory writes, path-aware link lint and graph analysis, and revision-checked staging from upstream 4f457e154653756b04802d5aeb25c20f42b4770c. Record source hashes and exclusions. Preserve local archive/catalog citations, curated manifest metadata, config location, file-index and MCP behavior. Wire maintenance guidance into four skills without migrating vault contents or changing running services. Validation: 609 tests passed, including existing MCP/file-asset regressions and imported upstream maintenance tests. Read-only vault status left shared metadata hashes unchanged. Wiki MCP remains inactive; orders MCP is active.
915 lines
34 KiB
Python
915 lines
34 KiB
Python
"""Vault graph analysis: community detection, god nodes, bridges, surprising connections.
|
|
|
|
Reads the vault's wikilink graph (from page frontmatter and [[wikilinks]]),
|
|
then runs the same family of algorithms graphify uses on code graphs — all in
|
|
pure Python (no networkx / numpy):
|
|
1. Degree-based god-node ranking (most-linked pages)
|
|
2. Community detection — Leiden via `obsidian-wiki[graph]` if installed,
|
|
else greedy label propagation (pure Python, no binary deps)
|
|
3. Community cohesion — intra-community edge density (graphify
|
|
`cohesion_score`); < 0.15 flags a fragmented cluster
|
|
4. Bridge pages — Brandes betweenness centrality; the pages that sit on
|
|
the most shortest paths between other pages (graphify's bridge nodes)
|
|
5. Surprising connections — cross-community edges ranked by unexpectedness,
|
|
de-duplicated one-per-community-pair so a single hub can't dominate
|
|
6. Suggested questions — derived from bridges, isolates, dead-end hubs and
|
|
low-cohesion communities (graphify `suggest_questions`)
|
|
7. Path finding / neighbourhoods — `shortest_path` and BFS `neighborhood`
|
|
(graphify `path` and blast-radius / `affected`)
|
|
8. Graph diff — compare against a previous snapshot (graphify `graph_diff`)
|
|
|
|
Output JSON:
|
|
{
|
|
"god_nodes": [{"page": "...", "degree": N, "in_degree": N, "out_degree": N}, ...],
|
|
"bridges": [{"page": "...", "betweenness": 0.N, "connects": [comm_id, ...]}, ...],
|
|
"communities": [{"id": N, "size": N, "pages": [...], "label": "...", "cohesion": 0.N}, ...],
|
|
"surprising_connections": [{"source": "...", "target": "...", "score": 0.N,
|
|
"note": "bridges community A -> community B"}, ...],
|
|
"suggested_questions": [{"type": "...", "question": "...", "why": "..."}, ...],
|
|
"dead_ends": ["page with 0 outgoing links", ...],
|
|
"isolated": ["page with 0 links at all", ...],
|
|
"stats": {"pages": N, "edges": N, "communities": N, "density": 0.N}
|
|
}
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import random
|
|
import re
|
|
import time
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from obsidian_wiki.links import PageIndex, strip_code
|
|
from obsidian_wiki.provenance import archive_wikilink_relpath
|
|
from obsidian_wiki.vault import SKIP_DIRS as VAULT_SKIP_DIRS
|
|
from obsidian_wiki.vault import iter_md, okignore_patterns, okignored, skipped_dir # noqa: F401 (re-exported)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wikilink / frontmatter parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_WIKILINK_RE = re.compile(r"\[\[([^\]|#]+?)(?:[|#][^\]]*?)?\]\]")
|
|
# Local `.md` targets only: skip URLs (any `scheme:`) and require `.md` to end
|
|
# the path, before an optional `#anchor` or `"title"`.
|
|
_MD_LINK_RE = re.compile(
|
|
r"\[.*?\]\((?![A-Za-z][A-Za-z0-9+.-]*:)([^)#\s]+\.md)(?:[#\s][^)]*)?\)"
|
|
)
|
|
_TAGS_RE = re.compile(r"^tags:\s*\[([^\]]+)\]", re.MULTILINE)
|
|
_TAGS_LIST_RE = re.compile(r"^tags:\s*\n((?:\s+-\s+\S+\n)+)", re.MULTILINE)
|
|
|
|
|
|
def _slug(page_name: str) -> str:
|
|
"""Normalise a wikilink target to a page slug."""
|
|
return page_name.strip().lower().replace(" ", "-")
|
|
|
|
|
|
def _page_slug(path: Path, root: Path) -> str:
|
|
return _slug(path.stem)
|
|
|
|
|
|
# --- Shared page selection -------------------------------------------------
|
|
# Every module that builds a graph over the vault MUST use these, so the
|
|
# wikilink graph is identical no matter which entry point produced it.
|
|
|
|
#: Directories that hold staging / archive / config material, not knowledge.
|
|
#: Dot-prefixed paths (`.venv`, `.git`) are skipped wholesale by `is_wiki_page`;
|
|
#: this list also covers the non-hidden equivalents (`venv/`, `node_modules/`).
|
|
SKIP_DIRS = VAULT_SKIP_DIRS | {"_meta", "_readouts"}
|
|
|
|
#: Vault bookkeeping files at the vault ROOT. They link to (almost) every page,
|
|
#: so leaving them in makes every pair of pages look ~2 hops apart and lets a
|
|
#: table of contents outrank real knowledge in every degree/centrality ranking.
|
|
#: Only the root-level files are skipped — a legitimate `concepts/index.md`
|
|
#: is still a normal page.
|
|
SKIP_ROOT_FILES = frozenset({"index", "log", "hot", "_insights"})
|
|
|
|
|
|
def is_wiki_page(path: Path, vault: Path) -> bool:
|
|
"""True if `path` is a knowledge page of `vault` (not staging/bookkeeping)."""
|
|
try:
|
|
rel = path.relative_to(vault)
|
|
except ValueError:
|
|
return False
|
|
if skipped_dir(rel, SKIP_DIRS):
|
|
return False
|
|
if len(rel.parts) == 1 and path.stem in SKIP_ROOT_FILES:
|
|
return False
|
|
return True
|
|
|
|
|
|
def iter_pages(vault: Path) -> list[Path]:
|
|
"""All knowledge pages in the vault, in stable sorted order."""
|
|
return [p for p in iter_md(vault, SKIP_DIRS) if is_wiki_page(p, vault)]
|
|
|
|
|
|
def parse_vault_graph(vault: Path) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
|
|
"""Parse all .md pages and return (outgoing_edges, page_tags).
|
|
|
|
outgoing_edges: {source_slug: [target_slug, ...]}
|
|
page_tags: {slug: [tag, ...]}
|
|
"""
|
|
outgoing: dict[str, list[str]] = defaultdict(list)
|
|
tags_map: dict[str, list[str]] = {}
|
|
|
|
pages: list[Path] = iter_pages(vault)
|
|
rels = {page: page.relative_to(vault).as_posix() for page in pages}
|
|
index = PageIndex(rels.values())
|
|
|
|
for page in pages:
|
|
src = index.id_for(rels[page])
|
|
try:
|
|
text = page.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
continue
|
|
links_text = strip_code(text)
|
|
|
|
# Tags
|
|
m = _TAGS_RE.search(text)
|
|
if m:
|
|
tags_map[src] = [t.strip().strip("'\"") for t in m.group(1).split(",")]
|
|
else:
|
|
m2 = _TAGS_LIST_RE.search(text)
|
|
if m2:
|
|
tags_map[src] = [ln.strip().lstrip("- ") for ln in m2.group(1).splitlines() if ln.strip()]
|
|
|
|
# Wikilinks
|
|
for link in _WIKILINK_RE.findall(links_text):
|
|
if archive_wikilink_relpath(vault, link) is not None:
|
|
continue
|
|
target = index.resolve(link, rels[page])
|
|
if target and target != src:
|
|
outgoing[src].append(target)
|
|
|
|
# Markdown links (when OBSIDIAN_LINK_FORMAT=markdown)
|
|
for href in _MD_LINK_RE.findall(links_text):
|
|
if archive_wikilink_relpath(vault, href) is not None:
|
|
continue
|
|
target = index.resolve(href, rels[page])
|
|
if target and target != src:
|
|
outgoing[src].append(target)
|
|
|
|
if src not in outgoing:
|
|
outgoing[src] = []
|
|
|
|
# Ensure every known page appears as a key
|
|
for page_id in index.ids():
|
|
outgoing.setdefault(page_id, [])
|
|
|
|
return dict(outgoing), tags_map
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Graph metrics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_degree_tables(
|
|
outgoing: dict[str, list[str]]
|
|
) -> tuple[dict[str, int], dict[str, int], dict[str, int]]:
|
|
out_deg: dict[str, int] = {n: len(v) for n, v in outgoing.items()}
|
|
in_deg: dict[str, int] = defaultdict(int)
|
|
for targets in outgoing.values():
|
|
for t in targets:
|
|
in_deg[t] += 1
|
|
degree = {n: out_deg.get(n, 0) + in_deg.get(n, 0) for n in outgoing}
|
|
for n in in_deg:
|
|
degree.setdefault(n, in_deg[n])
|
|
return degree, dict(in_deg), out_deg
|
|
|
|
|
|
def god_nodes(outgoing: dict[str, list[str]], top_n: int = 20) -> list[dict]:
|
|
degree, in_deg, out_deg = _build_degree_tables(outgoing)
|
|
ranked = sorted(degree, key=lambda n: -degree[n])[:top_n]
|
|
return [
|
|
{"page": n, "degree": degree[n],
|
|
"in_degree": in_deg.get(n, 0), "out_degree": out_deg.get(n, 0)}
|
|
for n in ranked
|
|
]
|
|
|
|
|
|
def dead_ends(outgoing: dict[str, list[str]]) -> list[str]:
|
|
"""Pages with no outgoing links."""
|
|
return [n for n, targets in outgoing.items() if not targets]
|
|
|
|
|
|
def isolated(outgoing: dict[str, list[str]]) -> list[str]:
|
|
"""Pages with zero links in either direction."""
|
|
_, in_deg, out_deg = _build_degree_tables(outgoing)
|
|
return [n for n in outgoing if in_deg.get(n, 0) == 0 and out_deg.get(n, 0) == 0]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Community detection — greedy modularity (Newman-Girvan, pure Python)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _modularity(communities: list[set[str]], outgoing: dict[str, list[str]]) -> float:
|
|
"""Approximate modularity Q for an undirected projection of outgoing edges."""
|
|
all_edges = [(s, t) for s, targets in outgoing.items() for t in targets]
|
|
m = len(all_edges)
|
|
if m == 0:
|
|
return 0.0
|
|
degree = defaultdict(int)
|
|
for s, t in all_edges:
|
|
degree[s] += 1
|
|
degree[t] += 1
|
|
node_comm: dict[str, int] = {}
|
|
for i, comm in enumerate(communities):
|
|
for n in comm:
|
|
node_comm[n] = i
|
|
Q = 0.0
|
|
for s, t in all_edges:
|
|
if node_comm.get(s) == node_comm.get(t):
|
|
Q += 1 - (degree[s] * degree[t]) / (2 * m)
|
|
return Q / m
|
|
|
|
|
|
def detect_communities_greedy(outgoing: dict[str, list[str]]) -> list[set[str]]:
|
|
"""Greedy modularity community detection (label propagation variant).
|
|
|
|
Fast O(n·k) approach suitable for vaults up to ~5 000 pages. Each node
|
|
adopts the most frequent label among its neighbours; iterate until stable.
|
|
Falls back gracefully to one community per page if the graph is empty.
|
|
"""
|
|
nodes = list(outgoing.keys())
|
|
if not nodes:
|
|
return []
|
|
|
|
# Build undirected adjacency
|
|
adj: dict[str, list[str]] = defaultdict(list)
|
|
for src, targets in outgoing.items():
|
|
for t in targets:
|
|
adj[src].append(t)
|
|
adj[t].append(src)
|
|
|
|
# Initialise: each node in its own community (label = index)
|
|
labels: dict[str, int] = {n: i for i, n in enumerate(nodes)}
|
|
|
|
# Visit nodes in a shuffled order each round. A fixed sequential sweep
|
|
# cascades one label along chains and collapses well-separated clusters
|
|
# into a single community (a barbell graph comes back as one group). The
|
|
# RNG is seeded from the node set, so the result stays reproducible.
|
|
rng = random.Random(len(nodes))
|
|
order = list(nodes)
|
|
|
|
for _ in range(20): # max 20 rounds
|
|
changed = False
|
|
rng.shuffle(order)
|
|
for n in order:
|
|
neighbours = adj[n]
|
|
if not neighbours:
|
|
continue
|
|
freq: dict[int, int] = defaultdict(int)
|
|
for nb in neighbours:
|
|
freq[labels[nb]] += 1
|
|
best = max(freq, key=lambda lbl: (freq[lbl], -lbl))
|
|
if best != labels[n]:
|
|
labels[n] = best
|
|
changed = True
|
|
if not changed:
|
|
break
|
|
|
|
# Group by label
|
|
groups: dict[int, set[str]] = defaultdict(set)
|
|
for n, lbl in labels.items():
|
|
groups[lbl].add(n)
|
|
return list(groups.values())
|
|
|
|
|
|
def detect_communities(outgoing: dict[str, list[str]]) -> list[set[str]]:
|
|
"""Try Leiden (leidenalg + igraph) first; fall back to greedy label propagation."""
|
|
try:
|
|
import igraph as ig
|
|
import leidenalg
|
|
|
|
nodes = list(outgoing.keys())
|
|
node_idx = {n: i for i, n in enumerate(nodes)}
|
|
edges = [
|
|
(node_idx[s], node_idx[t])
|
|
for s, targets in outgoing.items()
|
|
for t in targets
|
|
if t in node_idx
|
|
]
|
|
g = ig.Graph(n=len(nodes), edges=edges, directed=False)
|
|
partition = leidenalg.find_partition(g, leidenalg.ModularityVertexPartition)
|
|
return [
|
|
{nodes[i] for i in cluster}
|
|
for cluster in partition
|
|
]
|
|
except ImportError:
|
|
return detect_communities_greedy(outgoing)
|
|
|
|
|
|
def _undirected_adj(outgoing: dict[str, list[str]]) -> dict[str, set[str]]:
|
|
"""Undirected simple-graph adjacency (duplicate/self edges collapsed)."""
|
|
adj: dict[str, set[str]] = {n: set() for n in outgoing}
|
|
for src, targets in outgoing.items():
|
|
for t in targets:
|
|
if t == src:
|
|
continue
|
|
adj[src].add(t)
|
|
adj.setdefault(t, set()).add(src)
|
|
return adj
|
|
|
|
|
|
def _node_community_map(communities: list[set[str]]) -> dict[str, int]:
|
|
return {n: i for i, comm in enumerate(communities) for n in comm}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Community cohesion (graphify: cluster.cohesion_score)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def cohesion_score(outgoing: dict[str, list[str]], community: set[str]) -> float:
|
|
"""Ratio of actual intra-community (undirected) edges to the maximum possible.
|
|
|
|
1.0 = clique, 0.0 = no internal links. Singletons score 1.0.
|
|
"""
|
|
n = len(community)
|
|
if n <= 1:
|
|
return 1.0
|
|
adj = _undirected_adj(outgoing)
|
|
actual = sum(1 for a in community for b in adj.get(a, ()) if b in community and a < b)
|
|
possible = n * (n - 1) / 2
|
|
return round(actual / possible, 4) if possible else 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Betweenness centrality — Brandes (2001), pure Python
|
|
# (graphify: nx.betweenness_centrality → "bridge nodes")
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def betweenness_centrality(
|
|
outgoing: dict[str, list[str]],
|
|
normalized: bool = True,
|
|
) -> dict[str, float]:
|
|
"""Exact node betweenness on the undirected projection (Brandes algorithm).
|
|
|
|
O(V·E) — fine for vaults up to a few thousand pages. Values are normalised
|
|
to [0, 1] by 2/((n-1)(n-2)) so scores are comparable across vault sizes.
|
|
"""
|
|
adj = _undirected_adj(outgoing)
|
|
nodes = list(adj)
|
|
bc: dict[str, float] = {n: 0.0 for n in nodes}
|
|
for s in nodes:
|
|
stack: list[str] = []
|
|
preds: dict[str, list[str]] = {n: [] for n in nodes}
|
|
sigma: dict[str, float] = {n: 0.0 for n in nodes}
|
|
dist: dict[str, int] = {n: -1 for n in nodes}
|
|
sigma[s] = 1.0
|
|
dist[s] = 0
|
|
queue = [s]
|
|
qi = 0
|
|
while qi < len(queue):
|
|
v = queue[qi]
|
|
qi += 1
|
|
stack.append(v)
|
|
for w in adj[v]:
|
|
if dist[w] < 0:
|
|
dist[w] = dist[v] + 1
|
|
queue.append(w)
|
|
if dist[w] == dist[v] + 1:
|
|
sigma[w] += sigma[v]
|
|
preds[w].append(v)
|
|
delta: dict[str, float] = {n: 0.0 for n in nodes}
|
|
while stack:
|
|
w = stack.pop()
|
|
for v in preds[w]:
|
|
delta[v] += (sigma[v] / sigma[w]) * (1.0 + delta[w])
|
|
if w != s:
|
|
bc[w] += delta[w]
|
|
n = len(nodes)
|
|
# Undirected: every pair counted twice
|
|
scale = 0.5
|
|
if normalized and n > 2:
|
|
scale = 1.0 / ((n - 1) * (n - 2))
|
|
return {k: v * scale for k, v in bc.items()}
|
|
|
|
|
|
# --- Betweenness cache -----------------------------------------------------
|
|
# Betweenness is the ONLY expensive step in this module: O(V*E), ~42s on a
|
|
# 5 000-page vault, versus 0.13s to parse it and 0.03s for communities. So it
|
|
# is the only thing worth caching.
|
|
#
|
|
# The cache key is a hash of the GRAPH ITSELF, not file mtimes. That makes a
|
|
# stale hit impossible by construction: if any link changed, the key changes;
|
|
# if only prose changed, the old (still-correct) result is reused. No
|
|
# invalidation logic to get wrong.
|
|
|
|
CACHE_FILENAME = ".graph-cache.json"
|
|
_CACHE_MAX_ENTRIES = 3
|
|
#: Don't write a cache file for graphs that recompute faster than this — small
|
|
#: vaults should not accumulate cache files for no gain.
|
|
_CACHE_MIN_SECONDS = 0.5
|
|
|
|
|
|
def graph_fingerprint(outgoing: dict[str, list[str]]) -> str:
|
|
"""Stable hash of the undirected graph topology (nodes + edges)."""
|
|
adj = _undirected_adj(outgoing)
|
|
h = hashlib.sha256()
|
|
for node in sorted(adj):
|
|
h.update(node.encode("utf-8"))
|
|
h.update(b"\x00")
|
|
for nb in sorted(adj[node]):
|
|
h.update(nb.encode("utf-8"))
|
|
h.update(b"\x01")
|
|
h.update(b"\x02")
|
|
return h.hexdigest()[:32]
|
|
|
|
|
|
def _cache_path(vault: Path) -> Path:
|
|
return vault / CACHE_FILENAME
|
|
|
|
|
|
def _read_cache(vault: Path) -> dict:
|
|
"""Read the cache, failing open (a corrupt or unreadable cache is ignored)."""
|
|
try:
|
|
data = json.loads(_cache_path(vault).read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError, ValueError):
|
|
return {}
|
|
return data if isinstance(data, dict) and data.get("version") == 1 else {}
|
|
|
|
|
|
def _write_cache(vault: Path, data: dict) -> None:
|
|
"""Atomically replace the cache file. Never raises — caching is best-effort."""
|
|
target = _cache_path(vault)
|
|
tmp = target.with_suffix(f".tmp{os.getpid()}")
|
|
try:
|
|
tmp.write_text(json.dumps(data), encoding="utf-8")
|
|
os.replace(tmp, target) # atomic; concurrent writers can't tear the file
|
|
except OSError:
|
|
try:
|
|
tmp.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def betweenness_cached(
|
|
outgoing: dict[str, list[str]],
|
|
vault: Path | None = None,
|
|
normalized: bool = True,
|
|
) -> dict[str, float]:
|
|
"""`betweenness_centrality`, memoised on disk under `vault` by graph topology.
|
|
|
|
Falls back to a plain in-process computation when `vault` is None or the
|
|
cache is unusable — the result is always identical either way.
|
|
"""
|
|
if vault is None:
|
|
return betweenness_centrality(outgoing, normalized=normalized)
|
|
|
|
key = f"{graph_fingerprint(outgoing)}:{int(normalized)}"
|
|
cache = _read_cache(vault)
|
|
hit = cache.get("betweenness", {}).get(key)
|
|
if isinstance(hit, dict) and hit.keys() >= outgoing.keys():
|
|
return {n: float(hit[n]) for n in outgoing}
|
|
|
|
started = time.monotonic()
|
|
scores = betweenness_centrality(outgoing, normalized=normalized)
|
|
elapsed = time.monotonic() - started
|
|
|
|
if elapsed >= _CACHE_MIN_SECONDS:
|
|
entries = cache.get("betweenness", {})
|
|
if not isinstance(entries, dict):
|
|
entries = {}
|
|
entries[key] = {n: round(v, 10) for n, v in scores.items()}
|
|
# Keep the cache bounded — most recent keys win.
|
|
for old in list(entries)[:-_CACHE_MAX_ENTRIES]:
|
|
entries.pop(old, None)
|
|
_write_cache(vault, {"version": 1, "betweenness": entries})
|
|
return scores
|
|
|
|
|
|
def bridge_pages(
|
|
outgoing: dict[str, list[str]],
|
|
communities: list[set[str]],
|
|
top_n: int = 10,
|
|
vault: Path | None = None,
|
|
) -> list[dict]:
|
|
"""Pages with the highest betweenness — removing them fragments the graph.
|
|
|
|
Each entry lists which communities the page's neighbours belong to, so the
|
|
caller can say "`P` bridges cluster A ↔ cluster B".
|
|
"""
|
|
bc = betweenness_cached(outgoing, vault)
|
|
if not bc:
|
|
return []
|
|
adj = _undirected_adj(outgoing)
|
|
node_comm = _node_community_map(communities)
|
|
ranked = sorted((n for n, s in bc.items() if s > 0), key=lambda n: -bc[n])[:top_n]
|
|
out = []
|
|
for n in ranked:
|
|
own = node_comm.get(n)
|
|
others = sorted({node_comm[nb] for nb in adj[n] if nb in node_comm and node_comm[nb] != own})
|
|
out.append({
|
|
"page": n,
|
|
"betweenness": round(bc[n], 4),
|
|
"community": own,
|
|
"connects": others,
|
|
})
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Surprising connections
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def surprising_connections(
|
|
outgoing: dict[str, list[str]],
|
|
communities: list[set[str]],
|
|
top_n: int = 20,
|
|
) -> list[dict]:
|
|
"""Edges that cross community boundaries, ranked by unexpectedness.
|
|
|
|
Score = 1 / sqrt(cross_degree(source) * cross_degree(target))
|
|
Low cross-degree nodes connected across communities are the most surprising.
|
|
|
|
Results are ordered so that each (community A, community B) boundary is
|
|
represented once before any boundary repeats — graphify's dedup, which
|
|
stops one high-betweenness hub from filling every slot.
|
|
"""
|
|
node_comm = _node_community_map(communities)
|
|
|
|
# Count how many cross-community edges each node already has
|
|
cross_deg: dict[str, int] = defaultdict(int)
|
|
for src, targets in outgoing.items():
|
|
for t in targets:
|
|
if node_comm.get(src) != node_comm.get(t):
|
|
cross_deg[src] += 1
|
|
cross_deg[t] += 1
|
|
|
|
results = []
|
|
seen: set[tuple[str, str]] = set()
|
|
for src, targets in outgoing.items():
|
|
for t in targets:
|
|
pair = tuple(sorted((src, t)))
|
|
if pair in seen:
|
|
continue
|
|
if node_comm.get(src) != node_comm.get(t):
|
|
cd_s = cross_deg.get(src, 1)
|
|
cd_t = cross_deg.get(t, 1)
|
|
score = 1.0 / math.sqrt(cd_s * cd_t)
|
|
cs, ct = node_comm.get(src), node_comm.get(t)
|
|
results.append({
|
|
"source": src, "target": t, "score": round(score, 4),
|
|
"note": f"bridges community {cs} -> community {ct}",
|
|
"_pair": (min(cs, ct), max(cs, ct)) if cs is not None and ct is not None else None,
|
|
})
|
|
seen.add(pair)
|
|
|
|
results.sort(key=lambda x: -x["score"])
|
|
# One representative per community pair first, then the rest by score.
|
|
seen_pairs: set = set()
|
|
first: list[dict] = []
|
|
rest: list[dict] = []
|
|
for r in results:
|
|
pair = r.pop("_pair")
|
|
if pair not in seen_pairs:
|
|
seen_pairs.add(pair)
|
|
first.append(r)
|
|
else:
|
|
rest.append(r)
|
|
return (first + rest)[:top_n]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Path finding & neighbourhoods (graphify: `path`, `affected` blast radius)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _node_for(outgoing: dict[str, list[str]], term: str) -> str:
|
|
"""Node id for a page reference: an exact id, else the shortest id whose
|
|
path ends with it. Pages that share a name have path-qualified ids, so a
|
|
bare name has to be matched by suffix."""
|
|
term = "/".join(_slug(part) for part in term.strip().removesuffix(".md").split("/") if part)
|
|
if term in outgoing:
|
|
return term
|
|
matches = sorted((n for n in outgoing if n.endswith(f"/{term}")), key=lambda n: (n.count("/"), n))
|
|
return matches[0] if matches else term
|
|
|
|
|
|
def shortest_path(
|
|
outgoing: dict[str, list[str]],
|
|
source: str,
|
|
target: str,
|
|
directed: bool = False,
|
|
) -> list[str] | None:
|
|
"""BFS shortest path between two pages. Follows links in either direction
|
|
unless `directed=True`. Returns the node list, or None if unreachable."""
|
|
source, target = _node_for(outgoing, source), _node_for(outgoing, target)
|
|
if source not in outgoing or target not in outgoing:
|
|
return None
|
|
if source == target:
|
|
return [source]
|
|
if directed:
|
|
adj: dict[str, Any] = {n: set(v) for n, v in outgoing.items()}
|
|
else:
|
|
adj = _undirected_adj(outgoing)
|
|
prev: dict[str, str | None] = {source: None}
|
|
queue = [source]
|
|
qi = 0
|
|
while qi < len(queue):
|
|
v = queue[qi]
|
|
qi += 1
|
|
for w in sorted(adj.get(v, ())):
|
|
if w in prev:
|
|
continue
|
|
prev[w] = v
|
|
if w == target:
|
|
path = [w]
|
|
while prev[path[-1]] is not None:
|
|
path.append(prev[path[-1]]) # type: ignore[arg-type]
|
|
return path[::-1]
|
|
queue.append(w)
|
|
return None
|
|
|
|
|
|
def neighborhood(
|
|
outgoing: dict[str, list[str]],
|
|
seed: str,
|
|
depth: int = 2,
|
|
direction: str = "both",
|
|
) -> list[dict]:
|
|
"""Breadth-first neighbourhood of a page up to `depth` hops.
|
|
|
|
direction: "out" (pages this one links to), "in" (pages linking here —
|
|
the blast radius if this page is renamed/removed), or "both".
|
|
Returns [{"page", "depth", "via"}] ordered by depth then name.
|
|
"""
|
|
seed = _node_for(outgoing, seed)
|
|
if seed not in outgoing:
|
|
return []
|
|
incoming: dict[str, set[str]] = defaultdict(set)
|
|
for s, targets in outgoing.items():
|
|
for t in targets:
|
|
incoming[t].add(s)
|
|
|
|
def nbrs(n: str) -> set[str]:
|
|
out: set[str] = set()
|
|
if direction in ("out", "both"):
|
|
out |= set(outgoing.get(n, ()))
|
|
if direction in ("in", "both"):
|
|
out |= incoming.get(n, set())
|
|
out.discard(n)
|
|
return out
|
|
|
|
seen = {seed: 0}
|
|
via = {seed: None}
|
|
frontier = [seed]
|
|
for d in range(1, depth + 1):
|
|
nxt = []
|
|
for v in frontier:
|
|
for w in sorted(nbrs(v)):
|
|
if w not in seen:
|
|
seen[w] = d
|
|
via[w] = v
|
|
nxt.append(w)
|
|
frontier = nxt
|
|
if not frontier:
|
|
break
|
|
return [
|
|
{"page": n, "depth": d, "via": via[n]}
|
|
for n, d in sorted(seen.items(), key=lambda kv: (kv[1], kv[0]))
|
|
if n != seed
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Graph diff (graphify: analyze.graph_diff)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_SNAPSHOT_RE = re.compile(r"<!--\s*GRAPH_SNAPSHOT:\s*(\{.*?\})\s*-->", re.DOTALL)
|
|
|
|
|
|
def snapshot(outgoing: dict[str, list[str]]) -> dict[str, list]:
|
|
"""Compact, JSON-serialisable snapshot for embedding in `_insights.md`."""
|
|
edges = sorted({(s, t) for s, ts in outgoing.items() for t in ts})
|
|
return {"nodes": sorted(outgoing), "edges": [list(e) for e in edges]}
|
|
|
|
|
|
def load_snapshot(path: Path) -> dict[str, list] | None:
|
|
"""Read a snapshot from a JSON file or from the `<!-- GRAPH_SNAPSHOT: ... -->`
|
|
comment inside a previous `_insights.md`. Returns None if not found."""
|
|
try:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return None
|
|
m = _SNAPSHOT_RE.search(text)
|
|
raw = m.group(1) if m else text
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
if not isinstance(data, dict) or "nodes" not in data:
|
|
return None
|
|
return {"nodes": list(data.get("nodes", [])), "edges": [list(e) for e in data.get("edges", [])]}
|
|
|
|
|
|
def graph_diff(old: dict[str, list], new: dict[str, list]) -> dict[str, Any]:
|
|
"""What changed between two snapshots (see `snapshot`)."""
|
|
old_nodes, new_nodes = set(old.get("nodes", [])), set(new.get("nodes", []))
|
|
old_edges = {tuple(e) for e in old.get("edges", [])}
|
|
new_edges = {tuple(e) for e in new.get("edges", [])}
|
|
|
|
def in_targets(edges: set) -> set[str]:
|
|
return {t for _, t in edges}
|
|
|
|
old_in, new_in = in_targets(old_edges), in_targets(new_edges)
|
|
added_edges = sorted(new_edges - old_edges)
|
|
removed_edges = sorted(old_edges - new_edges)
|
|
return {
|
|
"added_pages": sorted(new_nodes - old_nodes),
|
|
"removed_pages": sorted(old_nodes - new_nodes),
|
|
"added_edges": [list(e) for e in added_edges],
|
|
"removed_edges": [list(e) for e in removed_edges],
|
|
"newly_connected": sorted((new_in - old_in) & old_nodes),
|
|
"lost_incoming": sorted((old_in - new_in) & new_nodes),
|
|
"summary": {
|
|
"pages": len(new_nodes) - len(old_nodes),
|
|
"edges": len(new_edges) - len(old_edges),
|
|
},
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Suggested questions (graphify: analyze.suggest_questions)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def suggest_questions(
|
|
outgoing: dict[str, list[str]],
|
|
communities: list[set[str]],
|
|
labels: dict[int, str],
|
|
bridges: list[dict],
|
|
top_n: int = 7,
|
|
) -> list[dict]:
|
|
"""Questions the graph structure is uniquely positioned to raise.
|
|
|
|
Priority: bridge pages → dead-end hubs → isolated pages → low-cohesion
|
|
communities. Each has `type`, `question`, `why`.
|
|
"""
|
|
questions: list[dict] = []
|
|
degree, in_deg, out_deg = _build_degree_tables(outgoing)
|
|
|
|
# 1. Bridge pages -> cross-cutting questions
|
|
for b in bridges[:3]:
|
|
if not b["connects"]:
|
|
continue
|
|
own = labels.get(b["community"], f"community {b['community']}")
|
|
others = ", ".join(f"`{labels.get(c, f'community {c}')}`" for c in b["connects"])
|
|
questions.append({
|
|
"type": "bridge_node",
|
|
"question": f"Why does `{b['page']}` connect `{own}` to {others}?",
|
|
"why": f"High betweenness ({b['betweenness']}) — this page is a cross-community bridge.",
|
|
})
|
|
|
|
# 2. Sink hubs: many incoming, no outgoing
|
|
sinks = sorted(
|
|
(n for n in outgoing if out_deg.get(n, 0) == 0 and in_deg.get(n, 0) >= 3),
|
|
key=lambda n: -in_deg[n],
|
|
)[:2]
|
|
for n in sinks:
|
|
questions.append({
|
|
"type": "sink_hub",
|
|
"question": f"What should `{n}` link out to?",
|
|
"why": f"{in_deg[n]} pages point at it but it links nowhere — a dead end in a busy area.",
|
|
})
|
|
|
|
# 3. Isolated / weakly connected pages
|
|
weak = sorted(n for n in outgoing if degree.get(n, 0) <= 1)
|
|
if weak:
|
|
shown = ", ".join(f"`{n}`" for n in weak[:3])
|
|
questions.append({
|
|
"type": "isolated_nodes",
|
|
"question": f"What connects {shown} to the rest of the vault?",
|
|
"why": f"{len(weak)} weakly-connected pages — possible gaps or missing links.",
|
|
})
|
|
|
|
# 4. Low-cohesion communities
|
|
for i, comm in enumerate(communities):
|
|
if len(comm) >= 5:
|
|
c = cohesion_score(outgoing, comm)
|
|
if c < 0.15:
|
|
questions.append({
|
|
"type": "low_cohesion",
|
|
"question": f"Should `{labels.get(i, f'community {i}')}` be split into more focused topics?",
|
|
"why": f"Cohesion {c} across {len(comm)} pages — they share a cluster but are weakly interlinked.",
|
|
})
|
|
|
|
if not questions:
|
|
return [{
|
|
"type": "no_signal",
|
|
"question": None,
|
|
"why": "Not enough structure to raise questions — no bridges, sink hubs, isolates or fragmented clusters.",
|
|
}]
|
|
return questions[:top_n]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Community labelling (heuristic: most common tag or longest shared prefix)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _label_community(
|
|
pages: list[str],
|
|
tags_map: dict[str, list[str]],
|
|
taken: set[str] | None = None,
|
|
) -> str:
|
|
"""Most common tag; if that label is already `taken` by a bigger community,
|
|
qualify it with the next most common tag (`ai-security/roadmap`) or a
|
|
top page name so every community label is unique."""
|
|
freq: dict[str, int] = defaultdict(int)
|
|
for p in pages:
|
|
for tag in tags_map.get(p, []):
|
|
if not tag.startswith("visibility/"):
|
|
freq[tag] += 1
|
|
if freq:
|
|
ranked = sorted(freq, key=lambda t: (-freq[t], t))
|
|
label = ranked[0]
|
|
if taken is not None and label in taken:
|
|
for alt in ranked[1:]:
|
|
cand = f"{label}/{alt}"
|
|
if cand not in taken:
|
|
return cand
|
|
return f"{label}/{pages[0]}"
|
|
return label
|
|
# Fallback: shared prefix of page names
|
|
if len(pages) == 1:
|
|
return pages[0]
|
|
prefix = pages[0]
|
|
for p in pages[1:]:
|
|
while not p.startswith(prefix):
|
|
prefix = prefix[:-1]
|
|
if not prefix:
|
|
break
|
|
return prefix.rstrip("-_") or f"cluster-{pages[0][:20]}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def analyse_vault(
|
|
vault: Path,
|
|
top_n: int = 20,
|
|
previous_snapshot: dict[str, list] | None = None,
|
|
include_snapshot: bool = False,
|
|
) -> dict[str, Any]:
|
|
outgoing, tags_map = parse_vault_graph(vault)
|
|
|
|
if not outgoing:
|
|
return {
|
|
"god_nodes": [], "bridges": [], "communities": [],
|
|
"surprising_connections": [], "suggested_questions": [],
|
|
"dead_ends": [], "isolated": [],
|
|
"stats": {"pages": 0, "edges": 0, "communities": 0, "density": 0.0},
|
|
}
|
|
|
|
communities_raw = detect_communities(outgoing)
|
|
# Sort communities largest-first
|
|
communities_raw.sort(key=lambda c: -len(c))
|
|
labels: dict[int, str] = {}
|
|
taken: set[str] = set()
|
|
for i, c in enumerate(communities_raw):
|
|
lbl = _label_community(sorted(c), tags_map, taken)
|
|
labels[i] = lbl
|
|
taken.add(lbl)
|
|
|
|
total_edges = sum(len(v) for v in outgoing.values())
|
|
n_pages = len(outgoing)
|
|
undirected_edges = sum(len(v) for v in _undirected_adj(outgoing).values()) // 2
|
|
density = undirected_edges / (n_pages * (n_pages - 1) / 2) if n_pages > 1 else 0.0
|
|
|
|
bridges = bridge_pages(outgoing, communities_raw, top_n=min(top_n, 10), vault=vault)
|
|
|
|
result: dict[str, Any] = {
|
|
"god_nodes": god_nodes(outgoing, top_n),
|
|
"bridges": bridges,
|
|
"communities": [
|
|
{
|
|
"id": i,
|
|
"size": len(comm),
|
|
"pages": sorted(comm),
|
|
"label": labels[i],
|
|
"cohesion": cohesion_score(outgoing, comm),
|
|
}
|
|
for i, comm in enumerate(communities_raw)
|
|
],
|
|
"surprising_connections": surprising_connections(outgoing, communities_raw, top_n),
|
|
"suggested_questions": suggest_questions(outgoing, communities_raw, labels, bridges),
|
|
"dead_ends": dead_ends(outgoing),
|
|
"isolated": isolated(outgoing),
|
|
"stats": {
|
|
"pages": n_pages,
|
|
"edges": total_edges,
|
|
"communities": len(communities_raw),
|
|
"density": round(density, 4),
|
|
},
|
|
}
|
|
if previous_snapshot is not None:
|
|
result["diff"] = graph_diff(previous_snapshot, snapshot(outgoing))
|
|
if include_snapshot:
|
|
result["snapshot"] = snapshot(outgoing)
|
|
return result
|