feat(wiki): sync upstream v2026.10.4 core maintenance
Selectively port source hashing, locked manifest/memory writes, path-aware link lint and graph analysis, and revision-checked staging from upstream 4f457e154653756b04802d5aeb25c20f42b4770c. Record source hashes and exclusions. Preserve local archive/catalog citations, curated manifest metadata, config location, file-index and MCP behavior. Wire maintenance guidance into four skills without migrating vault contents or changing running services. Validation: 609 tests passed, including existing MCP/file-asset regressions and imported upstream maintenance tests. Read-only vault status left shared metadata hashes unchanged. Wiki MCP remains inactive; orders MCP is active.
This commit is contained in:
@@ -0,0 +1,914 @@
|
||||
"""Vault graph analysis: community detection, god nodes, bridges, surprising connections.
|
||||
|
||||
Reads the vault's wikilink graph (from page frontmatter and [[wikilinks]]),
|
||||
then runs the same family of algorithms graphify uses on code graphs — all in
|
||||
pure Python (no networkx / numpy):
|
||||
1. Degree-based god-node ranking (most-linked pages)
|
||||
2. Community detection — Leiden via `obsidian-wiki[graph]` if installed,
|
||||
else greedy label propagation (pure Python, no binary deps)
|
||||
3. Community cohesion — intra-community edge density (graphify
|
||||
`cohesion_score`); < 0.15 flags a fragmented cluster
|
||||
4. Bridge pages — Brandes betweenness centrality; the pages that sit on
|
||||
the most shortest paths between other pages (graphify's bridge nodes)
|
||||
5. Surprising connections — cross-community edges ranked by unexpectedness,
|
||||
de-duplicated one-per-community-pair so a single hub can't dominate
|
||||
6. Suggested questions — derived from bridges, isolates, dead-end hubs and
|
||||
low-cohesion communities (graphify `suggest_questions`)
|
||||
7. Path finding / neighbourhoods — `shortest_path` and BFS `neighborhood`
|
||||
(graphify `path` and blast-radius / `affected`)
|
||||
8. Graph diff — compare against a previous snapshot (graphify `graph_diff`)
|
||||
|
||||
Output JSON:
|
||||
{
|
||||
"god_nodes": [{"page": "...", "degree": N, "in_degree": N, "out_degree": N}, ...],
|
||||
"bridges": [{"page": "...", "betweenness": 0.N, "connects": [comm_id, ...]}, ...],
|
||||
"communities": [{"id": N, "size": N, "pages": [...], "label": "...", "cohesion": 0.N}, ...],
|
||||
"surprising_connections": [{"source": "...", "target": "...", "score": 0.N,
|
||||
"note": "bridges community A -> community B"}, ...],
|
||||
"suggested_questions": [{"type": "...", "question": "...", "why": "..."}, ...],
|
||||
"dead_ends": ["page with 0 outgoing links", ...],
|
||||
"isolated": ["page with 0 links at all", ...],
|
||||
"stats": {"pages": N, "edges": N, "communities": N, "density": 0.N}
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from obsidian_wiki.links import PageIndex, strip_code
|
||||
from obsidian_wiki.provenance import archive_wikilink_relpath
|
||||
from obsidian_wiki.vault import SKIP_DIRS as VAULT_SKIP_DIRS
|
||||
from obsidian_wiki.vault import iter_md, okignore_patterns, okignored, skipped_dir # noqa: F401 (re-exported)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wikilink / frontmatter parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[([^\]|#]+?)(?:[|#][^\]]*?)?\]\]")
|
||||
# Local `.md` targets only: skip URLs (any `scheme:`) and require `.md` to end
|
||||
# the path, before an optional `#anchor` or `"title"`.
|
||||
_MD_LINK_RE = re.compile(
|
||||
r"\[.*?\]\((?![A-Za-z][A-Za-z0-9+.-]*:)([^)#\s]+\.md)(?:[#\s][^)]*)?\)"
|
||||
)
|
||||
_TAGS_RE = re.compile(r"^tags:\s*\[([^\]]+)\]", re.MULTILINE)
|
||||
_TAGS_LIST_RE = re.compile(r"^tags:\s*\n((?:\s+-\s+\S+\n)+)", re.MULTILINE)
|
||||
|
||||
|
||||
def _slug(page_name: str) -> str:
|
||||
"""Normalise a wikilink target to a page slug."""
|
||||
return page_name.strip().lower().replace(" ", "-")
|
||||
|
||||
|
||||
def _page_slug(path: Path, root: Path) -> str:
|
||||
return _slug(path.stem)
|
||||
|
||||
|
||||
# --- Shared page selection -------------------------------------------------
|
||||
# Every module that builds a graph over the vault MUST use these, so the
|
||||
# wikilink graph is identical no matter which entry point produced it.
|
||||
|
||||
#: Directories that hold staging / archive / config material, not knowledge.
|
||||
#: Dot-prefixed paths (`.venv`, `.git`) are skipped wholesale by `is_wiki_page`;
|
||||
#: this list also covers the non-hidden equivalents (`venv/`, `node_modules/`).
|
||||
SKIP_DIRS = VAULT_SKIP_DIRS | {"_meta", "_readouts"}
|
||||
|
||||
#: Vault bookkeeping files at the vault ROOT. They link to (almost) every page,
|
||||
#: so leaving them in makes every pair of pages look ~2 hops apart and lets a
|
||||
#: table of contents outrank real knowledge in every degree/centrality ranking.
|
||||
#: Only the root-level files are skipped — a legitimate `concepts/index.md`
|
||||
#: is still a normal page.
|
||||
SKIP_ROOT_FILES = frozenset({"index", "log", "hot", "_insights"})
|
||||
|
||||
|
||||
def is_wiki_page(path: Path, vault: Path) -> bool:
|
||||
"""True if `path` is a knowledge page of `vault` (not staging/bookkeeping)."""
|
||||
try:
|
||||
rel = path.relative_to(vault)
|
||||
except ValueError:
|
||||
return False
|
||||
if skipped_dir(rel, SKIP_DIRS):
|
||||
return False
|
||||
if len(rel.parts) == 1 and path.stem in SKIP_ROOT_FILES:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def iter_pages(vault: Path) -> list[Path]:
|
||||
"""All knowledge pages in the vault, in stable sorted order."""
|
||||
return [p for p in iter_md(vault, SKIP_DIRS) if is_wiki_page(p, vault)]
|
||||
|
||||
|
||||
def parse_vault_graph(vault: Path) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
|
||||
"""Parse all .md pages and return (outgoing_edges, page_tags).
|
||||
|
||||
outgoing_edges: {source_slug: [target_slug, ...]}
|
||||
page_tags: {slug: [tag, ...]}
|
||||
"""
|
||||
outgoing: dict[str, list[str]] = defaultdict(list)
|
||||
tags_map: dict[str, list[str]] = {}
|
||||
|
||||
pages: list[Path] = iter_pages(vault)
|
||||
rels = {page: page.relative_to(vault).as_posix() for page in pages}
|
||||
index = PageIndex(rels.values())
|
||||
|
||||
for page in pages:
|
||||
src = index.id_for(rels[page])
|
||||
try:
|
||||
text = page.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
continue
|
||||
links_text = strip_code(text)
|
||||
|
||||
# Tags
|
||||
m = _TAGS_RE.search(text)
|
||||
if m:
|
||||
tags_map[src] = [t.strip().strip("'\"") for t in m.group(1).split(",")]
|
||||
else:
|
||||
m2 = _TAGS_LIST_RE.search(text)
|
||||
if m2:
|
||||
tags_map[src] = [ln.strip().lstrip("- ") for ln in m2.group(1).splitlines() if ln.strip()]
|
||||
|
||||
# Wikilinks
|
||||
for link in _WIKILINK_RE.findall(links_text):
|
||||
if archive_wikilink_relpath(vault, link) is not None:
|
||||
continue
|
||||
target = index.resolve(link, rels[page])
|
||||
if target and target != src:
|
||||
outgoing[src].append(target)
|
||||
|
||||
# Markdown links (when OBSIDIAN_LINK_FORMAT=markdown)
|
||||
for href in _MD_LINK_RE.findall(links_text):
|
||||
if archive_wikilink_relpath(vault, href) is not None:
|
||||
continue
|
||||
target = index.resolve(href, rels[page])
|
||||
if target and target != src:
|
||||
outgoing[src].append(target)
|
||||
|
||||
if src not in outgoing:
|
||||
outgoing[src] = []
|
||||
|
||||
# Ensure every known page appears as a key
|
||||
for page_id in index.ids():
|
||||
outgoing.setdefault(page_id, [])
|
||||
|
||||
return dict(outgoing), tags_map
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_degree_tables(
|
||||
outgoing: dict[str, list[str]]
|
||||
) -> tuple[dict[str, int], dict[str, int], dict[str, int]]:
|
||||
out_deg: dict[str, int] = {n: len(v) for n, v in outgoing.items()}
|
||||
in_deg: dict[str, int] = defaultdict(int)
|
||||
for targets in outgoing.values():
|
||||
for t in targets:
|
||||
in_deg[t] += 1
|
||||
degree = {n: out_deg.get(n, 0) + in_deg.get(n, 0) for n in outgoing}
|
||||
for n in in_deg:
|
||||
degree.setdefault(n, in_deg[n])
|
||||
return degree, dict(in_deg), out_deg
|
||||
|
||||
|
||||
def god_nodes(outgoing: dict[str, list[str]], top_n: int = 20) -> list[dict]:
|
||||
degree, in_deg, out_deg = _build_degree_tables(outgoing)
|
||||
ranked = sorted(degree, key=lambda n: -degree[n])[:top_n]
|
||||
return [
|
||||
{"page": n, "degree": degree[n],
|
||||
"in_degree": in_deg.get(n, 0), "out_degree": out_deg.get(n, 0)}
|
||||
for n in ranked
|
||||
]
|
||||
|
||||
|
||||
def dead_ends(outgoing: dict[str, list[str]]) -> list[str]:
|
||||
"""Pages with no outgoing links."""
|
||||
return [n for n, targets in outgoing.items() if not targets]
|
||||
|
||||
|
||||
def isolated(outgoing: dict[str, list[str]]) -> list[str]:
|
||||
"""Pages with zero links in either direction."""
|
||||
_, in_deg, out_deg = _build_degree_tables(outgoing)
|
||||
return [n for n in outgoing if in_deg.get(n, 0) == 0 and out_deg.get(n, 0) == 0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Community detection — greedy modularity (Newman-Girvan, pure Python)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _modularity(communities: list[set[str]], outgoing: dict[str, list[str]]) -> float:
|
||||
"""Approximate modularity Q for an undirected projection of outgoing edges."""
|
||||
all_edges = [(s, t) for s, targets in outgoing.items() for t in targets]
|
||||
m = len(all_edges)
|
||||
if m == 0:
|
||||
return 0.0
|
||||
degree = defaultdict(int)
|
||||
for s, t in all_edges:
|
||||
degree[s] += 1
|
||||
degree[t] += 1
|
||||
node_comm: dict[str, int] = {}
|
||||
for i, comm in enumerate(communities):
|
||||
for n in comm:
|
||||
node_comm[n] = i
|
||||
Q = 0.0
|
||||
for s, t in all_edges:
|
||||
if node_comm.get(s) == node_comm.get(t):
|
||||
Q += 1 - (degree[s] * degree[t]) / (2 * m)
|
||||
return Q / m
|
||||
|
||||
|
||||
def detect_communities_greedy(outgoing: dict[str, list[str]]) -> list[set[str]]:
|
||||
"""Greedy modularity community detection (label propagation variant).
|
||||
|
||||
Fast O(n·k) approach suitable for vaults up to ~5 000 pages. Each node
|
||||
adopts the most frequent label among its neighbours; iterate until stable.
|
||||
Falls back gracefully to one community per page if the graph is empty.
|
||||
"""
|
||||
nodes = list(outgoing.keys())
|
||||
if not nodes:
|
||||
return []
|
||||
|
||||
# Build undirected adjacency
|
||||
adj: dict[str, list[str]] = defaultdict(list)
|
||||
for src, targets in outgoing.items():
|
||||
for t in targets:
|
||||
adj[src].append(t)
|
||||
adj[t].append(src)
|
||||
|
||||
# Initialise: each node in its own community (label = index)
|
||||
labels: dict[str, int] = {n: i for i, n in enumerate(nodes)}
|
||||
|
||||
# Visit nodes in a shuffled order each round. A fixed sequential sweep
|
||||
# cascades one label along chains and collapses well-separated clusters
|
||||
# into a single community (a barbell graph comes back as one group). The
|
||||
# RNG is seeded from the node set, so the result stays reproducible.
|
||||
rng = random.Random(len(nodes))
|
||||
order = list(nodes)
|
||||
|
||||
for _ in range(20): # max 20 rounds
|
||||
changed = False
|
||||
rng.shuffle(order)
|
||||
for n in order:
|
||||
neighbours = adj[n]
|
||||
if not neighbours:
|
||||
continue
|
||||
freq: dict[int, int] = defaultdict(int)
|
||||
for nb in neighbours:
|
||||
freq[labels[nb]] += 1
|
||||
best = max(freq, key=lambda lbl: (freq[lbl], -lbl))
|
||||
if best != labels[n]:
|
||||
labels[n] = best
|
||||
changed = True
|
||||
if not changed:
|
||||
break
|
||||
|
||||
# Group by label
|
||||
groups: dict[int, set[str]] = defaultdict(set)
|
||||
for n, lbl in labels.items():
|
||||
groups[lbl].add(n)
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def detect_communities(outgoing: dict[str, list[str]]) -> list[set[str]]:
|
||||
"""Try Leiden (leidenalg + igraph) first; fall back to greedy label propagation."""
|
||||
try:
|
||||
import igraph as ig
|
||||
import leidenalg
|
||||
|
||||
nodes = list(outgoing.keys())
|
||||
node_idx = {n: i for i, n in enumerate(nodes)}
|
||||
edges = [
|
||||
(node_idx[s], node_idx[t])
|
||||
for s, targets in outgoing.items()
|
||||
for t in targets
|
||||
if t in node_idx
|
||||
]
|
||||
g = ig.Graph(n=len(nodes), edges=edges, directed=False)
|
||||
partition = leidenalg.find_partition(g, leidenalg.ModularityVertexPartition)
|
||||
return [
|
||||
{nodes[i] for i in cluster}
|
||||
for cluster in partition
|
||||
]
|
||||
except ImportError:
|
||||
return detect_communities_greedy(outgoing)
|
||||
|
||||
|
||||
def _undirected_adj(outgoing: dict[str, list[str]]) -> dict[str, set[str]]:
|
||||
"""Undirected simple-graph adjacency (duplicate/self edges collapsed)."""
|
||||
adj: dict[str, set[str]] = {n: set() for n in outgoing}
|
||||
for src, targets in outgoing.items():
|
||||
for t in targets:
|
||||
if t == src:
|
||||
continue
|
||||
adj[src].add(t)
|
||||
adj.setdefault(t, set()).add(src)
|
||||
return adj
|
||||
|
||||
|
||||
def _node_community_map(communities: list[set[str]]) -> dict[str, int]:
|
||||
return {n: i for i, comm in enumerate(communities) for n in comm}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Community cohesion (graphify: cluster.cohesion_score)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cohesion_score(outgoing: dict[str, list[str]], community: set[str]) -> float:
|
||||
"""Ratio of actual intra-community (undirected) edges to the maximum possible.
|
||||
|
||||
1.0 = clique, 0.0 = no internal links. Singletons score 1.0.
|
||||
"""
|
||||
n = len(community)
|
||||
if n <= 1:
|
||||
return 1.0
|
||||
adj = _undirected_adj(outgoing)
|
||||
actual = sum(1 for a in community for b in adj.get(a, ()) if b in community and a < b)
|
||||
possible = n * (n - 1) / 2
|
||||
return round(actual / possible, 4) if possible else 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Betweenness centrality — Brandes (2001), pure Python
|
||||
# (graphify: nx.betweenness_centrality → "bridge nodes")
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def betweenness_centrality(
|
||||
outgoing: dict[str, list[str]],
|
||||
normalized: bool = True,
|
||||
) -> dict[str, float]:
|
||||
"""Exact node betweenness on the undirected projection (Brandes algorithm).
|
||||
|
||||
O(V·E) — fine for vaults up to a few thousand pages. Values are normalised
|
||||
to [0, 1] by 2/((n-1)(n-2)) so scores are comparable across vault sizes.
|
||||
"""
|
||||
adj = _undirected_adj(outgoing)
|
||||
nodes = list(adj)
|
||||
bc: dict[str, float] = {n: 0.0 for n in nodes}
|
||||
for s in nodes:
|
||||
stack: list[str] = []
|
||||
preds: dict[str, list[str]] = {n: [] for n in nodes}
|
||||
sigma: dict[str, float] = {n: 0.0 for n in nodes}
|
||||
dist: dict[str, int] = {n: -1 for n in nodes}
|
||||
sigma[s] = 1.0
|
||||
dist[s] = 0
|
||||
queue = [s]
|
||||
qi = 0
|
||||
while qi < len(queue):
|
||||
v = queue[qi]
|
||||
qi += 1
|
||||
stack.append(v)
|
||||
for w in adj[v]:
|
||||
if dist[w] < 0:
|
||||
dist[w] = dist[v] + 1
|
||||
queue.append(w)
|
||||
if dist[w] == dist[v] + 1:
|
||||
sigma[w] += sigma[v]
|
||||
preds[w].append(v)
|
||||
delta: dict[str, float] = {n: 0.0 for n in nodes}
|
||||
while stack:
|
||||
w = stack.pop()
|
||||
for v in preds[w]:
|
||||
delta[v] += (sigma[v] / sigma[w]) * (1.0 + delta[w])
|
||||
if w != s:
|
||||
bc[w] += delta[w]
|
||||
n = len(nodes)
|
||||
# Undirected: every pair counted twice
|
||||
scale = 0.5
|
||||
if normalized and n > 2:
|
||||
scale = 1.0 / ((n - 1) * (n - 2))
|
||||
return {k: v * scale for k, v in bc.items()}
|
||||
|
||||
|
||||
# --- Betweenness cache -----------------------------------------------------
|
||||
# Betweenness is the ONLY expensive step in this module: O(V*E), ~42s on a
|
||||
# 5 000-page vault, versus 0.13s to parse it and 0.03s for communities. So it
|
||||
# is the only thing worth caching.
|
||||
#
|
||||
# The cache key is a hash of the GRAPH ITSELF, not file mtimes. That makes a
|
||||
# stale hit impossible by construction: if any link changed, the key changes;
|
||||
# if only prose changed, the old (still-correct) result is reused. No
|
||||
# invalidation logic to get wrong.
|
||||
|
||||
CACHE_FILENAME = ".graph-cache.json"
|
||||
_CACHE_MAX_ENTRIES = 3
|
||||
#: Don't write a cache file for graphs that recompute faster than this — small
|
||||
#: vaults should not accumulate cache files for no gain.
|
||||
_CACHE_MIN_SECONDS = 0.5
|
||||
|
||||
|
||||
def graph_fingerprint(outgoing: dict[str, list[str]]) -> str:
|
||||
"""Stable hash of the undirected graph topology (nodes + edges)."""
|
||||
adj = _undirected_adj(outgoing)
|
||||
h = hashlib.sha256()
|
||||
for node in sorted(adj):
|
||||
h.update(node.encode("utf-8"))
|
||||
h.update(b"\x00")
|
||||
for nb in sorted(adj[node]):
|
||||
h.update(nb.encode("utf-8"))
|
||||
h.update(b"\x01")
|
||||
h.update(b"\x02")
|
||||
return h.hexdigest()[:32]
|
||||
|
||||
|
||||
def _cache_path(vault: Path) -> Path:
|
||||
return vault / CACHE_FILENAME
|
||||
|
||||
|
||||
def _read_cache(vault: Path) -> dict:
|
||||
"""Read the cache, failing open (a corrupt or unreadable cache is ignored)."""
|
||||
try:
|
||||
data = json.loads(_cache_path(vault).read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, ValueError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) and data.get("version") == 1 else {}
|
||||
|
||||
|
||||
def _write_cache(vault: Path, data: dict) -> None:
|
||||
"""Atomically replace the cache file. Never raises — caching is best-effort."""
|
||||
target = _cache_path(vault)
|
||||
tmp = target.with_suffix(f".tmp{os.getpid()}")
|
||||
try:
|
||||
tmp.write_text(json.dumps(data), encoding="utf-8")
|
||||
os.replace(tmp, target) # atomic; concurrent writers can't tear the file
|
||||
except OSError:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def betweenness_cached(
|
||||
outgoing: dict[str, list[str]],
|
||||
vault: Path | None = None,
|
||||
normalized: bool = True,
|
||||
) -> dict[str, float]:
|
||||
"""`betweenness_centrality`, memoised on disk under `vault` by graph topology.
|
||||
|
||||
Falls back to a plain in-process computation when `vault` is None or the
|
||||
cache is unusable — the result is always identical either way.
|
||||
"""
|
||||
if vault is None:
|
||||
return betweenness_centrality(outgoing, normalized=normalized)
|
||||
|
||||
key = f"{graph_fingerprint(outgoing)}:{int(normalized)}"
|
||||
cache = _read_cache(vault)
|
||||
hit = cache.get("betweenness", {}).get(key)
|
||||
if isinstance(hit, dict) and hit.keys() >= outgoing.keys():
|
||||
return {n: float(hit[n]) for n in outgoing}
|
||||
|
||||
started = time.monotonic()
|
||||
scores = betweenness_centrality(outgoing, normalized=normalized)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
if elapsed >= _CACHE_MIN_SECONDS:
|
||||
entries = cache.get("betweenness", {})
|
||||
if not isinstance(entries, dict):
|
||||
entries = {}
|
||||
entries[key] = {n: round(v, 10) for n, v in scores.items()}
|
||||
# Keep the cache bounded — most recent keys win.
|
||||
for old in list(entries)[:-_CACHE_MAX_ENTRIES]:
|
||||
entries.pop(old, None)
|
||||
_write_cache(vault, {"version": 1, "betweenness": entries})
|
||||
return scores
|
||||
|
||||
|
||||
def bridge_pages(
|
||||
outgoing: dict[str, list[str]],
|
||||
communities: list[set[str]],
|
||||
top_n: int = 10,
|
||||
vault: Path | None = None,
|
||||
) -> list[dict]:
|
||||
"""Pages with the highest betweenness — removing them fragments the graph.
|
||||
|
||||
Each entry lists which communities the page's neighbours belong to, so the
|
||||
caller can say "`P` bridges cluster A ↔ cluster B".
|
||||
"""
|
||||
bc = betweenness_cached(outgoing, vault)
|
||||
if not bc:
|
||||
return []
|
||||
adj = _undirected_adj(outgoing)
|
||||
node_comm = _node_community_map(communities)
|
||||
ranked = sorted((n for n, s in bc.items() if s > 0), key=lambda n: -bc[n])[:top_n]
|
||||
out = []
|
||||
for n in ranked:
|
||||
own = node_comm.get(n)
|
||||
others = sorted({node_comm[nb] for nb in adj[n] if nb in node_comm and node_comm[nb] != own})
|
||||
out.append({
|
||||
"page": n,
|
||||
"betweenness": round(bc[n], 4),
|
||||
"community": own,
|
||||
"connects": others,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Surprising connections
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def surprising_connections(
|
||||
outgoing: dict[str, list[str]],
|
||||
communities: list[set[str]],
|
||||
top_n: int = 20,
|
||||
) -> list[dict]:
|
||||
"""Edges that cross community boundaries, ranked by unexpectedness.
|
||||
|
||||
Score = 1 / sqrt(cross_degree(source) * cross_degree(target))
|
||||
Low cross-degree nodes connected across communities are the most surprising.
|
||||
|
||||
Results are ordered so that each (community A, community B) boundary is
|
||||
represented once before any boundary repeats — graphify's dedup, which
|
||||
stops one high-betweenness hub from filling every slot.
|
||||
"""
|
||||
node_comm = _node_community_map(communities)
|
||||
|
||||
# Count how many cross-community edges each node already has
|
||||
cross_deg: dict[str, int] = defaultdict(int)
|
||||
for src, targets in outgoing.items():
|
||||
for t in targets:
|
||||
if node_comm.get(src) != node_comm.get(t):
|
||||
cross_deg[src] += 1
|
||||
cross_deg[t] += 1
|
||||
|
||||
results = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for src, targets in outgoing.items():
|
||||
for t in targets:
|
||||
pair = tuple(sorted((src, t)))
|
||||
if pair in seen:
|
||||
continue
|
||||
if node_comm.get(src) != node_comm.get(t):
|
||||
cd_s = cross_deg.get(src, 1)
|
||||
cd_t = cross_deg.get(t, 1)
|
||||
score = 1.0 / math.sqrt(cd_s * cd_t)
|
||||
cs, ct = node_comm.get(src), node_comm.get(t)
|
||||
results.append({
|
||||
"source": src, "target": t, "score": round(score, 4),
|
||||
"note": f"bridges community {cs} -> community {ct}",
|
||||
"_pair": (min(cs, ct), max(cs, ct)) if cs is not None and ct is not None else None,
|
||||
})
|
||||
seen.add(pair)
|
||||
|
||||
results.sort(key=lambda x: -x["score"])
|
||||
# One representative per community pair first, then the rest by score.
|
||||
seen_pairs: set = set()
|
||||
first: list[dict] = []
|
||||
rest: list[dict] = []
|
||||
for r in results:
|
||||
pair = r.pop("_pair")
|
||||
if pair not in seen_pairs:
|
||||
seen_pairs.add(pair)
|
||||
first.append(r)
|
||||
else:
|
||||
rest.append(r)
|
||||
return (first + rest)[:top_n]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path finding & neighbourhoods (graphify: `path`, `affected` blast radius)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _node_for(outgoing: dict[str, list[str]], term: str) -> str:
|
||||
"""Node id for a page reference: an exact id, else the shortest id whose
|
||||
path ends with it. Pages that share a name have path-qualified ids, so a
|
||||
bare name has to be matched by suffix."""
|
||||
term = "/".join(_slug(part) for part in term.strip().removesuffix(".md").split("/") if part)
|
||||
if term in outgoing:
|
||||
return term
|
||||
matches = sorted((n for n in outgoing if n.endswith(f"/{term}")), key=lambda n: (n.count("/"), n))
|
||||
return matches[0] if matches else term
|
||||
|
||||
|
||||
def shortest_path(
|
||||
outgoing: dict[str, list[str]],
|
||||
source: str,
|
||||
target: str,
|
||||
directed: bool = False,
|
||||
) -> list[str] | None:
|
||||
"""BFS shortest path between two pages. Follows links in either direction
|
||||
unless `directed=True`. Returns the node list, or None if unreachable."""
|
||||
source, target = _node_for(outgoing, source), _node_for(outgoing, target)
|
||||
if source not in outgoing or target not in outgoing:
|
||||
return None
|
||||
if source == target:
|
||||
return [source]
|
||||
if directed:
|
||||
adj: dict[str, Any] = {n: set(v) for n, v in outgoing.items()}
|
||||
else:
|
||||
adj = _undirected_adj(outgoing)
|
||||
prev: dict[str, str | None] = {source: None}
|
||||
queue = [source]
|
||||
qi = 0
|
||||
while qi < len(queue):
|
||||
v = queue[qi]
|
||||
qi += 1
|
||||
for w in sorted(adj.get(v, ())):
|
||||
if w in prev:
|
||||
continue
|
||||
prev[w] = v
|
||||
if w == target:
|
||||
path = [w]
|
||||
while prev[path[-1]] is not None:
|
||||
path.append(prev[path[-1]]) # type: ignore[arg-type]
|
||||
return path[::-1]
|
||||
queue.append(w)
|
||||
return None
|
||||
|
||||
|
||||
def neighborhood(
|
||||
outgoing: dict[str, list[str]],
|
||||
seed: str,
|
||||
depth: int = 2,
|
||||
direction: str = "both",
|
||||
) -> list[dict]:
|
||||
"""Breadth-first neighbourhood of a page up to `depth` hops.
|
||||
|
||||
direction: "out" (pages this one links to), "in" (pages linking here —
|
||||
the blast radius if this page is renamed/removed), or "both".
|
||||
Returns [{"page", "depth", "via"}] ordered by depth then name.
|
||||
"""
|
||||
seed = _node_for(outgoing, seed)
|
||||
if seed not in outgoing:
|
||||
return []
|
||||
incoming: dict[str, set[str]] = defaultdict(set)
|
||||
for s, targets in outgoing.items():
|
||||
for t in targets:
|
||||
incoming[t].add(s)
|
||||
|
||||
def nbrs(n: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
if direction in ("out", "both"):
|
||||
out |= set(outgoing.get(n, ()))
|
||||
if direction in ("in", "both"):
|
||||
out |= incoming.get(n, set())
|
||||
out.discard(n)
|
||||
return out
|
||||
|
||||
seen = {seed: 0}
|
||||
via = {seed: None}
|
||||
frontier = [seed]
|
||||
for d in range(1, depth + 1):
|
||||
nxt = []
|
||||
for v in frontier:
|
||||
for w in sorted(nbrs(v)):
|
||||
if w not in seen:
|
||||
seen[w] = d
|
||||
via[w] = v
|
||||
nxt.append(w)
|
||||
frontier = nxt
|
||||
if not frontier:
|
||||
break
|
||||
return [
|
||||
{"page": n, "depth": d, "via": via[n]}
|
||||
for n, d in sorted(seen.items(), key=lambda kv: (kv[1], kv[0]))
|
||||
if n != seed
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph diff (graphify: analyze.graph_diff)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SNAPSHOT_RE = re.compile(r"<!--\s*GRAPH_SNAPSHOT:\s*(\{.*?\})\s*-->", re.DOTALL)
|
||||
|
||||
|
||||
def snapshot(outgoing: dict[str, list[str]]) -> dict[str, list]:
|
||||
"""Compact, JSON-serialisable snapshot for embedding in `_insights.md`."""
|
||||
edges = sorted({(s, t) for s, ts in outgoing.items() for t in ts})
|
||||
return {"nodes": sorted(outgoing), "edges": [list(e) for e in edges]}
|
||||
|
||||
|
||||
def load_snapshot(path: Path) -> dict[str, list] | None:
|
||||
"""Read a snapshot from a JSON file or from the `<!-- GRAPH_SNAPSHOT: ... -->`
|
||||
comment inside a previous `_insights.md`. Returns None if not found."""
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return None
|
||||
m = _SNAPSHOT_RE.search(text)
|
||||
raw = m.group(1) if m else text
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(data, dict) or "nodes" not in data:
|
||||
return None
|
||||
return {"nodes": list(data.get("nodes", [])), "edges": [list(e) for e in data.get("edges", [])]}
|
||||
|
||||
|
||||
def graph_diff(old: dict[str, list], new: dict[str, list]) -> dict[str, Any]:
|
||||
"""What changed between two snapshots (see `snapshot`)."""
|
||||
old_nodes, new_nodes = set(old.get("nodes", [])), set(new.get("nodes", []))
|
||||
old_edges = {tuple(e) for e in old.get("edges", [])}
|
||||
new_edges = {tuple(e) for e in new.get("edges", [])}
|
||||
|
||||
def in_targets(edges: set) -> set[str]:
|
||||
return {t for _, t in edges}
|
||||
|
||||
old_in, new_in = in_targets(old_edges), in_targets(new_edges)
|
||||
added_edges = sorted(new_edges - old_edges)
|
||||
removed_edges = sorted(old_edges - new_edges)
|
||||
return {
|
||||
"added_pages": sorted(new_nodes - old_nodes),
|
||||
"removed_pages": sorted(old_nodes - new_nodes),
|
||||
"added_edges": [list(e) for e in added_edges],
|
||||
"removed_edges": [list(e) for e in removed_edges],
|
||||
"newly_connected": sorted((new_in - old_in) & old_nodes),
|
||||
"lost_incoming": sorted((old_in - new_in) & new_nodes),
|
||||
"summary": {
|
||||
"pages": len(new_nodes) - len(old_nodes),
|
||||
"edges": len(new_edges) - len(old_edges),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Suggested questions (graphify: analyze.suggest_questions)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def suggest_questions(
|
||||
outgoing: dict[str, list[str]],
|
||||
communities: list[set[str]],
|
||||
labels: dict[int, str],
|
||||
bridges: list[dict],
|
||||
top_n: int = 7,
|
||||
) -> list[dict]:
|
||||
"""Questions the graph structure is uniquely positioned to raise.
|
||||
|
||||
Priority: bridge pages → dead-end hubs → isolated pages → low-cohesion
|
||||
communities. Each has `type`, `question`, `why`.
|
||||
"""
|
||||
questions: list[dict] = []
|
||||
degree, in_deg, out_deg = _build_degree_tables(outgoing)
|
||||
|
||||
# 1. Bridge pages -> cross-cutting questions
|
||||
for b in bridges[:3]:
|
||||
if not b["connects"]:
|
||||
continue
|
||||
own = labels.get(b["community"], f"community {b['community']}")
|
||||
others = ", ".join(f"`{labels.get(c, f'community {c}')}`" for c in b["connects"])
|
||||
questions.append({
|
||||
"type": "bridge_node",
|
||||
"question": f"Why does `{b['page']}` connect `{own}` to {others}?",
|
||||
"why": f"High betweenness ({b['betweenness']}) — this page is a cross-community bridge.",
|
||||
})
|
||||
|
||||
# 2. Sink hubs: many incoming, no outgoing
|
||||
sinks = sorted(
|
||||
(n for n in outgoing if out_deg.get(n, 0) == 0 and in_deg.get(n, 0) >= 3),
|
||||
key=lambda n: -in_deg[n],
|
||||
)[:2]
|
||||
for n in sinks:
|
||||
questions.append({
|
||||
"type": "sink_hub",
|
||||
"question": f"What should `{n}` link out to?",
|
||||
"why": f"{in_deg[n]} pages point at it but it links nowhere — a dead end in a busy area.",
|
||||
})
|
||||
|
||||
# 3. Isolated / weakly connected pages
|
||||
weak = sorted(n for n in outgoing if degree.get(n, 0) <= 1)
|
||||
if weak:
|
||||
shown = ", ".join(f"`{n}`" for n in weak[:3])
|
||||
questions.append({
|
||||
"type": "isolated_nodes",
|
||||
"question": f"What connects {shown} to the rest of the vault?",
|
||||
"why": f"{len(weak)} weakly-connected pages — possible gaps or missing links.",
|
||||
})
|
||||
|
||||
# 4. Low-cohesion communities
|
||||
for i, comm in enumerate(communities):
|
||||
if len(comm) >= 5:
|
||||
c = cohesion_score(outgoing, comm)
|
||||
if c < 0.15:
|
||||
questions.append({
|
||||
"type": "low_cohesion",
|
||||
"question": f"Should `{labels.get(i, f'community {i}')}` be split into more focused topics?",
|
||||
"why": f"Cohesion {c} across {len(comm)} pages — they share a cluster but are weakly interlinked.",
|
||||
})
|
||||
|
||||
if not questions:
|
||||
return [{
|
||||
"type": "no_signal",
|
||||
"question": None,
|
||||
"why": "Not enough structure to raise questions — no bridges, sink hubs, isolates or fragmented clusters.",
|
||||
}]
|
||||
return questions[:top_n]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Community labelling (heuristic: most common tag or longest shared prefix)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _label_community(
|
||||
pages: list[str],
|
||||
tags_map: dict[str, list[str]],
|
||||
taken: set[str] | None = None,
|
||||
) -> str:
|
||||
"""Most common tag; if that label is already `taken` by a bigger community,
|
||||
qualify it with the next most common tag (`ai-security/roadmap`) or a
|
||||
top page name so every community label is unique."""
|
||||
freq: dict[str, int] = defaultdict(int)
|
||||
for p in pages:
|
||||
for tag in tags_map.get(p, []):
|
||||
if not tag.startswith("visibility/"):
|
||||
freq[tag] += 1
|
||||
if freq:
|
||||
ranked = sorted(freq, key=lambda t: (-freq[t], t))
|
||||
label = ranked[0]
|
||||
if taken is not None and label in taken:
|
||||
for alt in ranked[1:]:
|
||||
cand = f"{label}/{alt}"
|
||||
if cand not in taken:
|
||||
return cand
|
||||
return f"{label}/{pages[0]}"
|
||||
return label
|
||||
# Fallback: shared prefix of page names
|
||||
if len(pages) == 1:
|
||||
return pages[0]
|
||||
prefix = pages[0]
|
||||
for p in pages[1:]:
|
||||
while not p.startswith(prefix):
|
||||
prefix = prefix[:-1]
|
||||
if not prefix:
|
||||
break
|
||||
return prefix.rstrip("-_") or f"cluster-{pages[0][:20]}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def analyse_vault(
|
||||
vault: Path,
|
||||
top_n: int = 20,
|
||||
previous_snapshot: dict[str, list] | None = None,
|
||||
include_snapshot: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
outgoing, tags_map = parse_vault_graph(vault)
|
||||
|
||||
if not outgoing:
|
||||
return {
|
||||
"god_nodes": [], "bridges": [], "communities": [],
|
||||
"surprising_connections": [], "suggested_questions": [],
|
||||
"dead_ends": [], "isolated": [],
|
||||
"stats": {"pages": 0, "edges": 0, "communities": 0, "density": 0.0},
|
||||
}
|
||||
|
||||
communities_raw = detect_communities(outgoing)
|
||||
# Sort communities largest-first
|
||||
communities_raw.sort(key=lambda c: -len(c))
|
||||
labels: dict[int, str] = {}
|
||||
taken: set[str] = set()
|
||||
for i, c in enumerate(communities_raw):
|
||||
lbl = _label_community(sorted(c), tags_map, taken)
|
||||
labels[i] = lbl
|
||||
taken.add(lbl)
|
||||
|
||||
total_edges = sum(len(v) for v in outgoing.values())
|
||||
n_pages = len(outgoing)
|
||||
undirected_edges = sum(len(v) for v in _undirected_adj(outgoing).values()) // 2
|
||||
density = undirected_edges / (n_pages * (n_pages - 1) / 2) if n_pages > 1 else 0.0
|
||||
|
||||
bridges = bridge_pages(outgoing, communities_raw, top_n=min(top_n, 10), vault=vault)
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"god_nodes": god_nodes(outgoing, top_n),
|
||||
"bridges": bridges,
|
||||
"communities": [
|
||||
{
|
||||
"id": i,
|
||||
"size": len(comm),
|
||||
"pages": sorted(comm),
|
||||
"label": labels[i],
|
||||
"cohesion": cohesion_score(outgoing, comm),
|
||||
}
|
||||
for i, comm in enumerate(communities_raw)
|
||||
],
|
||||
"surprising_connections": surprising_connections(outgoing, communities_raw, top_n),
|
||||
"suggested_questions": suggest_questions(outgoing, communities_raw, labels, bridges),
|
||||
"dead_ends": dead_ends(outgoing),
|
||||
"isolated": isolated(outgoing),
|
||||
"stats": {
|
||||
"pages": n_pages,
|
||||
"edges": total_edges,
|
||||
"communities": len(communities_raw),
|
||||
"density": round(density, 4),
|
||||
},
|
||||
}
|
||||
if previous_snapshot is not None:
|
||||
result["diff"] = graph_diff(previous_snapshot, snapshot(outgoing))
|
||||
if include_snapshot:
|
||||
result["snapshot"] = snapshot(outgoing)
|
||||
return result
|
||||
Reference in New Issue
Block a user