Codex
e943bb329e
setup / install-leaves-tree-clean (push) Waiting to run
Selectively port source coverage and redirect lint, ignore-aware provenance, staging and trust checks from upstream 055d720. Preserve local archive metadata and adapt source-date and defuddle skill guidance. Keep the existing skills directory, file index and MCP interfaces. Record per-module upstream baselines and exclusions without migrating vault data or changing services. Validation: 625 tests passed; git diff --check passed.
803 lines
31 KiB
Python
803 lines
31 KiB
Python
"""Vault lint checks for wiki structure and metadata hygiene."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections import defaultdict
|
|
from collections.abc import Collection
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from obsidian_wiki.cache import _iter_entries, _load_manifest
|
|
from obsidian_wiki.graph_analysis import _page_slug as graph_page_slug
|
|
from obsidian_wiki.graph_analysis import iter_pages as iter_graph_pages
|
|
from obsidian_wiki.links import PageIndex, strip_code
|
|
from obsidian_wiki.provenance import (
|
|
archive_wikilink_relpath,
|
|
clip_url_index,
|
|
expected_snapshots_for_page,
|
|
invert_pages,
|
|
parse_snapshots_field,
|
|
)
|
|
from obsidian_wiki.vault import SKIP_DIRS as VAULT_SKIP_DIRS
|
|
from obsidian_wiki.vault import iter_instruction_files, iter_md, okignore_patterns, okignored, split_frontmatter
|
|
from obsidian_wiki.temporal import (
|
|
SUPERSEDED_FIELD,
|
|
superseded_target,
|
|
validity_window,
|
|
)
|
|
from obsidian_wiki.trust import (
|
|
ALLOWED_LIFECYCLES,
|
|
TRUST_LEDGER_RELATIVE_PATH,
|
|
check_lifecycle_transitions,
|
|
check_trust_ledger,
|
|
validate_trust_metadata,
|
|
)
|
|
|
|
SKIP_DIRS = VAULT_SKIP_DIRS | {"_bootstrap"}
|
|
REQUIRED_FRONTMATTER = (
|
|
"title",
|
|
"category",
|
|
"tags",
|
|
"sources",
|
|
"created",
|
|
"updated",
|
|
)
|
|
# Introduced by the trust-ledger rollout (#28, #132). Legacy pages that predate
|
|
# the schema are missing these by construction; enforcement is staged behind
|
|
# lint_vault's strict_trust switch so upgrading obsidian-wiki doesn't fail-close
|
|
# every pre-existing page until a vault owner explicitly opts into strict mode
|
|
# after a backfill/review pass.
|
|
TRUST_REQUIRED_FRONTMATTER = (
|
|
"base_confidence",
|
|
"lifecycle",
|
|
)
|
|
RESERVED_PAGE_STEMS = frozenset({"index", "log", "hot", "_insights"})
|
|
ALLOWED_RELATIONSHIP_TYPES = frozenset(
|
|
{"extends", "implements", "contradicts", "derived_from", "uses", "replaces", "related_to"}
|
|
)
|
|
|
|
_FIELD_RE = re.compile(r"^([A-Za-z_][\w-]*):", re.MULTILINE)
|
|
_WIKILINK_RE = re.compile(r"\[\[([^\]|#]+?)(?:[|#][^\]]*?)?\]\]")
|
|
# Local `.md` targets only: skip URLs (any `scheme:`) and require `.md` to end
|
|
# the path, before an optional `#anchor` or `"title"`.
|
|
_MD_LINK_RE = re.compile(
|
|
r"\[.*?\]\((?![A-Za-z][A-Za-z0-9+.-]*:)([^)#\s]+\.md)(?:[#\s][^)]*)?\)"
|
|
)
|
|
_RELATIONSHIP_LIST_FIELD_RE = re.compile(
|
|
r"^\s*-\s*(type|target):\s*(.*?)\s*$"
|
|
)
|
|
_RELATIONSHIP_ITEM_START_RE = re.compile(r"^\s*-\s*(?:#.*)?$")
|
|
_RELATIONSHIP_FIELD_RE = re.compile(r"^\s+(type|target):\s*(.*?)\s*$")
|
|
_WINDOWS_ABS_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
|
# Freshness: a count or amount with no date beside it reads as true forever
|
|
# and silently rots ("23 stars", "the pipeline has 13 deals"). Every such fact
|
|
# should be stamped `(as of YYYY-MM-DD)`, live in a dated note or under a
|
|
# dated heading, or point at where the truth lives.
|
|
# ponytail: fixed noun list tuned for precision; widen it if real vaults miss claims.
|
|
_UNDATED_CLAIM_RE = re.compile(
|
|
r"\b\d+(?:[.,]\d+)*\s*[km]?\+?\s*(?:★|\s(?:open\s+|active\s+|paying\s+)?"
|
|
r"(?:users|customers|clients|deals|stars|subscribers|downloads|members|employees|"
|
|
r"tickets|PRs|pull requests|followers|signups|installs)\b)",
|
|
re.IGNORECASE,
|
|
)
|
|
_UNDATED_MONEY_RE = re.compile(
|
|
r"\b(?:MRR|ARR|revenue|balance|runway)\b[^.\n]{0,20}?\b(?:is|of|at)\s+[$€£]?\s?\d",
|
|
re.IGNORECASE,
|
|
)
|
|
_DATE_HINT_RE = re.compile(r"\b(?:19|20)\d{2}\b|\bas of\b", re.IGNORECASE)
|
|
_HEADING_RE = re.compile(r"^(#{1,6})\s")
|
|
_FENCE_LINE_RE = re.compile(r"^\s*(```|~~~)")
|
|
_INLINE_CODE_SPAN_RE = re.compile(r"`[^`\n]*`")
|
|
_DATED_STEM_RE = re.compile(r"^(?:19|20)\d{2}-\d{2}")
|
|
|
|
|
|
def _frontmatter_field_block(frontmatter: str, field: str) -> str:
|
|
"""The raw value of a frontmatter *field*, including a following block list.
|
|
|
|
Returns the inline value plus any subsequent ``- item`` lines indented under
|
|
it; an empty string when the field is absent.
|
|
"""
|
|
lines = frontmatter.splitlines()
|
|
for index, line in enumerate(lines):
|
|
match = re.match(rf"^{re.escape(field)}\s*:(.*)$", line)
|
|
if not match:
|
|
continue
|
|
block = [match.group(1)]
|
|
for following in lines[index + 1:]:
|
|
if re.match(r"^\s+-\s", following) or not following.strip():
|
|
block.append(following)
|
|
continue
|
|
break
|
|
return "\n".join(block)
|
|
return ""
|
|
|
|
|
|
def _absolute_source_entries(frontmatter: str) -> list[str]:
|
|
"""`sources:` entries that are machine absolute paths (contract violation).
|
|
|
|
The source key contract bans a stored absolute path in page frontmatter just
|
|
as it does in the manifest; ``~``-relative and vault-relative entries are
|
|
fine. Only the ``sources:`` field is inspected — a path in prose may be a
|
|
legitimate citation.
|
|
"""
|
|
raw = _frontmatter_field_block(frontmatter, "sources")
|
|
if not raw.strip():
|
|
return []
|
|
lines = raw.splitlines()
|
|
inline = lines[0].strip()
|
|
entries: list[str] = []
|
|
if inline.startswith("["):
|
|
entries.extend(inline.strip("[]").split(","))
|
|
elif inline:
|
|
# A scalar `sources: <value>` is a single entry — not a block list, and
|
|
# not a flow list, but still a stored source key.
|
|
entries.append(inline)
|
|
entries.extend(
|
|
line.strip()[1:] for line in lines[1:] if line.strip().startswith("-")
|
|
)
|
|
bad: list[str] = []
|
|
for entry in entries:
|
|
value = entry.strip().strip("'\"").strip()
|
|
if not value:
|
|
continue
|
|
if value.startswith("/") or _WINDOWS_ABS_RE.match(value):
|
|
bad.append(value)
|
|
return bad
|
|
|
|
|
|
def _slug(text: str) -> str:
|
|
return text.strip().lower().replace(" ", "-")
|
|
|
|
|
|
def _parse_frontmatter_values(frontmatter: str) -> dict[str, str]:
|
|
values: dict[str, str] = {}
|
|
lines = frontmatter.splitlines()
|
|
i = 0
|
|
while i < len(lines):
|
|
line = lines[i]
|
|
if not line or ":" not in line or line.startswith((" ", "\t")):
|
|
i += 1
|
|
continue
|
|
key, raw = line.split(":", 1)
|
|
key = key.strip()
|
|
value = raw.strip()
|
|
if value in {">", ">-", "|", "|-"}:
|
|
block: list[str] = []
|
|
i += 1
|
|
while i < len(lines):
|
|
child = lines[i]
|
|
if child.startswith(" ") or child.startswith("\t"):
|
|
block.append(child.strip())
|
|
i += 1
|
|
continue
|
|
break
|
|
values[key] = " ".join(part for part in block if part).strip()
|
|
continue
|
|
values[key] = value.strip("'\"")
|
|
i += 1
|
|
return values
|
|
|
|
|
|
def _relationship_scalar(raw: str) -> str:
|
|
value = raw.strip()
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
|
return value[1:-1].strip()
|
|
return value.split(" #", 1)[0].strip()
|
|
|
|
|
|
def _parse_relationships(frontmatter: str) -> list[dict[str, str]]:
|
|
relationships: list[dict[str, str]] = []
|
|
current: dict[str, str] | None = None
|
|
in_relationships = False
|
|
for line in frontmatter.splitlines():
|
|
if line.startswith("relationships:") and not line.startswith((" ", "\t")):
|
|
in_relationships = True
|
|
inline = line.split(":", 1)[1].strip()
|
|
if inline in {"[]", "null", "~"}:
|
|
return []
|
|
if inline:
|
|
relationships.append({"parse_error": "inline_relationships_not_supported"})
|
|
return relationships
|
|
continue
|
|
if in_relationships and line and not line.startswith((" ", "\t")):
|
|
break
|
|
if not in_relationships:
|
|
continue
|
|
item_match = _RELATIONSHIP_LIST_FIELD_RE.match(line)
|
|
if item_match:
|
|
if current is not None:
|
|
relationships.append(current)
|
|
current = {item_match.group(1): _relationship_scalar(item_match.group(2))}
|
|
continue
|
|
if _RELATIONSHIP_ITEM_START_RE.match(line):
|
|
if current is not None:
|
|
relationships.append(current)
|
|
current = {}
|
|
continue
|
|
field_match = _RELATIONSHIP_FIELD_RE.match(line)
|
|
if field_match and current is not None:
|
|
key = field_match.group(1)
|
|
if key in current:
|
|
current["parse_error"] = f"duplicate_relationship_{key}"
|
|
else:
|
|
current[key] = _relationship_scalar(field_match.group(2))
|
|
continue
|
|
if line.strip() and not line.lstrip().startswith("#"):
|
|
if current is None:
|
|
current = {"parse_error": "malformed_relationship_entry"}
|
|
else:
|
|
current.setdefault("parse_error", "malformed_relationship_entry")
|
|
if current is not None:
|
|
relationships.append(current)
|
|
return relationships
|
|
|
|
|
|
def _display_target(raw: str) -> str:
|
|
"""How a broken link is reported: the slugged last segment, sans `.md`."""
|
|
name = raw.rstrip("\\").split("/")[-1]
|
|
return _slug(name[:-3] if name.lower().endswith(".md") else name)
|
|
|
|
|
|
def _relationship_link(raw: str) -> str:
|
|
"""A `relationships:` target as a bare link target (no brackets, alias, anchor)."""
|
|
target = raw.strip().strip("\"'").removeprefix("[[").removesuffix("]]")
|
|
return target.split("|", 1)[0].split("#", 1)[0].strip()
|
|
|
|
|
|
def _normalise_node_id(raw: str) -> str:
|
|
target = raw.strip().removeprefix("[[").removesuffix("]]")
|
|
target = target.split("|", 1)[0].split("#", 1)[0].strip()
|
|
if target.lower().endswith(".md"):
|
|
target = target[:-3]
|
|
return "/".join(_slug(part) for part in target.strip("/").split("/") if part)
|
|
|
|
|
|
# Extensions Obsidian embeds as attachments. Anything else after a dot is part
|
|
# of the page name ("Node.js", "v1.2 release notes"), not a file extension.
|
|
_ATTACHMENT_SUFFIXES = frozenset({
|
|
".png", ".jpg", ".jpeg", ".gif", ".bmp", ".svg", ".webp", ".avif",
|
|
".mp3", ".wav", ".m4a", ".ogg", ".flac", ".mp4", ".webm", ".mov", ".ogv",
|
|
".pdf", ".canvas", ".base",
|
|
})
|
|
|
|
|
|
def _wikilink_page_target(raw: str) -> str | None:
|
|
"""Normalise a `_WIKILINK_RE` capture to a page name, or None for an
|
|
attachment embed `by_slug` (`.md`-only) can never hold. Also strips an
|
|
explicit `.md` suffix and a table-escaped pipe's trailing backslash.
|
|
"""
|
|
name = raw.rstrip("\\").split("/")[-1]
|
|
suffix = Path(name).suffix.lower()
|
|
if suffix == ".md":
|
|
return name[: -len(suffix)]
|
|
if suffix in _ATTACHMENT_SUFFIXES:
|
|
return None
|
|
return name
|
|
|
|
|
|
def _undated_claims(body: str, first_line: int) -> list[dict[str, Any]]:
|
|
"""Lines claiming a current count or amount with no date to anchor it."""
|
|
claims: list[dict[str, Any]] = []
|
|
in_fence = False
|
|
dated_level: int | None = None
|
|
for offset, line in enumerate(body.splitlines()):
|
|
if _FENCE_LINE_RE.match(line):
|
|
in_fence = not in_fence
|
|
continue
|
|
if in_fence or line.lstrip().startswith(">"):
|
|
continue
|
|
heading = _HEADING_RE.match(line)
|
|
if heading:
|
|
level = len(heading.group(1))
|
|
if _DATE_HINT_RE.search(line):
|
|
dated_level = level if dated_level is None else min(dated_level, level)
|
|
elif dated_level is not None and level <= dated_level:
|
|
dated_level = None
|
|
continue
|
|
if dated_level is not None:
|
|
continue
|
|
prose = _INLINE_CODE_SPAN_RE.sub("", line)
|
|
if _DATE_HINT_RE.search(prose):
|
|
continue
|
|
if _UNDATED_CLAIM_RE.search(prose) or _UNDATED_MONEY_RE.search(prose):
|
|
claims.append({"line": first_line + offset, "text": line.strip()[:160]})
|
|
return claims
|
|
|
|
|
|
def _parse_page(path: Path, vault: Path) -> dict[str, Any]:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
frontmatter, body = split_frontmatter(text)
|
|
relative_parts = path.relative_to(vault).parts
|
|
dated_page = relative_parts[0] == "journal" or bool(_DATED_STEM_RE.match(path.stem))
|
|
undated = (
|
|
[]
|
|
if dated_page
|
|
else _undated_claims(body, text[: len(text) - len(body)].count("\n") + 1)
|
|
)
|
|
fields = set(_FIELD_RE.findall(frontmatter))
|
|
values = _parse_frontmatter_values(frontmatter)
|
|
relative = path.relative_to(vault)
|
|
|
|
# Raw targets, resolved per page in `lint_vault` against the whole vault
|
|
# (`PageIndex`), so a project-relative link resolves like Obsidian does.
|
|
links: list[str] = []
|
|
broken_archive_links: list[dict[str, str]] = []
|
|
text = strip_code(text)
|
|
for raw in _WIKILINK_RE.findall(text):
|
|
archive_rel = archive_wikilink_relpath(vault, raw)
|
|
if archive_rel is not None:
|
|
if not (vault / archive_rel).is_file():
|
|
broken_archive_links.append(
|
|
{"page": relative.as_posix(), "target": archive_rel}
|
|
)
|
|
continue
|
|
if _wikilink_page_target(raw) and raw.strip():
|
|
links.append(raw.strip().rstrip("\\"))
|
|
for href in _MD_LINK_RE.findall(text):
|
|
archive_rel = archive_wikilink_relpath(vault, href)
|
|
if archive_rel is not None:
|
|
if not (vault / archive_rel).is_file():
|
|
broken_archive_links.append(
|
|
{"page": relative.as_posix(), "target": archive_rel}
|
|
)
|
|
continue
|
|
links.append(href)
|
|
|
|
return {
|
|
"path": relative.as_posix(),
|
|
"node_id": _normalise_node_id(relative.with_suffix("").as_posix()),
|
|
"slug": _slug(path.stem),
|
|
"title": values.get("title", "").strip() or path.stem,
|
|
"summary": values.get("summary", "").strip(),
|
|
"redirects_to": values.get("redirects_to", "").strip(),
|
|
"fields": fields,
|
|
"links": links,
|
|
"relationships": _parse_relationships(frontmatter),
|
|
"absolute_sources": _absolute_source_entries(frontmatter),
|
|
"snapshots": parse_snapshots_field(_frontmatter_field_block(frontmatter, "snapshots")),
|
|
"values": values,
|
|
"archive_broken": broken_archive_links,
|
|
"undated_claims": undated,
|
|
}
|
|
|
|
|
|
# Manifest fields that list the pages a source produced (provenance.py reads
|
|
# the first two; pages_updated is the same list for an update run).
|
|
_COVERAGE_PAGE_KEYS = ("pages_produced", "pages_created", "pages_updated")
|
|
|
|
|
|
def lint_vault(
|
|
vault: Path,
|
|
*,
|
|
require_trust_ledger: bool = True,
|
|
strict_trust: bool = False,
|
|
allowed_relationship_types: Collection[str] | None = None,
|
|
allowed_lifecycles: Collection[str] | None = None,
|
|
required_trust_fields: Collection[str] | None = None,
|
|
schema_source: str = "framework-defaults",
|
|
) -> dict[str, Any]:
|
|
relationship_types = frozenset(
|
|
ALLOWED_RELATIONSHIP_TYPES
|
|
if allowed_relationship_types is None
|
|
else allowed_relationship_types
|
|
)
|
|
lifecycles = frozenset(
|
|
ALLOWED_LIFECYCLES if allowed_lifecycles is None else allowed_lifecycles
|
|
)
|
|
trust_fields = (
|
|
tuple(required_trust_fields)
|
|
if required_trust_fields is not None
|
|
else TRUST_REQUIRED_FRONTMATTER
|
|
)
|
|
pages = [_parse_page(path, vault) for path in iter_md(vault, SKIP_DIRS)]
|
|
page_index = PageIndex(
|
|
[page["path"] for page in pages]
|
|
+ [path.relative_to(vault).as_posix() for path in iter_instruction_files(vault, SKIP_DIRS)]
|
|
)
|
|
for page in pages:
|
|
page["id"] = page_index.id_for(page["path"])
|
|
page["resolved"] = [
|
|
(raw, page_index.resolve(raw, page["path"])) for raw in page["links"]
|
|
]
|
|
incoming: dict[str, int] = defaultdict(int)
|
|
|
|
broken_links: list[dict[str, str]] = []
|
|
for page in pages:
|
|
for raw, target in page["resolved"]:
|
|
if target is None:
|
|
broken_links.append({"page": page["path"], "target": _display_target(raw)})
|
|
elif target != page["id"]:
|
|
incoming[target] += 1
|
|
broken_links.extend(page.get("archive_broken", []))
|
|
|
|
missing_frontmatter = []
|
|
confidence_missing_fields = []
|
|
trust_metadata_errors = []
|
|
for page in pages:
|
|
if page["slug"] in RESERVED_PAGE_STEMS:
|
|
continue
|
|
missing = [field for field in REQUIRED_FRONTMATTER if field not in page["fields"]]
|
|
if missing:
|
|
missing_frontmatter.append({"page": page["path"], "missing": missing})
|
|
missing_trust = [field for field in trust_fields if field not in page["fields"]]
|
|
if missing_trust:
|
|
confidence_missing_fields.append({"page": page["path"], "missing": missing_trust})
|
|
try:
|
|
validate_trust_metadata(
|
|
vault / page["path"],
|
|
allowed_lifecycles=lifecycles,
|
|
required_trust_keys=(),
|
|
)
|
|
except ValueError as exc:
|
|
trust_metadata_errors.append({"page": page["path"], "issue": str(exc)})
|
|
|
|
# Two pages with the same slugged stem make a bare `[[stem]]` link mean
|
|
# whichever is nearer to the linking page (`links.PageIndex`), which is
|
|
# easy to get wrong. Pages in different folders stay separate graph nodes;
|
|
# `Vector Search.md` beside `vector-search.md` still share one, since
|
|
# nothing distinguishes them. Key this check with the graph's own page
|
|
# selection so it reports exactly the pages the graph sees.
|
|
# Redirect stubs (wiki-dedup): a merged page keeps the secondary page's
|
|
# title and filename so old links stay usable, but it is not a content
|
|
# page. Stub paths are excluded from the duplicate checks below, and the
|
|
# pointers themselves are validated further down instead.
|
|
stub_paths = {page["path"] for page in pages if page["redirects_to"]}
|
|
|
|
stem_index: dict[str, list[str]] = defaultdict(list)
|
|
for graph_page in iter_graph_pages(vault):
|
|
graph_rel = graph_page.relative_to(vault).as_posix()
|
|
if graph_rel in stub_paths:
|
|
continue
|
|
stem_index[graph_page_slug(graph_page, vault)].append(graph_rel)
|
|
duplicate_stems = [
|
|
{"stem": stem, "pages": sorted(paths)}
|
|
for stem, paths in sorted(stem_index.items())
|
|
if len(paths) > 1
|
|
]
|
|
|
|
title_index: dict[str, list[str]] = defaultdict(list)
|
|
for page in pages:
|
|
if page["path"] in stub_paths:
|
|
continue
|
|
title_index[page["title"].strip().lower()].append(page["path"])
|
|
duplicate_titles = [
|
|
{"title": title, "pages": paths}
|
|
for title, paths in title_index.items()
|
|
if title and len(paths) > 1
|
|
]
|
|
duplicate_titles.sort(key=lambda item: (item["title"], item["pages"]))
|
|
|
|
missing_summaries = [
|
|
page["path"]
|
|
for page in pages
|
|
if page["slug"] not in RESERVED_PAGE_STEMS
|
|
and ("summary" not in page["fields"] or not page["summary"])
|
|
]
|
|
|
|
# Legacy provenance: a `sources:` entry holding a machine absolute path cannot
|
|
# resolve on another machine, so a synced vault loses the trail. Read-only —
|
|
# migration of page frontmatter is a separate, deliberate pass (see
|
|
# docs/cli.md → Source keys and legacy migration).
|
|
machine_path_sources = [
|
|
{"page": page["path"], "sources": page["absolute_sources"]}
|
|
for page in pages
|
|
if page["slug"] not in RESERVED_PAGE_STEMS and page["absolute_sources"]
|
|
]
|
|
|
|
undated_claims = [
|
|
{"page": page["path"], **claim}
|
|
for page in pages
|
|
if page["slug"] not in RESERVED_PAGE_STEMS
|
|
for claim in page["undated_claims"]
|
|
]
|
|
|
|
orphan_pages = []
|
|
for page in pages:
|
|
if page["slug"] in RESERVED_PAGE_STEMS:
|
|
continue
|
|
outgoing = sum(1 for _, target in page["resolved"] if target and target != page["id"])
|
|
if outgoing == 0 and incoming.get(page["id"], 0) == 0:
|
|
orphan_pages.append(page["path"])
|
|
|
|
manifest_sources = _load_manifest(vault)
|
|
inverted = invert_pages(manifest_sources)
|
|
need_urls = any(
|
|
(key or "").startswith("url:") for key, _ in _iter_entries(manifest_sources)
|
|
)
|
|
url_index = clip_url_index(vault) if need_urls else None
|
|
okignore = okignore_patterns(vault)
|
|
snapshot_mismatch = []
|
|
for page in pages:
|
|
if page["slug"] in RESERVED_PAGE_STEMS:
|
|
continue
|
|
# An ignored archive neither appears in the report nor counts as a mismatch,
|
|
# whether the manifest or the page is the side still naming it.
|
|
expected = [
|
|
rel
|
|
for rel in expected_snapshots_for_page(
|
|
vault,
|
|
page["path"],
|
|
manifest_sources,
|
|
inverted=inverted,
|
|
url_index=url_index,
|
|
)
|
|
if not okignored(Path(rel), okignore)
|
|
]
|
|
if not expected:
|
|
continue
|
|
actual = [rel for rel in page["snapshots"] if not okignored(Path(rel), okignore)]
|
|
if set(expected) != set(actual):
|
|
snapshot_mismatch.append(
|
|
{
|
|
"page": page["path"],
|
|
"expected": sorted(expected),
|
|
"actual": sorted(actual),
|
|
}
|
|
)
|
|
snapshot_mismatch.sort(key=lambda item: item["page"])
|
|
|
|
# Source coverage: every ingested source should still map to a page that
|
|
# exists. A source whose page list is empty or points only at missing
|
|
# files was dropped somewhere between ingest and now (a failed write, a
|
|
# rename, a dedup), and nothing else would surface it. Entries with no
|
|
# page list at all (older count-only shapes) are skipped, not guessed at.
|
|
uncovered_sources: list[dict[str, Any]] = []
|
|
for key, entry in _iter_entries(manifest_sources):
|
|
listed = [
|
|
str(raw).strip().strip("[]")
|
|
for field in _COVERAGE_PAGE_KEYS
|
|
if isinstance(entry.get(field), list)
|
|
for raw in entry[field]
|
|
if raw
|
|
]
|
|
if not any(isinstance(entry.get(field), list) for field in _COVERAGE_PAGE_KEYS):
|
|
continue
|
|
missing = sorted(
|
|
{
|
|
page
|
|
for page in listed
|
|
if not (vault / page).is_file() and not (vault / f"{page}.md").is_file()
|
|
}
|
|
)
|
|
if not listed or len(missing) == len(set(listed)):
|
|
uncovered_sources.append(
|
|
{
|
|
"source": key or "",
|
|
"issue": "no_pages" if not listed else "pages_missing",
|
|
"missing": missing,
|
|
}
|
|
)
|
|
uncovered_sources.sort(key=lambda item: item["source"])
|
|
|
|
typed_relationship_issues: list[dict[str, Any]] = []
|
|
for page in pages:
|
|
for index, relationship in enumerate(page["relationships"]):
|
|
if "parse_error" in relationship:
|
|
typed_relationship_issues.append(
|
|
{
|
|
"page": page["path"],
|
|
"index": index,
|
|
"issue": relationship["parse_error"],
|
|
}
|
|
)
|
|
continue
|
|
relation_type = relationship.get("type", "")
|
|
target_raw = relationship.get("target", "")
|
|
if relation_type not in relationship_types:
|
|
typed_relationship_issues.append(
|
|
{
|
|
"page": page["path"],
|
|
"index": index,
|
|
"issue": "invalid_type",
|
|
"type": relation_type,
|
|
}
|
|
)
|
|
continue
|
|
target = _normalise_node_id(target_raw)
|
|
link = _relationship_link(target_raw)
|
|
if page_index.is_ambiguous(link, page["path"]):
|
|
typed_relationship_issues.append(
|
|
{
|
|
"page": page["path"],
|
|
"index": index,
|
|
"issue": "ambiguous_target",
|
|
"target": target,
|
|
}
|
|
)
|
|
continue
|
|
resolved = page_index.resolve(link, page["path"]) if link else None
|
|
if resolved is None:
|
|
typed_relationship_issues.append(
|
|
{
|
|
"page": page["path"],
|
|
"index": index,
|
|
"issue": "missing_target",
|
|
"target": target,
|
|
}
|
|
)
|
|
elif resolved == page["id"]:
|
|
typed_relationship_issues.append(
|
|
{
|
|
"page": page["path"],
|
|
"index": index,
|
|
"issue": "self_reference",
|
|
"target": target,
|
|
}
|
|
)
|
|
|
|
# Event-time validity (obsidian_wiki.temporal). Malformed dates fail: a
|
|
# page whose window can't be parsed is silently treated as current by
|
|
# retrieval, so an unreported typo means a stale claim answers as fact.
|
|
# A dangling `superseded_by` only warns — the pointer is advisory, and the
|
|
# broken_links check already covers the wikilink if the body carries one.
|
|
temporal_errors: list[dict[str, str]] = []
|
|
superseded_dangling: list[dict[str, str]] = []
|
|
for page in pages:
|
|
try:
|
|
validity_window(page["values"])
|
|
except ValueError as exc:
|
|
temporal_errors.append({"page": page["path"], "issue": str(exc)})
|
|
raw_successor = page["values"].get(SUPERSEDED_FIELD, "")
|
|
if not raw_successor:
|
|
continue
|
|
successor = _slug(superseded_target(raw_successor))
|
|
resolved_successor = page_index.resolve(successor, page["path"]) if successor else None
|
|
if not successor:
|
|
temporal_errors.append(
|
|
{"page": page["path"], "issue": f"empty {SUPERSEDED_FIELD} value"}
|
|
)
|
|
elif resolved_successor == page["id"]:
|
|
superseded_dangling.append(
|
|
{"page": page["path"], "target": successor, "issue": "self_reference"}
|
|
)
|
|
elif resolved_successor is None:
|
|
superseded_dangling.append(
|
|
{"page": page["path"], "target": successor, "issue": "missing_target"}
|
|
)
|
|
|
|
# Redirect pointers get the same treatment as `superseded_by`: a stub
|
|
# whose target is gone strands every old link that relied on it, and a
|
|
# stub pointing at another stub (or itself) never reaches content.
|
|
redirect_issues: list[dict[str, str]] = []
|
|
pages_by_id = {page["id"]: page for page in pages}
|
|
for page in pages:
|
|
if page["path"] not in stub_paths:
|
|
continue
|
|
target = _relationship_link(page["redirects_to"])
|
|
resolved_redirect = page_index.resolve(target, page["path"]) if target else None
|
|
if resolved_redirect is None:
|
|
redirect_issues.append(
|
|
{"page": page["path"], "target": target, "issue": "missing_target"}
|
|
)
|
|
elif resolved_redirect == page["id"]:
|
|
redirect_issues.append(
|
|
{"page": page["path"], "target": target, "issue": "self_reference"}
|
|
)
|
|
elif pages_by_id[resolved_redirect]["path"] in stub_paths:
|
|
redirect_issues.append(
|
|
{"page": page["path"], "target": target, "issue": "redirect_chain"}
|
|
)
|
|
|
|
ledger_path = vault / TRUST_LEDGER_RELATIVE_PATH
|
|
trust_report = (
|
|
check_trust_ledger(
|
|
vault,
|
|
ledger_path,
|
|
allowed_lifecycles=lifecycles,
|
|
required_trust_keys=trust_fields,
|
|
schema_source=schema_source,
|
|
)
|
|
if ledger_path.is_file() or require_trust_ledger
|
|
else None
|
|
)
|
|
illegal_lifecycle_transitions = (
|
|
check_lifecycle_transitions(
|
|
vault,
|
|
ledger_path,
|
|
allowed_lifecycles=lifecycles,
|
|
required_trust_keys=trust_fields,
|
|
)
|
|
if ledger_path.is_file()
|
|
else []
|
|
)
|
|
|
|
findings = {
|
|
"broken_links": broken_links,
|
|
"missing_frontmatter": missing_frontmatter,
|
|
"duplicate_titles": duplicate_titles,
|
|
"duplicate_stems": duplicate_stems,
|
|
"missing_summaries": sorted(missing_summaries),
|
|
"machine_path_sources": machine_path_sources,
|
|
"snapshot_mismatch": snapshot_mismatch,
|
|
"uncovered_sources": uncovered_sources,
|
|
"orphan_pages": sorted(orphan_pages),
|
|
"undated_claims": undated_claims,
|
|
"typed_relationship_issues": typed_relationship_issues,
|
|
# Sorted by page: these findings get diffed between CI runs.
|
|
"temporal_errors": sorted(temporal_errors, key=lambda item: item["page"]),
|
|
"superseded_dangling": sorted(superseded_dangling, key=lambda item: item["page"]),
|
|
"redirect_issues": sorted(redirect_issues, key=lambda item: item["page"]),
|
|
"confidence_missing_fields": confidence_missing_fields,
|
|
"trust_metadata_errors": trust_metadata_errors,
|
|
"confidence_review_stale": trust_report["stale"] if trust_report else [],
|
|
"confidence_unreviewed": trust_report["unreviewed"] if trust_report else [],
|
|
"confidence_mismatches": trust_report["score_mismatches"] if trust_report else [],
|
|
"confidence_ledger_errors": trust_report["errors"] if trust_report else [],
|
|
"illegal_lifecycle_transitions": illegal_lifecycle_transitions,
|
|
}
|
|
counts = {name: len(items) for name, items in findings.items()}
|
|
|
|
# Staged migration (#28, #146): a missing trust ledger or trust frontmatter
|
|
# on legacy pages only fails the vault when the owner has explicitly opted
|
|
# into strict_trust. Ledger presence alone never silently enables strict
|
|
# enforcement; core structural findings (broken links, missing core
|
|
# frontmatter) always fail regardless of trust mode.
|
|
trust_finding_names = (
|
|
"confidence_missing_fields",
|
|
"confidence_mismatches",
|
|
"confidence_ledger_errors",
|
|
"confidence_review_stale",
|
|
"confidence_unreviewed",
|
|
"illegal_lifecycle_transitions",
|
|
)
|
|
trust_findings_present = any(counts[name] for name in trust_finding_names)
|
|
trust_fails = strict_trust and any(
|
|
counts[name]
|
|
for name in (
|
|
"confidence_missing_fields",
|
|
"confidence_mismatches",
|
|
"confidence_ledger_errors",
|
|
"confidence_review_stale",
|
|
"illegal_lifecycle_transitions",
|
|
)
|
|
)
|
|
|
|
if (
|
|
counts["broken_links"]
|
|
or counts["missing_frontmatter"]
|
|
or counts["trust_metadata_errors"]
|
|
or counts["temporal_errors"]
|
|
or trust_fails
|
|
):
|
|
status = "fail"
|
|
elif (
|
|
any(
|
|
counts[name]
|
|
for name in (
|
|
"duplicate_titles",
|
|
"duplicate_stems",
|
|
"missing_summaries",
|
|
"machine_path_sources",
|
|
"snapshot_mismatch",
|
|
"uncovered_sources",
|
|
"orphan_pages",
|
|
"undated_claims",
|
|
"typed_relationship_issues",
|
|
"superseded_dangling",
|
|
"redirect_issues",
|
|
)
|
|
)
|
|
or trust_findings_present
|
|
):
|
|
status = "warn"
|
|
else:
|
|
status = "pass"
|
|
|
|
return {
|
|
"status": status,
|
|
"schema": {
|
|
"source": schema_source,
|
|
"allowed_lifecycles": sorted(lifecycles),
|
|
"allowed_relationship_types": sorted(relationship_types),
|
|
"required_trust_fields": list(trust_fields),
|
|
},
|
|
"stats": {
|
|
"pages": len(pages),
|
|
"link_count": sum(len(page["links"]) for page in pages),
|
|
"findings": counts,
|
|
"trust": trust_report["counts"] if trust_report else {"ledger": "not_configured"},
|
|
},
|
|
"findings": findings,
|
|
}
|