obsidian-wiki/obsidian_wiki/memory.py

"""The vault's memory surface: log, index, hot cache, owner profile, todo index.

Before this module these files were maintained by prose. Fifteen-odd skills each
restated "append a line to ``log.md``, add the new pages to ``index.md``, rewrite
``hot.md``" in their own words, and every one of them rewrote the prose files
wholesale with no lock. Two skills running in parallel silently dropped one of
the two updates, and nothing enforced the documented ~500-word cap on the hot
cache.

Everything here is one code path instead, serialised by the same advisory lock
the manifest writer uses (:func:`obsidian_wiki.cache.advisory_lock`) and written
atomically, so a reader never sees a torn file.

Four artifacts, three of them generated and one hybrid:

``log.md``
    Append-only, one parseable ``- [TS] VERB key=value`` line per operation.
``index.md``
    Category catalog, reconciled against what is actually on disk. A preamble
    and any non-category section are preserved verbatim.
``hot.md``
    Generated session snapshot, word-capped. ``## Key Takeaways`` is the one
    LLM-owned slot and survives every rebuild unless it is explicitly replaced.
``_meta/profile.md`` / ``_meta/todos.md``
    The owner's durable facts and open threads. Markdown tables, so a human can
    edit them in Obsidian and the parser still round-trips them.
"""

from __future__ import annotations

import os
import re
from dataclasses import dataclass, replace
from datetime import date, datetime, timedelta, timezone, tzinfo
from pathlib import Path, PurePosixPath
from typing import Iterable, Optional, Sequence

from obsidian_wiki import clock
from obsidian_wiki.cache import advisory_lock
from obsidian_wiki.provenance import archive_wikilink_relpath
from obsidian_wiki.vault import BLOCK_SCALAR_RE, FRONTMATTER_RE, iter_md, split_frontmatter
from obsidian_wiki.vault import SKIP_DIRS as VAULT_SKIP_DIRS

MEMORY_LOCK_NAME = ".memory.lock"
PROFILE_REL = "_meta/profile.md"
TODOS_REL = "_meta/todos.md"
#: A scope name namespaces the *person-shaped* memory — the profile and the
#: todo list — so one deployment can serve several users or agents. Knowledge
#: pages stay shared: a vault is a shared brain, and only what it remembers
#: *about someone* is per-someone.
_SCOPE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")

#: Directories that hold staging, archives, or tool state rather than pages.
SKIP_DIRS = VAULT_SKIP_DIRS | {"_readouts", "_meta"}
#: Root files that are the memory surface itself, not catalog entries.
SKIP_FILES = frozenset({
    "index.md", "log.md", "hot.md", "_insights.md",
    "AGENTS.md", "CLAUDE.md", "GEMINI.md", "README.md",
})

#: Canonical ordering for index sections; anything else sorts after, alphabetically.
CATEGORY_ORDER = (
    "concepts", "entities", "skills", "references",
    "synthesis", "journal", "projects",
)

DEFAULT_HOT_MAX_WORDS = 500
DEFAULT_RECENT_ACTIVITY = 3
DEFAULT_ACTIVE_THREADS = 7
DEFAULT_CONTRADICTIONS = 5
DEFAULT_TODO_STALE_DAYS = 30

#: Frontmatter marker proving a file is ours to regenerate. A vault whose
#: index.md or hot.md predates this module is hand-curated: regenerating it
#: would reorder or discard someone's work, so we refuse until `memory migrate`
#: has run and taken a backup.
GENERATED_MARKER = "obsidian-wiki memory"
#: Durable adoption record. The per-file marker alone was fragile: a skill
#: that still rewrites hot.md by hand drops the marker, and the next sync
#: would then refuse as if the vault had never been migrated. Adoption is a
#: property of the vault, so it lives in the vault, not in a file a legacy
#: skill can overwrite.
ADOPTED_REL = "_meta/.memory-adopted"
#: Raw log lines can be enormous (a research ingest records every page it made).
#: Pasting them into a word-capped snapshot spends the whole budget on one line.
HOT_FIELDS_PER_ENTRY = 3
HOT_VALUE_CHARS = 60

_LOG_LINE_RE = re.compile(r"^-\s*\[(?P<ts>[^\]]+)\]\s+(?P<verb>[A-Z][A-Z0-9_-]*)\s*(?P<rest>.*)$")
_FIELD_RE = re.compile(r"""(?P<key>[A-Za-z_][\w-]*)=(?P<value>"[^"]*"|'[^']*'|\S*)""")
_VERB_RE = re.compile(r"^[A-Z][A-Z0-9_-]*$")
_TAG_RE = re.compile(r"#[\w/-]+")
_HEADING_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE)
_MD_DECORATION_RE = re.compile(r"[*_`]|^\s*[-*+]\s+|^\s*>\s?|^#{1,6}\s+")
_WIKILINK_RE = re.compile(r"\[\[([^\]|#]+?)(?:[|#][^\]]*?)?\]\]")
#: Split a table row on unescaped pipes only — `\|` is a literal in a cell.
_CELL_SPLIT_RE = re.compile(r"(?<!\\)\|")


class MemoryError_(RuntimeError):
    """A memory-surface operation could not be completed."""

    def __init__(self, code: str, message: str) -> None:
        super().__init__(message)
        self.code = code


# --------------------------------------------------------------------------
# primitives
# --------------------------------------------------------------------------


def utc_now() -> str:
    """UTC in the ``Z`` spelling: the stamp this writer produced before vaults had a timezone."""
    return clock.stamp(None)


def now_stamp(vault: Path) -> str:
    """The full timestamp for a write to *vault*, in the vault's configured zone.

    ``timezone:`` in ``index.md`` picks the zone; without it this is
    :func:`utc_now`. See :mod:`obsidian_wiki.clock`.
    """
    return clock.stamp(clock.wiki_timezone(vault))


def today(tz: Optional[tzinfo] = None) -> str:
    """Bare calendar date in *tz* (UTC when the vault has no zone)."""
    return clock.today(tz)


def memory_lock(vault: Path, *, timeout: float = 10.0):
    """Serialise writes to the prose memory files.

    Deliberately one lock for all four artifacts rather than one each: a skill
    that finishes an ingest touches the log, the index, and the hot cache as a
    single logical update, and interleaving another writer between them is what
    produced inconsistent snapshots before.
    """
    return advisory_lock(Path(vault) / MEMORY_LOCK_NAME, timeout=timeout)


def atomic_write(path: Path, text: str) -> None:
    """Replace *path* with *text* via a temp file and ``os.replace``."""
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
    try:
        tmp.write_text(text, encoding="utf-8")
        os.replace(tmp, path)  # atomic on POSIX and Windows
    except BaseException:
        try:
            tmp.unlink()
        except FileNotFoundError:
            pass
        raise


def _require_vault(vault: Path) -> Path:
    vault = Path(vault)
    if not vault.is_dir():
        raise MemoryError_("vault_not_found", f"vault not found: {vault}")
    return vault


def _scalar(value: str) -> str:
    value = value.strip()
    if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
        return value[1:-1].strip()
    return value


def parse_frontmatter(frontmatter: str) -> dict:
    """Scalars (including blocks), inline lists, and block lists for page headers.

    Fold both literal and folded blocks to one line, as in graphrag, since
    memory surfaces use titles and summaries as single-line previews.
    """
    values: dict = {}
    lines = frontmatter.splitlines()
    i = 0
    while i < len(lines):
        line = lines[i]
        if not line.strip() or line.startswith((" ", "\t")) or ":" not in line:
            i += 1
            continue
        key, raw = line.split(":", 1)
        key, raw = key.strip(), raw.strip()
        if raw.startswith("[") and raw.endswith("]"):
            values[key] = [_scalar(part) for part in raw[1:-1].split(",") if part.strip()]
            i += 1
            continue
        if BLOCK_SCALAR_RE.match(raw):
            block_lines = []
            j = i + 1
            while j < len(lines) and (lines[j].startswith((" ", "\t")) or not lines[j].strip()):
                stripped = lines[j].strip()
                if stripped:
                    block_lines.append(stripped)
                j += 1
            values[key] = " ".join(block_lines).strip()
            i = j
            continue
        if not raw:
            block: list = []
            j = i + 1
            while j < len(lines) and (lines[j].startswith((" ", "\t")) or not lines[j].strip()):
                stripped = lines[j].strip()
                if stripped.startswith("- "):
                    block.append(stripped[2:].strip())
                elif stripped and block and ":" in stripped:
                    # continuation of a mapping item, e.g. `type: contradicts`
                    block.append(stripped)
                j += 1
            values[key] = block if block else ""
            i = j
            continue
        values[key] = _scalar(raw)
        i += 1
    return values


def is_adopted(vault: Path) -> bool:
    return (Path(vault) / ADOPTED_REL).is_file()


def mark_adopted(vault: Path) -> None:
    """Record that this vault's memory files are ours to regenerate.

    Delete the file to re-arm the guard — say, after restoring a curated
    index.md from ``_archives/`` — and ``memory migrate`` will ask again.
    """
    atomic_write(Path(vault) / ADOPTED_REL, f"adopted: {now_stamp(vault)}\nby: {GENERATED_MARKER}\n")


def check_scope(scope: str) -> str:
    """Validate a scope name. It becomes a filename, so it is never trusted.

    In a server deployment the scope arrives as a ``user_id`` from a request,
    which makes it exactly the sort of value that must not contain ``..`` or a
    path separator.
    """
    scope = (scope or "").strip()
    if not scope:
        return ""
    if not _SCOPE_RE.match(scope):
        raise MemoryError_(
            "bad_scope",
            f"scope must be 1-64 chars of letters, digits, '.', '_' or '-': {scope!r}",
        )
    return scope


def profile_path(vault: Path, scope: str = "") -> Path:
    scope = check_scope(scope)
    return Path(vault) / (f"_meta/profile.{scope}.md" if scope else PROFILE_REL)


def todos_path(vault: Path, scope: str = "") -> Path:
    scope = check_scope(scope)
    return Path(vault) / (f"_meta/todos.{scope}.md" if scope else TODOS_REL)


def list_scopes(vault: Path) -> list:
    """Scope names that have a profile or a todo list in this vault."""
    meta = Path(vault) / "_meta"
    if not meta.is_dir():
        return []
    names = set()
    for path in meta.glob("*.md"):
        for prefix in ("profile.", "todos."):
            if path.name.startswith(prefix) and path.name != f"{prefix}md":
                names.add(path.name[len(prefix):-3])
    return sorted(names)


def is_generated(path: Path) -> bool:
    """True when *path* carries our ``generated_by`` marker.

    Absence means a human or an older version of the framework wrote it, so it
    is not ours to overwrite.
    """
    path = Path(path)
    if not path.is_file():
        return True  # nothing to clobber
    head = path.read_text(encoding="utf-8", errors="replace")[:2000]
    return f"generated_by: {GENERATED_MARKER}" in head


# --------------------------------------------------------------------------
# log.md
# --------------------------------------------------------------------------


@dataclass(frozen=True)
class LogEntry:
    timestamp: str
    verb: str
    fields: dict
    raw: str


def format_log_line(verb: str, fields: Optional[dict] = None, *, timestamp: Optional[str] = None) -> str:
    """Render one ``- [TS] VERB key=value`` line in the documented format."""
    verb = verb.strip().upper()
    if not _VERB_RE.match(verb):
        raise MemoryError_("bad_verb", f"log verb must be UPPERCASE alphanumeric: {verb!r}")
    parts = [f"- [{timestamp or utc_now()}] {verb}"]
    for key, value in (fields or {}).items():
        text = "" if value is None else str(value)
        text = text.replace("\n", " ").replace('"', "'")
        parts.append(f'{key}="{text}"' if (" " in text or not text) else f"{key}={text}")
    return " ".join(parts)


def parse_log_line(line: str) -> Optional[LogEntry]:
    match = _LOG_LINE_RE.match(line.strip())
    if not match:
        return None
    fields = {
        found.group("key"): _scalar(found.group("value"))
        for found in _FIELD_RE.finditer(match.group("rest"))
    }
    return LogEntry(match.group("ts"), match.group("verb"), fields, line.rstrip())


def append_log(
    vault: Path,
    verb: str,
    fields: Optional[dict] = None,
    *,
    timestamp: Optional[str] = None,
    lock: bool = True,
) -> str:
    """Append one operation line to ``log.md``. Returns the line written.

    Append-only and O_APPEND, so this stays correct even for a writer that
    skipped the lock; the lock is still taken by default so a caller doing
    log+index+hot as one update holds it across all three.
    """
    vault = _require_vault(vault)
    line = format_log_line(verb, fields, timestamp=timestamp or now_stamp(vault))
    log = vault / "log.md"

    def _write() -> None:
        if not log.exists():
            atomic_write(log, "---\ntitle: Wiki Log\n---\n\n# Wiki Log\n\n")
        existing = log.read_text(encoding="utf-8")
        prefix = "" if existing.endswith("\n") or not existing else "\n"
        with log.open("a", encoding="utf-8") as handle:
            handle.write(f"{prefix}{line}\n")

    if lock:
        with memory_lock(vault):
            _write()
    else:
        _write()
    return line


def read_log(vault: Path, *, limit: Optional[int] = None, verbs: Optional[Iterable[str]] = None) -> list:
    """Parsed log entries, newest last. *limit* keeps the newest *limit* entries."""
    log = Path(vault) / "log.md"
    if not log.is_file():
        return []
    wanted = {v.upper() for v in verbs} if verbs else None
    entries = []
    for line in log.read_text(encoding="utf-8").splitlines():
        entry = parse_log_line(line)
        if entry and (wanted is None or entry.verb in wanted):
            entries.append(entry)
    return entries[-limit:] if limit else entries


# --------------------------------------------------------------------------
# page scan
# --------------------------------------------------------------------------


@dataclass(frozen=True)
class PageInfo:
    path: str
    category: str
    title: str
    summary: str
    tags: tuple
    updated: str
    lifecycle: str
    contradicts: tuple


def _summary_from_body(body: str) -> str:
    for line in body.splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith(("#", ">", "---", "|", "!", "*[")):
            continue
        if stripped.startswith("*") and stripped.endswith("*") and len(stripped) > 2:
            continue  # the italic "this file is generated" note
        text = _MD_DECORATION_RE.sub("", stripped).strip()
        if text:
            return text[:200]
    return ""


def _contradicts_targets(values: dict) -> tuple:
    """Targets of ``relationships:`` items whose ``type`` is ``contradicts``."""
    raw = values.get("relationships")
    if not isinstance(raw, list):
        return ()
    targets: list = []
    pending_target = None
    pending_type = None
    for item in raw:
        text = item.strip()
        if text.startswith("target:"):
            if pending_target and pending_type == "contradicts":
                targets.append(pending_target)
            pending_target, pending_type = _scalar(text[7:]), None
        elif text.startswith("type:"):
            pending_type = _scalar(text[5:])
            if pending_type == "contradicts" and pending_target:
                targets.append(pending_target)
                pending_target, pending_type = None, None
    if pending_target and pending_type == "contradicts":
        targets.append(pending_target)
    cleaned = []
    for target in targets:
        match = _WIKILINK_RE.search(target)
        cleaned.append(match.group(1).strip() if match else target)
    return tuple(cleaned)


def page_from_path(path: Path, vault: Path) -> PageInfo:
    rel = path.relative_to(vault)
    text = path.read_text(encoding="utf-8", errors="replace")
    frontmatter, body = split_frontmatter(text)
    values = parse_frontmatter(frontmatter)
    tags_raw = values.get("tags", [])
    if isinstance(tags_raw, str):
        tags = tuple(tag.lstrip("#") for tag in _TAG_RE.findall(tags_raw)) or tuple(
            part.strip().lstrip("#") for part in tags_raw.split(",") if part.strip()
        )
    else:
        tags = tuple(str(tag).strip().lstrip("#") for tag in tags_raw if str(tag).strip())
    title = str(values.get("title") or "").strip() or path.stem.replace("-", " ")
    summary = str(values.get("summary") or "").strip() or _summary_from_body(body)
    return PageInfo(
        path=rel.as_posix(),
        category=rel.parts[0] if len(rel.parts) > 1 else "",
        title=title,
        summary=summary,
        tags=tags,
        updated=str(values.get("updated") or "").strip(),
        lifecycle=str(values.get("lifecycle") or "").strip(),
        contradicts=_contradicts_targets(values),
    )


def scan_pages(vault: Path) -> list:
    """Every real wiki page, excluding staging, archives, and the memory files."""
    vault = _require_vault(vault)
    pages = []
    for path in iter_md(vault, SKIP_DIRS):
        if path.name in SKIP_FILES and path.parent == vault:
            continue
        try:
            pages.append(page_from_path(path, vault))
        except OSError:
            continue
    return pages


def _category_sort_key(category: str):
    label = category or "uncategorized"
    try:
        return (0, CATEGORY_ORDER.index(label), label)
    except ValueError:
        return (1, 0, label)


def _heading_for(category: str) -> str:
    return (category or "uncategorized").replace("-", " ").replace("_", " ").title()


# --------------------------------------------------------------------------
# index.md
# --------------------------------------------------------------------------


#: Characters a ``[[wikilink]]`` target cannot carry: ``]`` ends the link, ``#``
#: starts a heading anchor and ``|`` starts an alias. A page whose name holds one
#: of them is unlinkable in that form.
_WIKILINK_UNSAFE = frozenset("]#|")


def _wikilink_target_is_valid(node: str) -> bool:
    return not (_WIKILINK_UNSAFE & set(node))


def _markdown_link_text(title: str) -> str:
    """Escape ``]`` so a title cannot close the link text early."""
    return title.replace("]", r"\]")


def _index_entry(page: PageInfo, link_format: str, *, hub_pages: Optional[int] = None) -> str:
    node = page.path[:-3] if page.path.endswith(".md") else page.path
    if link_format == "markdown" or not _wikilink_target_is_valid(node):
        # A name a wikilink cannot express falls back to the markdown form so
        # ``rebuild_index`` can read its own output back (and hand the entry to
        # Obsidian in a form it can resolve).
        link = f"[{_markdown_link_text(page.title)}]({page.path})"
    elif hub_pages is not None:
        # A hub's node ends in `/index`, which Obsidian would display as
        # "index"; the alias shows the folder's own title instead.
        link = f"[[{node}|{page.title}]]"
    else:
        link = f"[[{node}]]"
    # Format rule: a space after the opening paren, or tag parsing breaks.
    tags = f" ( {' '.join('#' + tag for tag in page.tags)})" if page.tags else ""
    summary = f" — {page.summary}" if page.summary else ""
    count = "" if hub_pages is None else f" ({hub_pages} page{'' if hub_pages == 1 else 's'})"
    return f"- {link}{summary}{count}{tags}"


#: Root ``index.md`` frontmatter key that turns on folder hubs (see
#: ``collapse_folder_hubs``). It lives in the file it governs, so the setting
#: travels with the vault instead of depending on which ``.env`` a caller saw.
FOLDER_HUBS_KEY = "index_folder_hubs"

_H1_RE = re.compile(r"^#\s+(.+?)\s*$", re.MULTILINE)


def folder_hubs_enabled(index_text: str) -> bool:
    values = parse_frontmatter(split_frontmatter(index_text)[0])
    return str(values.get(FOLDER_HUBS_KEY) or "").strip().lower() in {"true", "yes", "1"}


def collapse_folder_hubs(vault: Path, pages: Sequence) -> tuple:
    """Replace every page under a folder that has its own ``index.md`` with
    one entry for that index, so a sub-wiki is linked rather than inlined.

    Returns ``(entries, hub_counts)``: ``entries`` keeps every page outside a
    hub plus each hub's index page, and ``hub_counts`` maps a hub index's path
    to how many pages it stands for. The outermost hub wins when hubs nest.
    """
    hubs = sorted(
        {
            str(PurePosixPath(page.path).parent)
            for page in pages
            if PurePosixPath(page.path).name == "index.md" and "/" in page.path
        },
        key=len,
    )
    outermost: list = []
    for hub in hubs:
        if not any(hub.startswith(f"{outer}/") for outer in outermost):
            outermost.append(hub)

    entries, hub_counts = [], {}
    by_path = {page.path: page for page in pages}
    for page in pages:
        hub = next((h for h in outermost if page.path.startswith(f"{h}/")), None)
        if hub is None:
            entries.append(page)
        else:
            hub_counts[f"{hub}/index.md"] = hub_counts.get(f"{hub}/index.md", 0) + 1
    for hub_path, count in hub_counts.items():
        entries.append(_hub_page(vault, by_path[hub_path], count - 1))
        hub_counts[hub_path] = count - 1  # the index itself is not one of its pages
    return entries, hub_counts


def _hub_page(vault: Path, page: PageInfo, count: int) -> PageInfo:
    """The hub's index page, titled for display: frontmatter title, else its
    H1, else the folder name — never the bare stem "index"."""
    title = page.title
    if title == "index":
        body = split_frontmatter((vault / page.path).read_text(encoding="utf-8", errors="replace"))[1]
        heading = _H1_RE.search(body)
        folder = PurePosixPath(page.path).parent.name
        title = heading.group(1) if heading else folder.replace("-", " ").replace("_", " ").title()
    return replace(page, title=title)


@dataclass(frozen=True)
class IndexResult:
    added: tuple
    removed: tuple
    total: int
    changed: bool
    text: str


def render_index(
    pages: Sequence,
    *,
    link_format: str = "wikilink",
    preamble: str = "",
    extra_sections: str = "",
    hub_counts: Optional[dict] = None,
) -> str:
    hub_counts = hub_counts or {}
    by_category: dict = {}
    for page in pages:
        by_category.setdefault(page.category, []).append(page)
    chunks = [_ensure_index_marker(preamble.rstrip("\n"))]
    # Sections the author wrote come first, in their original order. Putting
    # the generated catalog above them reordered a curated document and pushed
    # hand-written reference material below a wall of auto-generated entries.
    if extra_sections.strip():
        chunks.append(extra_sections.strip("\n"))
    for category in sorted(by_category, key=_category_sort_key):
        entries = sorted(by_category[category], key=lambda p: (p.title.casefold(), p.path))
        chunks.append(
            f"## {_heading_for(category)}\n\n"
            + "\n".join(
                _index_entry(page, link_format, hub_pages=hub_counts.get(page.path))
                for page in entries
            )
        )
    if not by_category:
        chunks.append("## Concepts\n\n*No pages yet. Use `wiki-ingest` to add your first source.*")
    return "\n\n".join(chunks) + "\n"


def _ensure_index_marker(preamble: str) -> str:
    """Guarantee the preamble carries the ``generated_by`` marker.

    Migration preserves a vault's own preamble verbatim, so the marker has to
    be injected into it — otherwise the file stays unmarked, every later call
    sees an unmigrated vault, and the guard refuses in perpetuity.
    """
    marker = f"generated_by: {GENERATED_MARKER} index"
    if marker in preamble:
        return preamble
    match = FRONTMATTER_RE.match(preamble + "\n")
    if match:
        frontmatter = match.group(1)
        if re.search(r"^generated_by:", frontmatter, re.MULTILINE):
            frontmatter = re.sub(r"^generated_by:.*$", marker, frontmatter, count=1, flags=re.MULTILINE)
        else:
            frontmatter = f"{frontmatter}\n{marker}"
        return f"---\n{frontmatter}\n---" + preamble[match.end() - 1:].rstrip("\n")
    body = preamble.lstrip("\n")
    return f"---\ntitle: Wiki Index\n{marker}\n---\n\n{body}" if body else \
        f"---\ntitle: Wiki Index\n{marker}\n---\n\n# Wiki Index"


def _split_index(text: str) -> tuple:
    """Preamble, category-section paths, and verbatim non-category sections.

    A section is treated as generated when its heading matches a category that
    exists on disk; anything else is a human's own section and is preserved.
    """
    headings = list(_HEADING_RE.finditer(text))
    if not headings:
        return text, {}, ""
    preamble = text[: headings[0].start()].rstrip("\n")
    sections = {}
    for position, match in enumerate(headings):
        end = headings[position + 1].start() if position + 1 < len(headings) else len(text)
        sections[match.group(1).strip().casefold()] = text[match.start(): end]
    return preamble, sections, ""


def rebuild_index(
    vault: Path,
    *,
    link_format: str = "wikilink",
    write: bool = True,
    lock: bool = True,
    force: bool = False,
) -> IndexResult:
    """Reconcile ``index.md`` against the pages actually on disk.

    Regenerates one section per category and preserves the preamble plus any
    section whose heading is not a category — a hand-written "Reading queue"
    section survives, a stale page entry does not.

    With ``index_folder_hubs: true`` in the index frontmatter, a folder that
    has its own ``index.md`` is listed once, as a link to that index, instead
    of page by page (see ``collapse_folder_hubs``).
    """
    vault = _require_vault(vault)
    index = vault / "index.md"
    if write and not force and not (is_adopted(vault) or is_generated(index)):
        raise MemoryError_(
            "unmigrated",
            "index.md was not written by this tool; run `obsidian-wiki memory migrate` "
            "to review the change and take a backup first",
        )
    pages = scan_pages(vault)
    existing = index.read_text(encoding="utf-8") if index.is_file() else ""
    preamble, sections, _ = _split_index(existing)

    # A heading counts as generated when it names a category, whether or not a
    # page currently lives there. Deriving this from pages alone meant deleting
    # the last page in a category left its stale section behind, preserved as
    # though a human had written it.
    categories = {_heading_for(name).casefold() for name in CATEGORY_ORDER}
    categories.add(_heading_for("").casefold())
    categories |= {_heading_for(page.category).casefold() for page in pages}
    categories |= {
        _heading_for(child.name).casefold()
        for child in vault.iterdir()
        if child.is_dir() and child.name not in SKIP_DIRS and not child.name.startswith(".")
    }
    extra = "\n\n".join(
        body.strip("\n") for heading, body in sections.items() if heading not in categories
    )

    # Only the link that opens an entry line is what the index lists; links in
    # the entry's summary text point elsewhere and are not catalog entries.
    listed = set()
    for heading, body in sections.items():
        if heading in categories:
            for line in body.splitlines():
                entry = re.match(r"^-\s*(\[\[[^\]]+\]\])", line.strip())
                if entry:
                    match = _WIKILINK_RE.match(entry.group(1))
                    if match and archive_wikilink_relpath(vault, match.group(1)) is None:
                        listed.add(match.group(1).strip())
                    continue
                # Link text may carry an escaped ``]`` (see _index_entry); the
                # capture is the path, so the escape only has to be skipped.
                md = re.match(r"^-\s*\[(?:\\.|[^\]])*\]\(([^)]+)\)", line.strip())
                if md:
                    listed.add(md.group(1)[:-3] if md.group(1).endswith(".md") else md.group(1))

    entries, hub_counts = (
        collapse_folder_hubs(vault, pages) if folder_hubs_enabled(existing) else (pages, {})
    )
    current = {page.path[:-3] for page in entries}
    added = tuple(sorted(current - listed))
    removed = tuple(sorted(listed - current))
    text = render_index(
        entries,
        link_format=link_format,
        preamble=preamble,
        extra_sections=extra,
        hub_counts=hub_counts,
    )
    changed = text != existing

    if write and changed:
        if lock:
            with memory_lock(vault):
                atomic_write(index, text)
        else:
            atomic_write(index, text)
    return IndexResult(added=added, removed=removed, total=len(pages), changed=changed, text=text)


# --------------------------------------------------------------------------
# _meta/profile.md — who the vault owner is
# --------------------------------------------------------------------------


@dataclass(frozen=True)
class Fact:
    key: str
    value: str
    confidence: float
    source: str
    updated: str
    until: str = ""  # YYYY-MM-DD after which the fact no longer holds; empty = durable

    def is_expired(self, on: Optional[str] = None) -> bool:
        return bool(self.until) and self.until < (on or today())


def _cell(text: str) -> str:
    return str(text).replace("|", r"\|").replace("\n", " ").strip()


def _uncell(text: str) -> str:
    return text.replace(r"\|", "|").strip()


def _parse_table(text: str, columns: int, optional: int = 0) -> list:
    """Rows of the first markdown table in *text*, header and rule dropped.

    *optional* trailing columns may be missing (a table written before they
    existed); they come back as empty strings.
    """
    rows = []
    for line in text.splitlines():
        stripped = line.strip()
        if not stripped.startswith("|") or not stripped.endswith("|"):
            continue
        cells = [_uncell(cell) for cell in _CELL_SPLIT_RE.split(stripped[1:-1])]
        if len(cells) < columns:
            continue
        if all(set(cell) <= {"-", ":", " "} and cell for cell in cells):
            continue  # the |---|---| rule
        cells += [""] * optional
        rows.append(cells[: columns + optional])
    return rows[1:] if rows else []  # drop the header row


_PROFILE_HEADER = """---
title: Owner Profile
category: _meta
tags: [meta/profile]
sources: [conversation]
created: {created}
updated: {updated}
generated_by: obsidian-wiki memory profile
---

# Owner Profile

*Durable facts about the vault owner, injected into an agent's context at session
start. Written by `obsidian-wiki memory profile set`; safe to edit by hand — the
table round-trips. Confidence is the writer's own calibration, not a measurement.
A fact with an Until date drops out of the session recap once that date passes.*

| Fact | Value | Confidence | Source | Updated | Until |
|---|---|---|---|---|---|
"""


def load_profile(vault: Path, scope: str = "") -> list:
    path = profile_path(vault, scope)
    if not path.is_file():
        return []
    facts = []
    for row in _parse_table(path.read_text(encoding="utf-8"), 5, optional=1):
        key, value, confidence, source, updated, until = row
        if not key:
            continue
        try:
            score = float(confidence)
        except ValueError:
            score = 0.5
        facts.append(Fact(key, value, score, source, updated, until))
    return facts


def render_profile(
    facts: Sequence, *, created: str = "", updated: str = "", tz: Optional[tzinfo] = None
) -> str:
    header = _PROFILE_HEADER.format(created=created or today(tz), updated=updated or today(tz))
    rows = "\n".join(
        f"| {_cell(f.key)} | {_cell(f.value)} | {f.confidence:.2f} | {_cell(f.source)} | {_cell(f.updated)} | {_cell(f.until)} |"
        for f in sorted(facts, key=lambda f: f.key.casefold())
    )
    return header + (rows + "\n" if rows else "")


def set_fact(
    vault: Path,
    key: str,
    value: str,
    *,
    confidence: float = 0.6,
    source: str = "conversation",
    until: str = "",
    scope: str = "",
    lock: bool = True,
) -> Fact:
    """Add or replace one fact about the owner.

    *until* (YYYY-MM-DD) marks a time-bound fact — "exams this weekend" — so it
    stops being injected once it is over instead of lingering until a human
    notices. The row stays in the table; `forget` removes it.
    """
    vault = _require_vault(vault)
    key = key.strip()
    if not key:
        raise MemoryError_("bad_key", "fact key must not be empty")
    if not 0.0 <= confidence <= 1.0:
        raise MemoryError_("bad_confidence", f"confidence must be in [0.0, 1.0]: {confidence}")
    until = until.strip()
    if until:
        try:
            until = date.fromisoformat(until).isoformat()
        except ValueError:
            raise MemoryError_("bad_until", f"until must be YYYY-MM-DD: {until}") from None
    tz = clock.wiki_timezone(vault)
    fact = Fact(key, value.strip(), confidence, source.strip(), today(tz), until)

    def _write() -> None:
        facts = [f for f in load_profile(vault, scope) if f.key.casefold() != key.casefold()]
        facts.append(fact)
        atomic_write(profile_path(vault, scope), render_profile(facts, tz=tz))

    if lock:
        with memory_lock(vault):
            _write()
    else:
        _write()
    return fact


def forget_fact(vault: Path, key: str, *, scope: str = "", lock: bool = True) -> bool:
    vault = _require_vault(vault)

    def _write() -> bool:
        facts = load_profile(vault, scope)
        kept = [f for f in facts if f.key.casefold() != key.strip().casefold()]
        if len(kept) == len(facts):
            return False
        atomic_write(profile_path(vault, scope), render_profile(kept, tz=clock.wiki_timezone(vault)))
        return True

    if lock:
        with memory_lock(vault):
            return _write()
    return _write()


# --------------------------------------------------------------------------
# _meta/todos.md — open threads
# --------------------------------------------------------------------------


@dataclass(frozen=True)
class Todo:
    id: str
    text: str
    status: str  # open | done | dropped
    origin: str
    created: str
    touched: str

    def is_stale(self, *, days: int = DEFAULT_TODO_STALE_DAYS, now: Optional[datetime] = None) -> bool:
        if self.status != "open" or not self.touched:
            return False
        try:
            touched = datetime.strptime(self.touched, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        except ValueError:
            return False
        return (now or datetime.now(timezone.utc)) - touched > timedelta(days=days)


TODO_STATUSES = ("open", "done", "dropped")

_TODOS_HEADER = """---
title: Todo Index
category: _meta
tags: [meta/todos]
sources: [conversation]
created: {created}
updated: {updated}
generated_by: obsidian-wiki memory todo
---

# Todo Index

*Open threads carried between sessions. Written by `obsidian-wiki memory todo`;
safe to edit by hand — the table round-trips. An open item untouched for
{stale} days is reported stale rather than silently kept alive.*

| ID | Status | Thread | Origin | Created | Touched |
|---|---|---|---|---|---|
"""


def load_todos(vault: Path, scope: str = "") -> list:
    path = todos_path(vault, scope)
    if not path.is_file():
        return []
    todos = []
    for row in _parse_table(path.read_text(encoding="utf-8"), 6):
        todo_id, status, text, origin, created, touched = row
        if not todo_id:
            continue
        todos.append(Todo(todo_id, text, status or "open", origin, created, touched))
    return todos


def render_todos(
    todos: Sequence, *, created: str = "", updated: str = "", tz: Optional[tzinfo] = None
) -> str:
    header = _TODOS_HEADER.format(
        created=created or today(tz), updated=updated or today(tz), stale=DEFAULT_TODO_STALE_DAYS
    )
    order = {status: position for position, status in enumerate(TODO_STATUSES)}
    rows = "\n".join(
        f"| {_cell(t.id)} | {_cell(t.status)} | {_cell(t.text)} | {_cell(t.origin)} "
        f"| {_cell(t.created)} | {_cell(t.touched)} |"
        for t in sorted(todos, key=lambda t: (order.get(t.status, 9), _todo_number(t.id)))
    )
    return header + (rows + "\n" if rows else "")


def _todo_number(todo_id: str) -> int:
    match = re.search(r"(\d+)$", todo_id or "")
    return int(match.group(1)) if match else 0


def _next_todo_id(todos: Sequence) -> str:
    return f"t{max((_todo_number(t.id) for t in todos), default=0) + 1}"


def add_todo(vault: Path, text: str, *, origin: str = "", scope: str = "", lock: bool = True) -> Todo:
    vault = _require_vault(vault)
    text = text.strip()
    if not text:
        raise MemoryError_("bad_todo", "todo text must not be empty")
    tz = clock.wiki_timezone(vault)
    holder: list = []

    def _write() -> None:
        todos = load_todos(vault, scope)
        existing = next((t for t in todos if t.text.casefold() == text.casefold() and t.status == "open"), None)
        if existing is not None:  # idempotent: re-adding an open thread just touches it
            todo = replace(existing, touched=today(tz))
            todos = [todo if t.id == existing.id else t for t in todos]
        else:
            todo = Todo(_next_todo_id(todos), text, "open", origin.strip(), today(tz), today(tz))
            todos.append(todo)
        atomic_write(todos_path(vault, scope), render_todos(todos, tz=tz))
        holder.append(todo)

    if lock:
        with memory_lock(vault):
            _write()
    else:
        _write()
    return holder[0]


def set_todo_status(vault: Path, todo_id: str, status: str, *, scope: str = "", lock: bool = True) -> Todo:
    vault = _require_vault(vault)
    if status not in TODO_STATUSES:
        raise MemoryError_("bad_status", f"status must be one of {', '.join(TODO_STATUSES)}")
    tz = clock.wiki_timezone(vault)
    holder: list = []

    def _write() -> None:
        todos = load_todos(vault, scope)
        match = next((t for t in todos if t.id.casefold() == todo_id.strip().casefold()), None)
        if match is None:
            raise MemoryError_("no_such_todo", f"no todo with id {todo_id!r}")
        updated = replace(match, status=status, touched=today(tz))
        atomic_write(
            todos_path(vault, scope),
            render_todos([updated if t.id == match.id else t for t in todos], tz=tz),
        )
        holder.append(updated)

    if lock:
        with memory_lock(vault):
            _write()
    else:
        _write()
    return holder[0]


def prune_todos(vault: Path, *, scope: str = "", lock: bool = True) -> int:
    """Drop closed items from the table. Returns how many were removed."""
    vault = _require_vault(vault)
    holder: list = []

    def _write() -> None:
        todos = load_todos(vault, scope)
        kept = [t for t in todos if t.status == "open"]
        holder.append(len(todos) - len(kept))
        if len(kept) != len(todos):
            atomic_write(todos_path(vault, scope), render_todos(kept, tz=clock.wiki_timezone(vault)))

    if lock:
        with memory_lock(vault):
            _write()
    else:
        _write()
    return holder[0]


# --------------------------------------------------------------------------
# hot.md — the generated session snapshot
# --------------------------------------------------------------------------


_HOT_NOTE = (
    "*Generated snapshot of recent vault activity. Rebuilt by "
    "`obsidian-wiki memory hot` — edits to the generated sections are "
    "overwritten, so put durable prose in `## Key Takeaways`, which survives "
    "every rebuild.*"
)
_TAKEAWAYS_PLACEHOLDER = "*None yet.*"


def _section_body(text: str, heading: str, *, prefix: bool = False) -> str:
    suffix = r".*$" if prefix else r"\s*$"
    pattern = re.compile(rf"^##\s+{re.escape(heading)}{suffix}", re.MULTILINE | re.IGNORECASE)
    match = pattern.search(text)
    if not match:
        return ""
    rest = text[match.end():]
    following = _HEADING_RE.search(rest)
    return (rest[: following.start()] if following else rest).strip()


def word_count(text: str) -> int:
    return len(text.split())


def content_words(text: str) -> int:
    """Words a reader actually consumes: no frontmatter, no generated note.

    The documented cap is on the snapshot's content. Counting the YAML header
    and the "this file is generated" note against it put the floor within a few
    words of the cap, so trimming could spin without ever getting under budget.
    """
    body = split_frontmatter(text)[1].replace(_HOT_NOTE, "")
    return len(body.split())


def hot_max_words() -> int:
    raw = os.environ.get("OBSIDIAN_HOT_MAX_WORDS", "").strip()
    try:
        value = int(raw)
    except ValueError:
        return DEFAULT_HOT_MAX_WORDS
    return value if value > 0 else DEFAULT_HOT_MAX_WORDS


@dataclass(frozen=True)
class HotResult:
    words: int
    trimmed: bool
    activity: int
    threads: int
    contradictions: int
    text: str


def summarize_log_entry(entry: LogEntry) -> str:
    """One compact line for the hot cache.

    A raw log line is unbounded: a research ingest records every page it
    produced in a single ``note=`` field, which alone can exceed the whole
    word budget. Keep the date, the verb, and the first few fields.
    """
    date = entry.timestamp.split("T")[0]
    parts = []
    for index, (key, value) in enumerate(entry.fields.items()):
        if index >= HOT_FIELDS_PER_ENTRY:
            parts.append("...")
            break
        if len(value) > HOT_VALUE_CHARS:
            value = value[: HOT_VALUE_CHARS - 1] + "..."
        parts.append(f"{key}={value}" if value else key)
    return f"- [{date}] {entry.verb}" + (" " + " ".join(parts) if parts else "")


def _recent_activity_lines(vault: Path, limit: int) -> list:
    return [summarize_log_entry(entry) for entry in read_log(vault, limit=limit)]


def _contradiction_lines(pages: Sequence, limit: int, link_format: str) -> list:
    def _link(node: str) -> str:
        return f"[{node}]({node}.md)" if link_format == "markdown" else f"[[{node}]]"

    lines = []
    for page in sorted(pages, key=lambda p: p.updated, reverse=True):
        node = page.path[:-3]
        if page.lifecycle == "disputed":
            since = f" since {page.updated}" if page.updated else ""
            lines.append(f"- {_link(node)} — marked disputed{since}")
        for target in page.contradicts:
            lines.append(f"- {_link(node)} contradicts {_link(target)}")
        if len(lines) >= limit:
            break
    return lines[:limit]


def build_hot(
    vault: Path,
    *,
    takeaways: Optional[str] = None,
    link_format: str = "wikilink",
    max_words: Optional[int] = None,
    activity_limit: int = DEFAULT_RECENT_ACTIVITY,
    thread_limit: int = DEFAULT_ACTIVE_THREADS,
    contradiction_limit: int = DEFAULT_CONTRADICTIONS,
) -> HotResult:
    """Render the hot cache without writing it.

    Word-capped for real. When the render is over budget, sections are dropped
    in increasing order of value: activity lines first, then contradictions,
    then threads, and ``Key Takeaways`` is truncated last because it is the only
    part a model wrote on purpose.
    """
    vault = _require_vault(vault)
    cap = max_words or hot_max_words()
    hot = vault / "hot.md"
    existing = hot.read_text(encoding="utf-8") if hot.is_file() else ""

    if takeaways is None:
        carried = _section_body(existing, "Key Takeaways")
        takeaways = carried if carried else _TAKEAWAYS_PLACEHOLDER
    takeaways = takeaways.strip() or _TAKEAWAYS_PLACEHOLDER

    pages = scan_pages(vault)
    activity = _recent_activity_lines(vault, activity_limit)
    threads = [t for t in load_todos(vault) if t.status == "open"][:thread_limit]
    contradictions = _contradiction_lines(pages, contradiction_limit, link_format)

    def _render(act: list, con: list, thr: list, take: str) -> str:
        thread_lines = [
            f"- **{t.id}** {t.text if len(t.text) <= 110 else t.text[:109].rstrip() + '...'}"
            + (f" — {t.origin}" if t.origin else "")
            + f" (touched {t.touched})"
            for t in thr
        ]
        blocks = [
            "---\ntitle: Hot Cache\n"
            f"updated: {now_stamp(vault)}\n"
            f"pages: {len(pages)}\n"
            f"generated_by: {GENERATED_MARKER} hot\n---",
            "# Hot Cache",
            _HOT_NOTE,
            "## Recent Activity\n\n" + ("\n".join(act) if act else "*No logged operations yet.*"),
            "## Active Threads\n\n" + ("\n".join(thread_lines) if thread_lines else "*No open threads.*"),
            "## Key Takeaways\n\n" + take,
            "## Flagged Contradictions\n\n" + ("\n".join(con) if con else "*None flagged.*"),
        ]
        return "\n\n".join(blocks) + "\n"

    text = _render(activity, contradictions, threads, takeaways)
    trimmed = False
    while content_words(text) > cap:
        if len(activity) > 1:
            activity = activity[1:]
        elif contradictions:
            contradictions = contradictions[:-1]
        elif len(threads) > 1:
            threads = threads[:-1]
        else:
            words = takeaways.split()
            if len(words) <= 20:
                break  # the skeleton alone exceeds the cap; report it instead of looping
            takeaways = " ".join(words[: max(20, len(words) - 25)]) + " …"
        trimmed = True
        text = _render(activity, contradictions, threads, takeaways)

    return HotResult(
        words=content_words(text),
        trimmed=trimmed,
        activity=len(activity),
        threads=len(threads),
        contradictions=len(contradictions),
        text=text,
    )


def rebuild_hot(
    vault: Path, *, write: bool = True, lock: bool = True, force: bool = False, **kwargs
) -> HotResult:
    vault = _require_vault(vault)
    if write and not force and not (is_adopted(vault) or is_generated(vault / "hot.md")):
        raise MemoryError_(
            "unmigrated",
            "hot.md was not written by this tool; run `obsidian-wiki memory migrate` "
            "to review the change and take a backup first",
        )
    if lock:
        with memory_lock(vault):
            result = build_hot(vault, **kwargs)
            if write:
                atomic_write(vault / "hot.md", result.text)
            return result
    result = build_hot(vault, **kwargs)
    if write:
        atomic_write(vault / "hot.md", result.text)
    return result


# --------------------------------------------------------------------------
# recap — the block injected into a fresh session
# --------------------------------------------------------------------------


def build_recap(
    vault: Path,
    *,
    max_words: int = 400,
    min_confidence: float = 0.0,
    project: Optional[str] = None,
    scope: str = "",
) -> str:
    """Profile, open threads, and the newest activity as one injectable block.

    This is what a SessionStart or PreCompact hook feeds back into context: the
    left-hand column of the memory diagram, rendered once, instead of three
    separate file reads the model has to remember to make.
    """
    vault = _require_vault(vault)
    on = today(clock.wiki_timezone(vault))
    facts = [
        f for f in load_profile(vault, scope)
        if f.confidence >= min_confidence and not f.is_expired(on)
    ]
    todos = [t for t in load_todos(vault, scope) if t.status == "open"]
    entries = read_log(vault, limit=5)

    # Scoping keeps a session about project A from being handed project B's
    # threads. Facts describe the person, so they stay; threads and activity
    # are project-shaped. An unscoped thread is shown either way, and a filter
    # that matches nothing falls back rather than claiming there is no history.
    if project:
        needle = project.strip().casefold()
        scoped = [t for t in todos if needle in f"{t.origin} {t.text}".casefold()]
        todos = scoped if scoped else todos
        matched = [e for e in entries if needle in e.raw.casefold()]
        entries = matched or read_log(vault, limit=3)

    stale = [t for t in todos if t.is_stale()]

    lines = ["# Vault memory", ""]
    if facts:
        lines.append("## Who you are working with")
        lines.append("")
        lines += [f"- **{f.key}**: {f.value} ({f.confidence:.2f})" for f in facts]
        lines.append("")
    if todos:
        lines.append("## Open threads")
        lines.append("")
        for todo in todos:
            mark = " *(stale)*" if todo in stale else ""
            origin = f" — {todo.origin}" if todo.origin else ""
            lines.append(f"- **{todo.id}** {todo.text}{origin}{mark}")
        lines.append("")
    if entries:
        lines.append("## Recent vault activity")
        lines.append("")
        lines += [summarize_log_entry(entry) for entry in entries]
        lines.append("")
    if not facts and not todos and not entries:
        lines.append("*No vault memory recorded yet.*")
        lines.append("")
    elif not facts and not todos and all(e.verb == "INIT" for e in read_log(vault)):
        # A vault that was set up and never used: say what to do first rather
        # than replaying its creation line every session.
        lines.append("## Getting started")
        lines.append("")
        lines.append(
            "- The vault is empty. Save something worth keeping with `/wiki-capture`, "
            "add documents with `/wiki-ingest <path>` (or drop drafts in `_raw/`), "
            "and ask with `/wiki-query`."
        )
        lines.append("")

    text = "\n".join(lines).rstrip() + "\n"
    if word_count(text) > max_words:
        words = text.split()
        text = " ".join(words[:max_words]) + " …\n"
    return text


# --------------------------------------------------------------------------
# migration — adopting a vault that predates this module
# --------------------------------------------------------------------------


MIGRATION_BACKUP_DIR = "_archives"
_BULLET_RE = re.compile(r"^\s*(?:[-*+]|\d+\.)\s+(.*)$")


def extract_threads(hot_text: str) -> list:
    """Open threads written as prose in a hand-maintained ``Active Threads``.

    Migration would otherwise drop this outright: the generated section is
    built from the todo table, which is empty on a vault that never had one.
    Converting the bullets into real todos preserves the content *and* seeds
    the index that keeps it alive.
    """
    body = _section_body(hot_text, "Active Threads", prefix=True)
    threads = []
    for line in body.splitlines():
        match = _BULLET_RE.match(line)
        if not match:
            continue
        text = match.group(1).strip()
        text = re.sub(r"^\*\*(.+?)\*\*\s*[-—:]*\s*", r"\1: ", text).strip()
        text = _MD_DECORATION_RE.sub("", text).strip()
        if len(text) > 160:
            text = text[:159].rstrip() + "..."
        if text:
            threads.append(text)
    return threads


def migration_status(vault: Path) -> dict:
    """What adopting this vault would change, without changing anything.

    Existing vaults have hand-curated memory files. Regenerating them silently
    would reorder a document someone built on purpose and drop narrative the
    generator cannot reconstruct, so adoption is explicit and backed up.
    """
    vault = _require_vault(vault)
    index, hot = vault / "index.md", vault / "hot.md"
    adopted = is_adopted(vault)
    index_generated, hot_generated = adopted or is_generated(index), adopted or is_generated(hot)

    preview = rebuild_index(vault, write=False, lock=False, force=True)
    old_index = index.read_text(encoding="utf-8") if index.is_file() else ""
    custom = [
        heading for heading in _HEADING_RE.findall(preview.text)
        if heading in _HEADING_RE.findall(old_index)
    ]

    hot_text = hot.read_text(encoding="utf-8") if hot.is_file() else ""
    carried = _section_body(hot_text, "Key Takeaways")
    dropped = [
        name for name in ("Active Threads", "Recent Activity")
        if _section_body(hot_text, name, prefix=True) and not hot_generated
    ]
    recoverable = extract_threads(hot_text) if not hot_generated else []

    return {
        "vault": str(vault),
        "migrated": index_generated and hot_generated,
        "index": {
            "generated": index_generated,
            "lines_before": len(old_index.splitlines()),
            "lines_after": len(preview.text.splitlines()),
            "entries_added": len(preview.added),
            "sections_preserved": custom,
        },
        "hot": {
            "generated": hot_generated,
            "takeaways_carried": bool(carried.strip()) and carried.strip() != _TAKEAWAYS_PLACEHOLDER,
            "sections_regenerated": dropped,
            "threads_to_seed": recoverable,
        },
    }


def migrate(vault: Path, *, link_format: str = "wikilink", backup: bool = True) -> dict:
    """Adopt the memory files, taking a timestamped backup first.

    The backup is the whole point: everything the generator cannot reconstruct
    — a hand-written Active Threads narrative, a custom section's position — is
    recoverable from it.
    """
    vault = _require_vault(vault)
    status = migration_status(vault)
    hot = vault / "hot.md"
    stamp = clock.compact(clock.wiki_timezone(vault))
    backup_dir = vault / MIGRATION_BACKUP_DIR / f"pre-memory-migration-{stamp}"
    saved = []

    with memory_lock(vault):
        if backup:
            backup_dir.mkdir(parents=True, exist_ok=True)
            for name in ("index.md", "hot.md"):
                source = vault / name
                if source.is_file():
                    atomic_write(backup_dir / name, source.read_text(encoding="utf-8"))
                    saved.append(f"{MIGRATION_BACKUP_DIR}/{backup_dir.name}/{name}")
        # Seed the todo table from the old prose before regenerating, so the
        # rebuilt Active Threads section shows the same threads rather than
        # "none". Only when the table is empty — never overwrite real todos.
        seeded = []
        if not load_todos(vault):
            for text in extract_threads(
                (vault / "hot.md").read_text(encoding="utf-8") if hot.is_file() else ""
            ):
                seeded.append(add_todo(vault, text, origin="migrated from hot.md", lock=False).id)

        index = rebuild_index(vault, link_format=link_format, lock=False, force=True)
        hot_result = rebuild_hot(vault, lock=False, force=True, link_format=link_format)
        mark_adopted(vault)

    return {
        "vault": str(vault),
        "backup": saved,
        "backup_dir": str(backup_dir) if saved else "",
        "index": {"total": index.total, "added": list(index.added), "removed": list(index.removed)},
        "hot": {"words": hot_result.words, "trimmed": hot_result.trimmed},
        "threads_seeded": seeded,
        "was_migrated": status["migrated"],
    }


TIMEZONE_BACKUP_PREFIX = "pre-timezone-migration-"
_TIMEZONE_SKIP_DIRS = frozenset({MIGRATION_BACKUP_DIR, "_archived"})
_BARE_DATE_KEY_RE = re.compile(
    r"^(?:created|updated|lifecycle_changed|reviewed_at|adopted):[ \t]*\"?\d{4}-\d{2}-\d{2}\"?[ \t]*$",
    re.MULTILINE,
)


def _timezone_targets(vault: Path, skip: Sequence[str]) -> list:
    """Every file the timezone migration may rewrite: pages, the memory files,
    the ``_meta/`` tables, drafts in ``_raw/`` and ``_staging/``, and the
    adoption record. Never ``_archives/`` (the backups), ``_raw/_archived/``
    (source snapshots), hidden dirs, instruction files, ``.okignore`` matches,
    or anything that is not Markdown — JSON ledgers keep their own stamps."""
    import fnmatch

    from obsidian_wiki.vault import _walk_md, is_instruction_file, okignore_patterns, okignored, skipped_dir

    patterns = okignore_patterns(vault)
    targets = []
    for path in sorted(_walk_md(vault)):
        rel = path.relative_to(vault)
        if skipped_dir(rel, _TIMEZONE_SKIP_DIRS) or is_instruction_file(rel) or okignored(rel, patterns):
            continue
        targets.append(path)
    adopted = vault / ADOPTED_REL
    if adopted.is_file():
        targets.append(adopted)
    return [
        path for path in targets
        if not any(fnmatch.fnmatch(path.relative_to(vault).as_posix(), pattern) for pattern in skip)
    ]


def _timezone_target_zone(vault: Path, zone: str) -> str:
    name = (zone or "").strip().strip("\"'")
    if not name:
        name = clock.timezone_name(vault)
    if not name:
        raise MemoryError_(
            "no_timezone",
            "no timezone to convert to: pass one (`memory timezone America/Detroit`) "
            "or set `timezone:` in index.md first",
        )
    clock.parse_timezone(name)  # TimezoneError on an unknown name, before any file is read
    return name


def timezone_migration_plan(vault: Path, zone: str = "", *, skip: Sequence[str] = ()) -> dict:
    """What ``migrate_timezone`` would change, without writing anything.

    Every zoned stamp in every target file is re-rendered in the vault's zone
    (or *zone*, which ``--apply`` would also record in ``index.md``). Bare dates
    and naive stamps are counted and left alone.
    """
    vault = _require_vault(vault)
    name = _timezone_target_zone(vault, zone)
    tz = clock.parse_timezone(name)
    files = []
    stamps = naive_total = bare_total = 0
    for path in _timezone_targets(vault, skip):
        text = path.read_text(encoding="utf-8", errors="replace")
        _new, changes, naive = clock.convert_text(text, tz)
        naive_total += naive
        bare_total += len(_BARE_DATE_KEY_RE.findall(split_frontmatter(text)[0]))
        if changes:
            rel = path.relative_to(vault).as_posix()
            files.append({
                "path": rel,
                "stamps": [{"line": c.line, "old": c.old, "new": c.new} for c in changes],
            })
            stamps += len(changes)
    return {
        "vault": str(vault),
        "timezone": name,
        "key_current": clock.timezone_name(vault),
        "files": files,
        "file_count": len(files),
        "stamp_count": stamps,
        "naive_skipped": naive_total,
        "bare_dates": bare_total,
        "skip": list(skip),
    }


def migrate_timezone(vault: Path, zone: str = "", *, skip: Sequence[str] = (), backup: bool = True) -> dict:
    """Rewrite every zoned stamp in the vault in its zone, with a backup first.

    Under the memory lock: the original of every file that will change is
    copied to ``_archives/pre-timezone-migration-<stamp>/<same path>``, the
    key is recorded in ``index.md`` when *zone* was given, the files are
    rewritten atomically, and one ``TIMEZONE`` line is appended to the log —
    in the new zone, after the log itself was converted. Re-running is a
    no-op: a stamp already in the zone renders identically.
    """
    vault = _require_vault(vault)
    name = _timezone_target_zone(vault, zone)
    tz = clock.parse_timezone(name)
    backup_dir = vault / MIGRATION_BACKUP_DIR / f"{TIMEZONE_BACKUP_PREFIX}{clock.compact(tz)}"
    index = vault / "index.md"
    files = []
    stamps = naive_total = bare_total = 0
    saved: list = []

    def _save(path: Path, text: str) -> None:
        if not backup:
            return
        target = backup_dir / path.relative_to(vault)
        target.parent.mkdir(parents=True, exist_ok=True)
        atomic_write(target, text)
        saved.append(path.relative_to(vault).as_posix())

    with memory_lock(vault):
        set_key = bool(zone.strip()) and clock.timezone_name(vault) != name
        if set_key and index.is_file():
            _save(index, index.read_text(encoding="utf-8"))
            clock.set_timezone(vault, name)
        for path in _timezone_targets(vault, skip):
            text = path.read_text(encoding="utf-8", errors="replace")
            new_text, changes, naive = clock.convert_text(text, tz)
            naive_total += naive
            bare_total += len(_BARE_DATE_KEY_RE.findall(split_frontmatter(text)[0]))
            if not changes:
                continue
            rel = path.relative_to(vault).as_posix()
            if rel not in saved:
                _save(path, text)
            atomic_write(path, new_text)
            files.append({
                "path": rel,
                "stamps": [{"line": c.line, "old": c.old, "new": c.new} for c in changes],
            })
            stamps += len(changes)
        line = append_log(
            vault,
            "TIMEZONE",
            {
                "zone": name,
                "files": len(files),
                "stamps": stamps,
                "backup": f"{MIGRATION_BACKUP_DIR}/{backup_dir.name}" if saved else "",
            },
            lock=False,
        )

    return {
        "vault": str(vault),
        "timezone": name,
        "key_set": set_key,
        "files": files,
        "file_count": len(files),
        "stamp_count": stamps,
        "naive_skipped": naive_total,
        "bare_dates": bare_total,
        "skip": list(skip),
        "backup": saved,
        "backup_dir": str(backup_dir) if saved else "",
        "log_line": line,
    }


def memory_status(vault: Path) -> dict:
    """Machine-readable health of the memory surface."""
    vault = _require_vault(vault)
    hot = vault / "hot.md"
    hot_text = hot.read_text(encoding="utf-8") if hot.is_file() else ""
    hot_values = parse_frontmatter(split_frontmatter(hot_text)[0]) if hot_text else {}
    index_drift = rebuild_index(vault, write=False, lock=False, force=True)
    todos = load_todos(vault)
    return {
        "vault": str(vault),
        "timezone": clock.timezone_name(vault) or "UTC",
        "pages": index_drift.total,
        "index_drift": {
            "added": list(index_drift.added),
            "removed": list(index_drift.removed),
            "stale": index_drift.changed,
        },
        "log_entries": len(read_log(vault)),
        "hot": {
            "exists": hot.is_file(),
            "updated": str(hot_values.get("updated") or ""),
            "words": content_words(hot_text),
            "max_words": hot_max_words(),
            "over_budget": content_words(hot_text) > hot_max_words(),
            "generated": bool(hot_values.get("generated_by")),
        },
        "migrated": is_adopted(vault) or (is_generated(vault / "index.md") and is_generated(hot)),
        "profile_facts": len(load_profile(vault)),
        "todos": {
            "open": sum(1 for t in todos if t.status == "open"),
            "stale": sum(1 for t in todos if t.is_stale()),
            "closed": sum(1 for t in todos if t.status != "open"),
        },
    }