obsidian-wiki/obsidian_wiki/vault.py

"""Shared vault primitives: frontmatter splitting, skip dirs, and the page walker.

Every module that reads pages out of a vault goes through these, so a vault
parses and walks the same way no matter which command touches it (#238).
Modules layer their own extra skip dirs / reserved files on top.
"""

from __future__ import annotations

import fnmatch
import os
import re
from pathlib import Path
from typing import Iterable

#: Staging, archive, and tool-owned directories that never hold knowledge pages.
#: Dot-prefixed paths (`.venv`, `.git`, `.obsidian`) are skipped wholesale by
#: `iter_md`; this list adds the non-hidden equivalents.
SKIP_DIRS = frozenset({
    "_raw", "_archived", "_staging", "_archives",
    "_sources", "_meta", "_bot_exports", "_inbox",
    ".obsidian", ".git", "venv", "node_modules", "__pycache__",
})

#: CRLF-tolerant so a vault edited on Windows parses the same as one from Unix.
FRONTMATTER_RE = re.compile(r"^---\r?\n(.*?)\r?\n---(?:\r?\n|$)", re.DOTALL)
# A bare `>`, `>-`, `>+`, `|`, `|-`, `|+` (optionally followed by an indent
# indicator digit) marks a YAML block scalar — the real value lives on the
# following indented lines, not on this line.
BLOCK_SCALAR_RE = re.compile(r"^[>|][+-]?\d*$")


def split_frontmatter(text: str) -> tuple[str, str]:
    """(frontmatter, body); frontmatter is "" when the page has none."""
    match = FRONTMATTER_RE.match(text)
    return (match.group(1), text[match.end():]) if match else ("", text)


def okignore_patterns(vault: Path) -> list[str]:
    """Patterns from the vault-root `.okignore` (gitignore syntax, subset)."""
    try:
        lines = (vault / ".okignore").read_text(encoding="utf-8").splitlines()
    except OSError:
        return []
    # ponytail: no `!` negation or `**`-specific semantics; add if a vault needs them.
    return [
        line.strip().rstrip("/") for line in lines
        if line.strip() and not line.strip().startswith(("#", "!"))
    ]


def okignored(rel: Path, patterns: list[str]) -> bool:
    """True if vault-relative `rel` is excluded by any `.okignore` pattern.

    A pattern without a slash matches any path component (`_inbox`, `*.draft.md`);
    one with a slash is anchored to the vault root and matches that path or
    anything beneath it (`/drafts`, `notes/old`).
    """
    parts = rel.parts
    prefixes = ["/".join(parts[: i + 1]) for i in range(len(parts))]
    for pattern in patterns:
        if "/" in pattern:
            anchored = pattern.lstrip("/")
            if any(fnmatch.fnmatchcase(prefix, anchored) for prefix in prefixes):
                return True
        elif any(fnmatch.fnmatchcase(part, pattern) for part in parts):
            return True
    return False


#: Instructions for coding agents, not knowledge, wherever they sit: a vault
#: built from per-project sub-wikis carries one per project. A `README.md` is
#: only excluded at the vault root, where it describes the vault itself.
AGENT_INSTRUCTION_FILES = frozenset({"AGENTS.md", "CLAUDE.md", "GEMINI.md"})


def is_instruction_file(rel: Path) -> bool:
    """True if vault-relative `rel` is an agent-instruction file, not a page."""
    return rel.name in AGENT_INSTRUCTION_FILES or (len(rel.parts) == 1 and rel.name == "README.md")


def skipped_dir(rel: Path, skip_dirs: Iterable[str] = SKIP_DIRS) -> bool:
    """True if any component of vault-relative `rel` is hidden or in `skip_dirs`."""
    return any(part in skip_dirs or part.startswith(".") for part in rel.parts)


def _walk_md(vault: Path) -> Iterable[Path]:
    """Every `.md` under `vault`, descending into symlinked directories.

    Obsidian follows a symlinked folder as part of the vault, but `Path.rglob`
    does not descend into one before Python 3.13, so a vault that mounts its
    project folders by symlink looked empty and `memory index` dropped their
    entries. A directory whose real path was already visited is pruned, which
    breaks symlink loops.
    """
    seen: set[str] = set()
    for root, dirs, files in os.walk(vault, followlinks=True):
        real = os.path.realpath(root)
        if real in seen:
            dirs[:] = []
            continue
        seen.add(real)
        for name in files:
            if name.endswith(".md"):
                yield Path(root) / name


def iter_instruction_files(vault: Path, skip_dirs: Iterable[str] = SKIP_DIRS) -> list[Path]:
    """The agent-instruction files `iter_md` leaves out. Not pages, but real
    files a `[[CLAUDE]]` link opens, so link checks must still find them."""
    return sorted(
        path for path in _walk_md(vault)
        if is_instruction_file(path.relative_to(vault))
        and not skipped_dir(path.relative_to(vault), skip_dirs)
    )


def iter_md_under(vault: Path, root: Path) -> list[Path]:
    """Every `.md` under `root` (a directory in `vault`), sorted, minus `.okignore`.

    For the tool-owned subtrees `iter_md` skips, such as `_staging/` and `_raw/`.
    `rglob` rather than `_walk_md`: a symlinked folder here isn't part of the vault
    the way a mounted project folder is, and it could lead out of the vault.
    """
    patterns = okignore_patterns(vault)
    return sorted(path for path in root.rglob("*.md") if not okignored(path.relative_to(vault), patterns))


def iter_md(vault: Path, skip_dirs: Iterable[str] = SKIP_DIRS) -> list[Path]:
    """Every `.md` page under `vault`, sorted, minus hidden/skipped dirs,
    agent-instruction files, and `.okignore`."""
    patterns = okignore_patterns(vault)
    return sorted(
        path for path in _walk_md(vault)
        if not skipped_dir(path.relative_to(vault), skip_dirs)
        and not is_instruction_file(path.relative_to(vault))
        and not okignored(path.relative_to(vault), patterns)
    )