Files
obsidian-wiki/tests/test_cache.py
T
Codex 82890606aa feat(wiki): sync upstream v2026.10.4 core maintenance
Selectively port source hashing, locked manifest/memory writes, path-aware
link lint and graph analysis, and revision-checked staging from upstream
4f457e154653756b04802d5aeb25c20f42b4770c. Record source hashes and exclusions.

Preserve local archive/catalog citations, curated manifest metadata, config
location, file-index and MCP behavior. Wire maintenance guidance into four
skills without migrating vault contents or changing running services.

Validation: 609 tests passed, including existing MCP/file-asset regressions
and imported upstream maintenance tests. Read-only vault status left shared
metadata hashes unchanged. Wiki MCP remains inactive; orders MCP is active.
2026-10-31 12:39:48 +00:00

537 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Tests for the content-hash cache module."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
import pytest
from obsidian_wiki.cache import (
ManifestLockTimeout,
check_sources,
compute_hash,
hash_file,
manifest_lock,
sha256_file,
sha256_dir,
update_source,
_load_manifest,
_lock_path,
_manifest_path,
_write_manifest,
)
@pytest.fixture
def vault(tmp_path):
v = tmp_path / "vault"
v.mkdir()
return v
@pytest.fixture
def src_file(tmp_path):
f = tmp_path / "doc.md"
f.write_text("# Hello\nSome content.", encoding="utf-8")
return f
@pytest.fixture
def src_dir(tmp_path):
d = tmp_path / "repo"
d.mkdir()
(d / "a.py").write_text("x = 1")
(d / "b.py").write_text("y = 2")
return d
# ---------------------------------------------------------------------------
# Hash functions
# ---------------------------------------------------------------------------
class TestHashing:
def test_sha256_file_deterministic(self, src_file):
assert sha256_file(src_file) == sha256_file(src_file)
def test_sha256_file_changes_on_edit(self, src_file):
h1 = sha256_file(src_file)
src_file.write_text("# Different content")
h2 = sha256_file(src_file)
assert h1 != h2
def test_sha256_dir_deterministic(self, src_dir):
assert sha256_dir(src_dir) == sha256_dir(src_dir)
def test_sha256_dir_changes_on_edit(self, src_dir):
h1 = sha256_dir(src_dir)
(src_dir / "a.py").write_text("x = 999")
h2 = sha256_dir(src_dir)
assert h1 != h2
def test_compute_hash_dispatches(self, src_file, src_dir):
assert len(compute_hash(src_file)) == 64 # hex SHA-256
assert len(compute_hash(src_dir)) == 64
def test_sha256_dir_independent_of_path_separator(self, src_dir):
"""Hash must match hashlib computed with POSIX separators regardless of platform (#178)."""
import hashlib
(src_dir / "sub").mkdir()
(src_dir / "sub" / "c.py").write_text("z = 3")
h = hashlib.sha256()
for fp in sorted(src_dir.rglob("*"), key=lambda p: p.relative_to(src_dir).as_posix()):
if fp.is_file():
h.update(fp.relative_to(src_dir).as_posix().encode())
h.update(sha256_file(fp).encode())
assert sha256_dir(src_dir) == h.hexdigest()
def test_hash_file_alias(self, src_file):
assert hash_file(src_file) == sha256_file(src_file)
# ---------------------------------------------------------------------------
# check_sources
# ---------------------------------------------------------------------------
class TestCheckSources:
def test_new_source(self, vault, src_file):
result = check_sources(vault, [src_file])
assert str(src_file) in result["new"]
assert result["modified"] == []
assert result["unchanged"] == []
def test_unchanged_after_update(self, vault, src_file):
update_source(vault, src_file)
result = check_sources(vault, [src_file])
assert str(src_file) in result["unchanged"]
assert result["new"] == []
assert result["modified"] == []
def test_modified_after_content_change(self, vault, src_file):
update_source(vault, src_file)
src_file.write_text("# Changed content")
result = check_sources(vault, [src_file])
assert str(src_file) in result["modified"]
def test_missing_path(self, vault, tmp_path):
ghost = tmp_path / "ghost.md"
result = check_sources(vault, [ghost])
assert str(ghost) in result["missing"]
def test_empty_source_list(self, vault):
result = check_sources(vault, [])
assert result == {
"new": [], "modified": [], "unchanged": [], "missing": [], "unavailable": []
}
def test_multiple_sources(self, vault, src_file, src_dir):
update_source(vault, src_file)
result = check_sources(vault, [src_file, src_dir])
assert str(src_file) in result["unchanged"]
assert str(src_dir) in result["new"]
def test_timestamp_irrelevant(self, vault, src_file):
# Touch the file (change mtime) without changing content — still unchanged
update_source(vault, src_file)
src_file.touch()
result = check_sources(vault, [src_file])
assert str(src_file) in result["unchanged"]
def _write_relative_manifest(self, vault, rel_key, content_hash):
"""Write a manifest whose source key is stored vault-relative."""
_manifest_path(vault).write_text(
json.dumps(
{"sources": {rel_key: {"content_hash": content_hash, "last_ingested": "2026-07-14"}}}
),
encoding="utf-8",
)
def test_relative_manifest_key_unchanged_for_abs_path(self, vault):
# Manifest stores a vault-relative key; caller passes the absolute path.
src = vault / "_raw" / "articles" / "foo.md"
src.parent.mkdir(parents=True)
src.write_text("body", encoding="utf-8")
self._write_relative_manifest(vault, "_raw/articles/foo.md", sha256_file(src))
result = check_sources(vault, [src])
assert str(src) in result["unchanged"]
assert result["new"] == []
assert result["missing"] == []
def test_relative_manifest_key_not_falsely_missing(self, vault):
# A relative key whose file exists under the vault must not be flagged missing,
# even when CWD != vault root.
src = vault / "_raw" / "articles" / "foo.md"
src.parent.mkdir(parents=True)
src.write_text("body", encoding="utf-8")
self._write_relative_manifest(vault, "_raw/articles/foo.md", sha256_file(src))
result = check_sources(vault, [])
assert "_raw/articles/foo.md" not in result["missing"]
def test_relative_manifest_key_modified(self, vault):
src = vault / "_raw" / "articles" / "foo.md"
src.parent.mkdir(parents=True)
src.write_text("body", encoding="utf-8")
self._write_relative_manifest(vault, "_raw/articles/foo.md", "stale-hash")
result = check_sources(vault, [src])
assert str(src) in result["modified"]
def test_relative_manifest_key_genuinely_missing(self, vault):
# A relative key whose top-level directory really is part of this vault
# is a vault-local loss when the file is gone.
(vault / "_raw" / "articles").mkdir(parents=True)
self._write_relative_manifest(vault, "_raw/articles/gone.md", "abc")
result = check_sources(vault, [])
assert "_raw/articles/gone.md" in result["missing"]
def test_relative_key_under_an_unknown_top_dir_is_unavailable(self, vault):
# A relative key whose first segment is not a vault entry is relative to
# some other root (a legacy ingest root, or an out-of-vault namespace), so
# it is unavailable here rather than a missing vault source.
self._write_relative_manifest(vault, "-Users-x/abc.jsonl", "abc")
result = check_sources(vault, [])
assert result["missing"] == []
assert "-Users-x/abc.jsonl" in result["unavailable"]
def test_check_sources_resolves_each_path_once(self, vault, tmp_path, monkeypatch):
"""Matching resolves each entry and each query once (an index of forms), not once
per (entry, query) pair: with relative keys the string fast path never fires, and a
3,000-entry manifest made every queried file cost a second of ``realpath`` walks."""
src = tmp_path / "src"
src.mkdir()
files = []
for i in range(300):
f = src / f"f{i:03d}.md"
f.write_text(f"doc {i}", encoding="utf-8")
files.append(f)
sources = {}
for i, f in enumerate(files):
# "../src/f000.md" — the relative form the CLI writes for out-of-vault sources
sources[os.path.relpath(f, vault)] = {"content_hash": sha256_file(f) if i % 3 else "0" * 64}
for i in range(2000): # unrelated entries, like a real manifest
sources[f"../elsewhere/e{i:04d}.md"] = {"content_hash": "1" * 64}
_write_manifest(vault, {"version": 1, "sources": sources})
calls = {"n": 0}
real_resolve = Path.resolve
def counting_resolve(self, *args, **kwargs):
calls["n"] += 1
return real_resolve(self, *args, **kwargs)
monkeypatch.setattr(Path, "resolve", counting_resolve)
result = check_sources(vault, files)
assert len(result["unchanged"]) == 200
assert len(result["modified"]) == 100
assert result["new"] == []
assert len(result["missing"]) + len(result["unavailable"]) == 2000
# one resolve per entry and per query (plus a handful inside helpers), never entries × queries
assert calls["n"] <= 3 * (2300 + 300), calls["n"]
def test_check_sources_first_entry_wins_for_duplicate_forms(self, vault, tmp_path):
"""Two entries naming the same file in different forms: the earlier one is the match,
the later one is not reported missing (it is a duplicate of a scanned path)."""
f = tmp_path / "dup.md"
f.write_text("same", encoding="utf-8")
sources = {
os.path.relpath(f, vault): {"content_hash": "0" * 64}, # earlier: stale hash → modified
str(f): {"content_hash": sha256_file(f)}, # later: same file, absolute form
}
_write_manifest(vault, {"version": 1, "sources": sources})
result = check_sources(vault, [f])
assert result["modified"] == [str(f)] and result["unchanged"] == []
assert result["missing"] == [] and result["unavailable"] == []
# ---------------------------------------------------------------------------
# update_source / manifest
# ---------------------------------------------------------------------------
class TestUpdateSource:
def test_writes_manifest(self, vault, src_file):
update_source(vault, src_file)
assert _manifest_path(vault).exists()
def test_records_correct_hash(self, vault, src_file):
h = update_source(vault, src_file)
assert h == sha256_file(src_file)
sources = _load_manifest(vault)
assert sources[str(src_file)]["content_hash"] == h
def test_records_pages_produced(self, vault, src_file):
update_source(vault, src_file, pages_produced=["concepts/foo.md", "entities/bar.md"])
sources = _load_manifest(vault)
assert sources[str(src_file)]["pages_produced"] == ["concepts/foo.md", "entities/bar.md"]
def test_records_last_ingested_timestamp(self, vault, src_file):
update_source(vault, src_file)
sources = _load_manifest(vault)
assert "last_ingested" in sources[str(src_file)]
def test_update_overwrites_old_hash(self, vault, src_file):
update_source(vault, src_file)
src_file.write_text("new content")
h2 = update_source(vault, src_file)
sources = _load_manifest(vault)
assert sources[str(src_file)]["content_hash"] == h2
def test_preserves_other_manifest_entries(self, vault, src_file, src_dir):
update_source(vault, src_file)
update_source(vault, src_dir)
sources = _load_manifest(vault)
assert str(src_file) in sources
assert str(src_dir) in sources
def test_update_source_live_draft_beats_same_named_archive(self, vault):
staging = vault / "_raw" / "notes.md"
archived = vault / "_raw" / "_archived" / "notes.md"
archived.parent.mkdir(parents=True, exist_ok=True)
staging.parent.mkdir(parents=True, exist_ok=True)
staging.write_text("staging-bytes\n", encoding="utf-8")
archived.write_text("archived-bytes\n", encoding="utf-8")
h = update_source(vault, staging)
assert h == compute_hash(staging)
sources = _load_manifest(vault)
assert "_raw/notes.md" in sources
assert "_raw/_archived/notes.md" not in sources
def test_update_source_hashes_archive_when_staging_gone(self, vault):
staging = vault / "_raw" / "notes.md"
archived = vault / "_raw" / "_archived" / "notes.md"
archived.parent.mkdir(parents=True, exist_ok=True)
staging.parent.mkdir(parents=True, exist_ok=True)
staging.write_text("staging-bytes\n", encoding="utf-8")
archived.write_text("archived-bytes\n", encoding="utf-8")
update_source(vault, staging)
staging.unlink()
h = update_source(vault, staging)
assert h == compute_hash(archived)
assert "_raw/_archived/notes.md" in _load_manifest(vault)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
class TestCacheCLI:
def _run(self, *args, cwd=None):
return subprocess.run(
[sys.executable, "-m", "obsidian_wiki.cli", *args],
capture_output=True, text=True, cwd=cwd,
)
def test_cache_hash_file(self, src_file):
proc = self._run("cache-hash", str(src_file))
assert proc.returncode == 0
data = json.loads(proc.stdout)
assert data["sha256"] == sha256_file(src_file)
def test_cache_hash_missing_exits_nonzero(self, tmp_path):
proc = self._run("cache-hash", str(tmp_path / "nope.md"))
assert proc.returncode != 0
def test_cache_check_new(self, vault, src_file):
proc = self._run("cache-check", str(vault), str(src_file))
assert proc.returncode == 0
data = json.loads(proc.stdout)
assert str(src_file) in data["new"]
def test_cache_check_pretty(self, vault, src_file):
proc = self._run("cache-check", "--pretty", str(vault), str(src_file))
assert proc.returncode == 0
assert "\n " in proc.stdout
def test_cache_update_then_check_unchanged(self, vault, src_file):
self._run("cache-update", str(vault), str(src_file))
proc = self._run("cache-check", str(vault), str(src_file))
data = json.loads(proc.stdout)
assert str(src_file) in data["unchanged"]
def test_cache_update_with_pages(self, vault, src_file):
proc = self._run("cache-update", str(vault), str(src_file),
"--pages", "concepts/foo.md", "entities/bar.md")
assert proc.returncode == 0
sources = _load_manifest(vault)
assert sources[str(src_file)]["pages_produced"] == ["concepts/foo.md", "entities/bar.md"]
def test_cache_update_non_portable_source_warns_on_stderr(self, vault, src_file):
# src_file lives outside the vault and $HOME, so the fallback absolute
# key is stored — but the CLI must say so rather than stay silent.
proc = self._run("cache-update", str(vault), str(src_file))
assert proc.returncode == 0
assert "no portable key" in proc.stderr
def test_cache_update_explicit_key_warns_nothing(self, vault, src_file):
proc = self._run("cache-update", str(vault), str(src_file), "--key", "repo:o/n")
assert proc.returncode == 0
assert "no portable key" not in proc.stderr
assert "repo:o/n" in _load_manifest(vault)
def test_cache_update_cli_uses_archive_if_staging_gone(self, vault):
staging = vault / "_raw" / "notes.md"
archived = vault / "_raw" / "_archived" / "notes.md"
archived.parent.mkdir(parents=True, exist_ok=True)
staging.parent.mkdir(parents=True, exist_ok=True)
staging.write_text("staging-bytes\n", encoding="utf-8")
archived.write_text("archived-bytes\n", encoding="utf-8")
staging.unlink()
proc = self._run("cache-update", str(vault), str(staging))
assert proc.returncode == 0
data = json.loads(proc.stdout)
assert data["key"] == "_raw/_archived/notes.md"
assert data["content_hash"] == compute_hash(archived)
class TestCacheCLIVaultRelativeSources:
"""Relative source arguments must also resolve against the vault root.
The manifest stores vault-relative keys, and ``cache.resolve_key`` reads
them that way — so ``cache-update <vault> Raw/x.pdf`` has to work from any
CWD, not just when the shell happens to sit at the vault root.
"""
@pytest.fixture
def vault(self, tmp_path):
v = tmp_path / "vault"
(v / "Raw" / "database").mkdir(parents=True)
(v / "Raw" / "database" / "x.pdf").write_bytes(b"pdf-bytes")
return v
def _run(self, *args, cwd=None):
return subprocess.run(
[sys.executable, "-m", "obsidian_wiki.cli", *args],
capture_output=True, text=True, cwd=cwd,
)
def test_cache_update_vault_relative_from_other_cwd(self, vault, tmp_path):
elsewhere = tmp_path / "elsewhere"
elsewhere.mkdir()
proc = self._run("cache-update", str(vault), "Raw/database/x.pdf",
cwd=elsewhere)
assert proc.returncode == 0, proc.stderr
data = json.loads(proc.stdout)
assert data["key"] == "Raw/database/x.pdf"
assert "Raw/database/x.pdf" in json.dumps(_load_manifest(vault))
def test_cache_check_vault_relative_from_other_cwd(self, vault, tmp_path):
update_source(vault, vault / "Raw" / "database" / "x.pdf")
elsewhere = tmp_path / "elsewhere"
elsewhere.mkdir()
proc = self._run("cache-check", str(vault), "Raw/database/x.pdf",
cwd=elsewhere)
assert proc.returncode == 0, proc.stderr
data = json.loads(proc.stdout)
assert str(vault / "Raw" / "database" / "x.pdf") in data["unchanged"]
assert data["missing"] == []
def test_cache_update_cwd_relative_still_wins(self, vault, tmp_path):
# Backward compat: an out-of-vault relative path resolves against the
# CWD as before, even when the vault contains a same-named file.
workdir = tmp_path / "workdir"
(workdir / "Raw" / "database").mkdir(parents=True)
(workdir / "Raw" / "database" / "x.pdf").write_bytes(b"different-bytes")
proc = self._run("cache-update", str(vault), "Raw/database/x.pdf",
cwd=workdir)
assert proc.returncode == 0, proc.stderr
data = json.loads(proc.stdout)
assert data["path"] == str(workdir / "Raw" / "database" / "x.pdf")
# The typed string is byte-for-byte a manifest key, yet the vault file
# it names is not the one hashed — say so rather than choosing silently.
assert (f"note: Raw/database/x.pdf matched both "
f"{workdir / 'Raw' / 'database' / 'x.pdf'} and "
f"{vault / 'Raw' / 'database' / 'x.pdf'}") in proc.stderr
def test_cache_update_missing_source_clean_error(self, vault, tmp_path):
elsewhere = tmp_path / "elsewhere"
elsewhere.mkdir()
proc = self._run("cache-update", str(vault), "Raw/absent.md", cwd=elsewhere)
assert proc.returncode == 1
assert "Traceback" not in proc.stderr
assert str(elsewhere / "Raw" / "absent.md") in proc.stderr
assert str(vault / "Raw" / "absent.md") in proc.stderr
def test_cache_update_absolute_missing_clean_error(self, vault):
proc = self._run("cache-update", str(vault), str(vault / "Raw" / "gone.pdf"))
assert proc.returncode == 1
assert "Traceback" not in proc.stderr
assert str(vault / "Raw" / "gone.pdf") in proc.stderr
class TestManifestLock:
"""Concurrent writers must serialize instead of clobbering the manifest."""
def test_lock_is_released_after_use(self, vault):
with manifest_lock(vault):
assert _lock_path(vault).exists()
assert not _lock_path(vault).exists()
def test_second_holder_times_out(self, vault):
with manifest_lock(vault):
with pytest.raises(ManifestLockTimeout):
with manifest_lock(vault, timeout=0.3):
pass
def test_stale_lock_is_stolen(self, vault):
lock = _lock_path(vault)
lock.write_text("999999")
old = time.time() - 120
os.utime(lock, (old, old))
with manifest_lock(vault, timeout=0.5, stale_after=60.0):
pass
assert not lock.exists()
def test_lock_released_when_body_raises(self, vault):
with pytest.raises(ValueError):
with manifest_lock(vault):
raise ValueError("boom")
assert not _lock_path(vault).exists()
def test_update_source_leaves_no_lock_behind(self, vault, src_file):
update_source(vault, src_file)
assert not _lock_path(vault).exists()
def test_concurrent_updates_both_survive(self, vault, tmp_path):
"""The race the lock exists for: two processes, neither entry lost."""
a = tmp_path / "a.md"
a.write_text("alpha", encoding="utf-8")
b = tmp_path / "b.md"
b.write_text("beta", encoding="utf-8")
code = (
"import sys;from pathlib import Path;"
"from obsidian_wiki.cache import update_source;"
"update_source(Path(sys.argv[1]), Path(sys.argv[2]))"
)
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parent.parent)}
procs = [
subprocess.Popen([sys.executable, "-c", code, str(vault), str(src)], env=env)
for src in (a, b)
]
for p in procs:
assert p.wait(timeout=30) == 0
sources = _load_manifest(vault)
assert str(a) in sources and str(b) in sources
class TestAtomicWrite:
def test_failed_write_leaves_original_intact(self, vault, monkeypatch):
_write_manifest(vault, {"sources": {"kept": {"content_hash": "abc"}}})
def boom(*args, **kwargs):
raise OSError("disk full")
monkeypatch.setattr("obsidian_wiki.cache.os.replace", boom)
with pytest.raises(OSError):
_write_manifest(vault, {"sources": {"lost": {}}})
assert _load_manifest(vault) == {"kept": {"content_hash": "abc"}}
leftovers = list(vault.glob(".manifest.json.*.tmp"))
assert leftovers == []
def test_manifest_is_never_partially_written(self, vault, src_file):
update_source(vault, src_file)
# A complete, parseable JSON document — never a truncated prefix.
json.loads(_manifest_path(vault).read_text(encoding="utf-8"))