feat(wiki): sync upstream v2026.10.4 core maintenance

Selectively port source hashing, locked manifest/memory writes, path-aware link lint and graph analysis, and revision-checked staging from upstream 4f457e154653756b04802d5aeb25c20f42b4770c. Record source hashes and exclusions. Preserve local archive/catalog citations, curated manifest metadata, config location, file-index and MCP behavior. Wire maintenance guidance into four skills without migrating vault contents or changing running services. Validation: 609 tests passed, including existing MCP/file-asset regressions and imported upstream maintenance tests. Read-only vault status left shared metadata hashes unchanged. Wiki MCP remains inactive; orders MCP is active.
2026-10-31 12:39:48 +00:00
parent 1dd5d8d753
commit 82890606aa
35 changed files with 12632 additions and 44 deletions
@@ -0,0 +1,536 @@
+"""Tests for the content-hash cache module."""
+import json
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+from obsidian_wiki.cache import (
+    ManifestLockTimeout,
+    check_sources,
+    compute_hash,
+    hash_file,
+    manifest_lock,
+    sha256_file,
+    sha256_dir,
+    update_source,
+    _load_manifest,
+    _lock_path,
+    _manifest_path,
+    _write_manifest,
+)
+
+
+@pytest.fixture
+def vault(tmp_path):
+    v = tmp_path / "vault"
+    v.mkdir()
+    return v
+
+
+@pytest.fixture
+def src_file(tmp_path):
+    f = tmp_path / "doc.md"
+    f.write_text("# Hello\nSome content.", encoding="utf-8")
+    return f
+
+
+@pytest.fixture
+def src_dir(tmp_path):
+    d = tmp_path / "repo"
+    d.mkdir()
+    (d / "a.py").write_text("x = 1")
+    (d / "b.py").write_text("y = 2")
+    return d
+
+
+# ---------------------------------------------------------------------------
+# Hash functions
+# ---------------------------------------------------------------------------
+
+class TestHashing:
+    def test_sha256_file_deterministic(self, src_file):
+        assert sha256_file(src_file) == sha256_file(src_file)
+
+    def test_sha256_file_changes_on_edit(self, src_file):
+        h1 = sha256_file(src_file)
+        src_file.write_text("# Different content")
+        h2 = sha256_file(src_file)
+        assert h1 != h2
+
+    def test_sha256_dir_deterministic(self, src_dir):
+        assert sha256_dir(src_dir) == sha256_dir(src_dir)
+
+    def test_sha256_dir_changes_on_edit(self, src_dir):
+        h1 = sha256_dir(src_dir)
+        (src_dir / "a.py").write_text("x = 999")
+        h2 = sha256_dir(src_dir)
+        assert h1 != h2
+
+    def test_compute_hash_dispatches(self, src_file, src_dir):
+        assert len(compute_hash(src_file)) == 64  # hex SHA-256
+        assert len(compute_hash(src_dir)) == 64
+
+    def test_sha256_dir_independent_of_path_separator(self, src_dir):
+        """Hash must match hashlib computed with POSIX separators regardless of platform (#178)."""
+        import hashlib
+
+        (src_dir / "sub").mkdir()
+        (src_dir / "sub" / "c.py").write_text("z = 3")
+
+        h = hashlib.sha256()
+        for fp in sorted(src_dir.rglob("*"), key=lambda p: p.relative_to(src_dir).as_posix()):
+            if fp.is_file():
+                h.update(fp.relative_to(src_dir).as_posix().encode())
+                h.update(sha256_file(fp).encode())
+        assert sha256_dir(src_dir) == h.hexdigest()
+
+    def test_hash_file_alias(self, src_file):
+        assert hash_file(src_file) == sha256_file(src_file)
+
+
+# ---------------------------------------------------------------------------
+# check_sources
+# ---------------------------------------------------------------------------
+
+class TestCheckSources:
+    def test_new_source(self, vault, src_file):
+        result = check_sources(vault, [src_file])
+        assert str(src_file) in result["new"]
+        assert result["modified"] == []
+        assert result["unchanged"] == []
+
+    def test_unchanged_after_update(self, vault, src_file):
+        update_source(vault, src_file)
+        result = check_sources(vault, [src_file])
+        assert str(src_file) in result["unchanged"]
+        assert result["new"] == []
+        assert result["modified"] == []
+
+    def test_modified_after_content_change(self, vault, src_file):
+        update_source(vault, src_file)
+        src_file.write_text("# Changed content")
+        result = check_sources(vault, [src_file])
+        assert str(src_file) in result["modified"]
+
+    def test_missing_path(self, vault, tmp_path):
+        ghost = tmp_path / "ghost.md"
+        result = check_sources(vault, [ghost])
+        assert str(ghost) in result["missing"]
+
+    def test_empty_source_list(self, vault):
+        result = check_sources(vault, [])
+        assert result == {
+            "new": [], "modified": [], "unchanged": [], "missing": [], "unavailable": []
+        }
+
+    def test_multiple_sources(self, vault, src_file, src_dir):
+        update_source(vault, src_file)
+        result = check_sources(vault, [src_file, src_dir])
+        assert str(src_file) in result["unchanged"]
+        assert str(src_dir) in result["new"]
+
+    def test_timestamp_irrelevant(self, vault, src_file):
+        # Touch the file (change mtime) without changing content — still unchanged
+        update_source(vault, src_file)
+        src_file.touch()
+        result = check_sources(vault, [src_file])
+        assert str(src_file) in result["unchanged"]
+
+    def _write_relative_manifest(self, vault, rel_key, content_hash):
+        """Write a manifest whose source key is stored vault-relative."""
+        _manifest_path(vault).write_text(
+            json.dumps(
+                {"sources": {rel_key: {"content_hash": content_hash, "last_ingested": "2026-07-14"}}}
+            ),
+            encoding="utf-8",
+        )
+
+    def test_relative_manifest_key_unchanged_for_abs_path(self, vault):
+        # Manifest stores a vault-relative key; caller passes the absolute path.
+        src = vault / "_raw" / "articles" / "foo.md"
+        src.parent.mkdir(parents=True)
+        src.write_text("body", encoding="utf-8")
+        self._write_relative_manifest(vault, "_raw/articles/foo.md", sha256_file(src))
+        result = check_sources(vault, [src])
+        assert str(src) in result["unchanged"]
+        assert result["new"] == []
+        assert result["missing"] == []
+
+    def test_relative_manifest_key_not_falsely_missing(self, vault):
+        # A relative key whose file exists under the vault must not be flagged missing,
+        # even when CWD != vault root.
+        src = vault / "_raw" / "articles" / "foo.md"
+        src.parent.mkdir(parents=True)
+        src.write_text("body", encoding="utf-8")
+        self._write_relative_manifest(vault, "_raw/articles/foo.md", sha256_file(src))
+        result = check_sources(vault, [])
+        assert "_raw/articles/foo.md" not in result["missing"]
+
+    def test_relative_manifest_key_modified(self, vault):
+        src = vault / "_raw" / "articles" / "foo.md"
+        src.parent.mkdir(parents=True)
+        src.write_text("body", encoding="utf-8")
+        self._write_relative_manifest(vault, "_raw/articles/foo.md", "stale-hash")
+        result = check_sources(vault, [src])
+        assert str(src) in result["modified"]
+
+    def test_relative_manifest_key_genuinely_missing(self, vault):
+        # A relative key whose top-level directory really is part of this vault
+        # is a vault-local loss when the file is gone.
+        (vault / "_raw" / "articles").mkdir(parents=True)
+        self._write_relative_manifest(vault, "_raw/articles/gone.md", "abc")
+        result = check_sources(vault, [])
+        assert "_raw/articles/gone.md" in result["missing"]
+
+    def test_relative_key_under_an_unknown_top_dir_is_unavailable(self, vault):
+        # A relative key whose first segment is not a vault entry is relative to
+        # some other root (a legacy ingest root, or an out-of-vault namespace), so
+        # it is unavailable here rather than a missing vault source.
+        self._write_relative_manifest(vault, "-Users-x/abc.jsonl", "abc")
+        result = check_sources(vault, [])
+        assert result["missing"] == []
+        assert "-Users-x/abc.jsonl" in result["unavailable"]
+
+    def test_check_sources_resolves_each_path_once(self, vault, tmp_path, monkeypatch):
+        """Matching resolves each entry and each query once (an index of forms), not once
+        per (entry, query) pair: with relative keys the string fast path never fires, and a
+        3,000-entry manifest made every queried file cost a second of ``realpath`` walks."""
+        src = tmp_path / "src"
+        src.mkdir()
+        files = []
+        for i in range(300):
+            f = src / f"f{i:03d}.md"
+            f.write_text(f"doc {i}", encoding="utf-8")
+            files.append(f)
+        sources = {}
+        for i, f in enumerate(files):
+            # "../src/f000.md" — the relative form the CLI writes for out-of-vault sources
+            sources[os.path.relpath(f, vault)] = {"content_hash": sha256_file(f) if i % 3 else "0" * 64}
+        for i in range(2000):  # unrelated entries, like a real manifest
+            sources[f"../elsewhere/e{i:04d}.md"] = {"content_hash": "1" * 64}
+        _write_manifest(vault, {"version": 1, "sources": sources})
+
+        calls = {"n": 0}
+        real_resolve = Path.resolve
+
+        def counting_resolve(self, *args, **kwargs):
+            calls["n"] += 1
+            return real_resolve(self, *args, **kwargs)
+
+        monkeypatch.setattr(Path, "resolve", counting_resolve)
+        result = check_sources(vault, files)
+
+        assert len(result["unchanged"]) == 200
+        assert len(result["modified"]) == 100
+        assert result["new"] == []
+        assert len(result["missing"]) + len(result["unavailable"]) == 2000
+        # one resolve per entry and per query (plus a handful inside helpers), never entries × queries
+        assert calls["n"] <= 3 * (2300 + 300), calls["n"]
+
+    def test_check_sources_first_entry_wins_for_duplicate_forms(self, vault, tmp_path):
+        """Two entries naming the same file in different forms: the earlier one is the match,
+        the later one is not reported missing (it is a duplicate of a scanned path)."""
+        f = tmp_path / "dup.md"
+        f.write_text("same", encoding="utf-8")
+        sources = {
+            os.path.relpath(f, vault): {"content_hash": "0" * 64},           # earlier: stale hash → modified
+            str(f): {"content_hash": sha256_file(f)},                        # later: same file, absolute form
+        }
+        _write_manifest(vault, {"version": 1, "sources": sources})
+        result = check_sources(vault, [f])
+        assert result["modified"] == [str(f)] and result["unchanged"] == []
+        assert result["missing"] == [] and result["unavailable"] == []
+
+
+# ---------------------------------------------------------------------------
+# update_source / manifest
+# ---------------------------------------------------------------------------
+
+class TestUpdateSource:
+    def test_writes_manifest(self, vault, src_file):
+        update_source(vault, src_file)
+        assert _manifest_path(vault).exists()
+
+    def test_records_correct_hash(self, vault, src_file):
+        h = update_source(vault, src_file)
+        assert h == sha256_file(src_file)
+        sources = _load_manifest(vault)
+        assert sources[str(src_file)]["content_hash"] == h
+
+    def test_records_pages_produced(self, vault, src_file):
+        update_source(vault, src_file, pages_produced=["concepts/foo.md", "entities/bar.md"])
+        sources = _load_manifest(vault)
+        assert sources[str(src_file)]["pages_produced"] == ["concepts/foo.md", "entities/bar.md"]
+
+    def test_records_last_ingested_timestamp(self, vault, src_file):
+        update_source(vault, src_file)
+        sources = _load_manifest(vault)
+        assert "last_ingested" in sources[str(src_file)]
+
+    def test_update_overwrites_old_hash(self, vault, src_file):
+        update_source(vault, src_file)
+        src_file.write_text("new content")
+        h2 = update_source(vault, src_file)
+        sources = _load_manifest(vault)
+        assert sources[str(src_file)]["content_hash"] == h2
+
+    def test_preserves_other_manifest_entries(self, vault, src_file, src_dir):
+        update_source(vault, src_file)
+        update_source(vault, src_dir)
+        sources = _load_manifest(vault)
+        assert str(src_file) in sources
+        assert str(src_dir) in sources
+
+    def test_update_source_live_draft_beats_same_named_archive(self, vault):
+        staging = vault / "_raw" / "notes.md"
+        archived = vault / "_raw" / "_archived" / "notes.md"
+        archived.parent.mkdir(parents=True, exist_ok=True)
+        staging.parent.mkdir(parents=True, exist_ok=True)
+        staging.write_text("staging-bytes\n", encoding="utf-8")
+        archived.write_text("archived-bytes\n", encoding="utf-8")
+        h = update_source(vault, staging)
+        assert h == compute_hash(staging)
+        sources = _load_manifest(vault)
+        assert "_raw/notes.md" in sources
+        assert "_raw/_archived/notes.md" not in sources
+
+    def test_update_source_hashes_archive_when_staging_gone(self, vault):
+        staging = vault / "_raw" / "notes.md"
+        archived = vault / "_raw" / "_archived" / "notes.md"
+        archived.parent.mkdir(parents=True, exist_ok=True)
+        staging.parent.mkdir(parents=True, exist_ok=True)
+        staging.write_text("staging-bytes\n", encoding="utf-8")
+        archived.write_text("archived-bytes\n", encoding="utf-8")
+        update_source(vault, staging)
+        staging.unlink()
+        h = update_source(vault, staging)
+        assert h == compute_hash(archived)
+        assert "_raw/_archived/notes.md" in _load_manifest(vault)
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+class TestCacheCLI:
+    def _run(self, *args, cwd=None):
+        return subprocess.run(
+            [sys.executable, "-m", "obsidian_wiki.cli", *args],
+            capture_output=True, text=True, cwd=cwd,
+        )
+
+    def test_cache_hash_file(self, src_file):
+        proc = self._run("cache-hash", str(src_file))
+        assert proc.returncode == 0
+        data = json.loads(proc.stdout)
+        assert data["sha256"] == sha256_file(src_file)
+
+    def test_cache_hash_missing_exits_nonzero(self, tmp_path):
+        proc = self._run("cache-hash", str(tmp_path / "nope.md"))
+        assert proc.returncode != 0
+
+    def test_cache_check_new(self, vault, src_file):
+        proc = self._run("cache-check", str(vault), str(src_file))
+        assert proc.returncode == 0
+        data = json.loads(proc.stdout)
+        assert str(src_file) in data["new"]
+
+    def test_cache_check_pretty(self, vault, src_file):
+        proc = self._run("cache-check", "--pretty", str(vault), str(src_file))
+        assert proc.returncode == 0
+        assert "\n  " in proc.stdout
+
+    def test_cache_update_then_check_unchanged(self, vault, src_file):
+        self._run("cache-update", str(vault), str(src_file))
+        proc = self._run("cache-check", str(vault), str(src_file))
+        data = json.loads(proc.stdout)
+        assert str(src_file) in data["unchanged"]
+
+    def test_cache_update_with_pages(self, vault, src_file):
+        proc = self._run("cache-update", str(vault), str(src_file),
+                         "--pages", "concepts/foo.md", "entities/bar.md")
+        assert proc.returncode == 0
+        sources = _load_manifest(vault)
+        assert sources[str(src_file)]["pages_produced"] == ["concepts/foo.md", "entities/bar.md"]
+
+    def test_cache_update_non_portable_source_warns_on_stderr(self, vault, src_file):
+        # src_file lives outside the vault and $HOME, so the fallback absolute
+        # key is stored — but the CLI must say so rather than stay silent.
+        proc = self._run("cache-update", str(vault), str(src_file))
+        assert proc.returncode == 0
+        assert "no portable key" in proc.stderr
+
+    def test_cache_update_explicit_key_warns_nothing(self, vault, src_file):
+        proc = self._run("cache-update", str(vault), str(src_file), "--key", "repo:o/n")
+        assert proc.returncode == 0
+        assert "no portable key" not in proc.stderr
+        assert "repo:o/n" in _load_manifest(vault)
+
+    def test_cache_update_cli_uses_archive_if_staging_gone(self, vault):
+        staging = vault / "_raw" / "notes.md"
+        archived = vault / "_raw" / "_archived" / "notes.md"
+        archived.parent.mkdir(parents=True, exist_ok=True)
+        staging.parent.mkdir(parents=True, exist_ok=True)
+        staging.write_text("staging-bytes\n", encoding="utf-8")
+        archived.write_text("archived-bytes\n", encoding="utf-8")
+        staging.unlink()
+        proc = self._run("cache-update", str(vault), str(staging))
+        assert proc.returncode == 0
+        data = json.loads(proc.stdout)
+        assert data["key"] == "_raw/_archived/notes.md"
+        assert data["content_hash"] == compute_hash(archived)
+
+
+class TestCacheCLIVaultRelativeSources:
+    """Relative source arguments must also resolve against the vault root.
+
+    The manifest stores vault-relative keys, and ``cache.resolve_key`` reads
+    them that way — so ``cache-update <vault> Raw/x.pdf`` has to work from any
+    CWD, not just when the shell happens to sit at the vault root.
+    """
+
+    @pytest.fixture
+    def vault(self, tmp_path):
+        v = tmp_path / "vault"
+        (v / "Raw" / "database").mkdir(parents=True)
+        (v / "Raw" / "database" / "x.pdf").write_bytes(b"pdf-bytes")
+        return v
+
+    def _run(self, *args, cwd=None):
+        return subprocess.run(
+            [sys.executable, "-m", "obsidian_wiki.cli", *args],
+            capture_output=True, text=True, cwd=cwd,
+        )
+
+    def test_cache_update_vault_relative_from_other_cwd(self, vault, tmp_path):
+        elsewhere = tmp_path / "elsewhere"
+        elsewhere.mkdir()
+        proc = self._run("cache-update", str(vault), "Raw/database/x.pdf",
+                         cwd=elsewhere)
+        assert proc.returncode == 0, proc.stderr
+        data = json.loads(proc.stdout)
+        assert data["key"] == "Raw/database/x.pdf"
+        assert "Raw/database/x.pdf" in json.dumps(_load_manifest(vault))
+
+    def test_cache_check_vault_relative_from_other_cwd(self, vault, tmp_path):
+        update_source(vault, vault / "Raw" / "database" / "x.pdf")
+        elsewhere = tmp_path / "elsewhere"
+        elsewhere.mkdir()
+        proc = self._run("cache-check", str(vault), "Raw/database/x.pdf",
+                         cwd=elsewhere)
+        assert proc.returncode == 0, proc.stderr
+        data = json.loads(proc.stdout)
+        assert str(vault / "Raw" / "database" / "x.pdf") in data["unchanged"]
+        assert data["missing"] == []
+
+    def test_cache_update_cwd_relative_still_wins(self, vault, tmp_path):
+        # Backward compat: an out-of-vault relative path resolves against the
+        # CWD as before, even when the vault contains a same-named file.
+        workdir = tmp_path / "workdir"
+        (workdir / "Raw" / "database").mkdir(parents=True)
+        (workdir / "Raw" / "database" / "x.pdf").write_bytes(b"different-bytes")
+        proc = self._run("cache-update", str(vault), "Raw/database/x.pdf",
+                         cwd=workdir)
+        assert proc.returncode == 0, proc.stderr
+        data = json.loads(proc.stdout)
+        assert data["path"] == str(workdir / "Raw" / "database" / "x.pdf")
+        # The typed string is byte-for-byte a manifest key, yet the vault file
+        # it names is not the one hashed — say so rather than choosing silently.
+        assert (f"note: Raw/database/x.pdf matched both "
+                f"{workdir / 'Raw' / 'database' / 'x.pdf'} and "
+                f"{vault / 'Raw' / 'database' / 'x.pdf'}") in proc.stderr
+
+    def test_cache_update_missing_source_clean_error(self, vault, tmp_path):
+        elsewhere = tmp_path / "elsewhere"
+        elsewhere.mkdir()
+        proc = self._run("cache-update", str(vault), "Raw/absent.md", cwd=elsewhere)
+        assert proc.returncode == 1
+        assert "Traceback" not in proc.stderr
+        assert str(elsewhere / "Raw" / "absent.md") in proc.stderr
+        assert str(vault / "Raw" / "absent.md") in proc.stderr
+
+    def test_cache_update_absolute_missing_clean_error(self, vault):
+        proc = self._run("cache-update", str(vault), str(vault / "Raw" / "gone.pdf"))
+        assert proc.returncode == 1
+        assert "Traceback" not in proc.stderr
+        assert str(vault / "Raw" / "gone.pdf") in proc.stderr
+
+
+class TestManifestLock:
+    """Concurrent writers must serialize instead of clobbering the manifest."""
+
+    def test_lock_is_released_after_use(self, vault):
+        with manifest_lock(vault):
+            assert _lock_path(vault).exists()
+        assert not _lock_path(vault).exists()
+
+    def test_second_holder_times_out(self, vault):
+        with manifest_lock(vault):
+            with pytest.raises(ManifestLockTimeout):
+                with manifest_lock(vault, timeout=0.3):
+                    pass
+
+    def test_stale_lock_is_stolen(self, vault):
+        lock = _lock_path(vault)
+        lock.write_text("999999")
+        old = time.time() - 120
+        os.utime(lock, (old, old))
+        with manifest_lock(vault, timeout=0.5, stale_after=60.0):
+            pass
+        assert not lock.exists()
+
+    def test_lock_released_when_body_raises(self, vault):
+        with pytest.raises(ValueError):
+            with manifest_lock(vault):
+                raise ValueError("boom")
+        assert not _lock_path(vault).exists()
+
+    def test_update_source_leaves_no_lock_behind(self, vault, src_file):
+        update_source(vault, src_file)
+        assert not _lock_path(vault).exists()
+
+    def test_concurrent_updates_both_survive(self, vault, tmp_path):
+        """The race the lock exists for: two processes, neither entry lost."""
+        a = tmp_path / "a.md"
+        a.write_text("alpha", encoding="utf-8")
+        b = tmp_path / "b.md"
+        b.write_text("beta", encoding="utf-8")
+        code = (
+            "import sys;from pathlib import Path;"
+            "from obsidian_wiki.cache import update_source;"
+            "update_source(Path(sys.argv[1]), Path(sys.argv[2]))"
+        )
+        env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parent.parent)}
+        procs = [
+            subprocess.Popen([sys.executable, "-c", code, str(vault), str(src)], env=env)
+            for src in (a, b)
+        ]
+        for p in procs:
+            assert p.wait(timeout=30) == 0
+        sources = _load_manifest(vault)
+        assert str(a) in sources and str(b) in sources
+
+
+class TestAtomicWrite:
+    def test_failed_write_leaves_original_intact(self, vault, monkeypatch):
+        _write_manifest(vault, {"sources": {"kept": {"content_hash": "abc"}}})
+
+        def boom(*args, **kwargs):
+            raise OSError("disk full")
+
+        monkeypatch.setattr("obsidian_wiki.cache.os.replace", boom)
+        with pytest.raises(OSError):
+            _write_manifest(vault, {"sources": {"lost": {}}})
+
+        assert _load_manifest(vault) == {"kept": {"content_hash": "abc"}}
+        leftovers = list(vault.glob(".manifest.json.*.tmp"))
+        assert leftovers == []
+
+    def test_manifest_is_never_partially_written(self, vault, src_file):
+        update_source(vault, src_file)
+        # A complete, parseable JSON document — never a truncated prefix.
+        json.loads(_manifest_path(vault).read_text(encoding="utf-8"))