Codex
82890606aa
Selectively port source hashing, locked manifest/memory writes, path-aware link lint and graph analysis, and revision-checked staging from upstream 4f457e154653756b04802d5aeb25c20f42b4770c. Record source hashes and exclusions. Preserve local archive/catalog citations, curated manifest metadata, config location, file-index and MCP behavior. Wire maintenance guidance into four skills without migrating vault contents or changing running services. Validation: 609 tests passed, including existing MCP/file-asset regressions and imported upstream maintenance tests. Read-only vault status left shared metadata hashes unchanged. Wiki MCP remains inactive; orders MCP is active.
766 lines
28 KiB
Python
766 lines
28 KiB
Python
"""Content-hash cache for wiki-ingest source tracking.
|
|
|
|
Provides a reliable, platform-independent alternative to running `sha256sum`
|
|
in the skill. The agent calls `obsidian-wiki cache-check` / `cache-update`
|
|
instead of shelling out to sha256sum and manually parsing .manifest.json.
|
|
|
|
Manifest format (.manifest.json in the vault root). Two `sources` shapes are
|
|
supported, because real vaults contain both:
|
|
|
|
1. Dict keyed by path (this module's original format)::
|
|
|
|
{"sources": {"<abs-or-rel-path>": {"content_hash": "...", ...}}}
|
|
|
|
2. List of entry objects, each carrying its own ``path`` (the shape the
|
|
wiki-ingest skill writes)::
|
|
|
|
{"sources": [{"path": "_raw/foo.md", "content_hash": "sha256:...", ...}]}
|
|
|
|
Both shapes are read transparently, and `update_source` edits the manifest
|
|
*in place* — preserving its shape, any duplicate-path entries, and
|
|
skill-written fields (``pages_created``, ``size_bytes``, ``source_type``, …).
|
|
|
|
Hashes are compared prefix-insensitively: the skill records
|
|
``"sha256:<hex>"`` while this module computes a bare ``<hex>``, so an optional
|
|
``algo:`` prefix is stripped from both sides before comparison.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Iterator, TypedDict
|
|
|
|
|
|
class SourceEntry(TypedDict, total=False):
|
|
content_hash: str
|
|
last_ingested: str
|
|
pages_produced: list[str]
|
|
|
|
|
|
class CheckResult(TypedDict):
|
|
new: list[str]
|
|
modified: list[str]
|
|
unchanged: list[str]
|
|
missing: list[str] # vault-local source in manifest but no longer on disk
|
|
unavailable: list[str] # machine-local source absent here (e.g. synced from another host)
|
|
|
|
|
|
def _manifest_path(vault: Path) -> Path:
|
|
return vault / ".manifest.json"
|
|
|
|
|
|
def _load_raw(vault: Path) -> dict:
|
|
"""Return the full manifest object (``{}`` if absent/unreadable)."""
|
|
mp = _manifest_path(vault)
|
|
if not mp.exists():
|
|
return {}
|
|
try:
|
|
data = json.loads(mp.read_text(encoding="utf-8"))
|
|
return data if isinstance(data, dict) else {}
|
|
except (json.JSONDecodeError, OSError):
|
|
return {}
|
|
|
|
|
|
def _load_manifest(vault: Path):
|
|
"""Return the raw ``sources`` value — a dict, a list, or ``{}`` if absent.
|
|
|
|
Kept for backward compatibility; callers that need shape-agnostic access
|
|
should use :func:`_iter_entries`.
|
|
"""
|
|
return _load_raw(vault).get("sources", {})
|
|
|
|
|
|
def _lock_path(vault: Path) -> Path:
|
|
return vault / ".manifest.lock"
|
|
|
|
|
|
class AdvisoryLockTimeout(RuntimeError):
|
|
"""Raised when an advisory lockfile could not be claimed within the timeout."""
|
|
|
|
|
|
class ManifestLockTimeout(AdvisoryLockTimeout):
|
|
"""Raised when the manifest lock could not be acquired within the timeout."""
|
|
|
|
|
|
@contextmanager
|
|
def advisory_lock(
|
|
lock: Path,
|
|
*,
|
|
timeout: float = 10.0,
|
|
stale_after: float = 60.0,
|
|
error_cls: type = AdvisoryLockTimeout,
|
|
):
|
|
"""Claim *lock* as an advisory lockfile for a read-modify-write.
|
|
|
|
``O_CREAT | O_EXCL`` is the portable stdlib primitive here — ``fcntl`` is
|
|
POSIX-only and this ships on Windows. A lockfile older than *stale_after*
|
|
is assumed to belong to a crashed process and is stolen.
|
|
|
|
Shared by the manifest writer and the prose-memory writer
|
|
(:mod:`obsidian_wiki.memory`) so both serialise the same way.
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
fd = None
|
|
while True:
|
|
try:
|
|
fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
break
|
|
except FileExistsError:
|
|
try:
|
|
age = time.time() - lock.stat().st_mtime
|
|
except FileNotFoundError:
|
|
continue # holder released it between our open and stat
|
|
if age > stale_after:
|
|
# ponytail: last-writer-wins steal; a pid check would narrow the
|
|
# window if two processes ever race to steal the same stale lock
|
|
try:
|
|
lock.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
continue
|
|
if time.monotonic() >= deadline:
|
|
raise error_cls(
|
|
f"could not acquire {lock} within {timeout}s "
|
|
f"(held for {age:.1f}s; stale after {stale_after}s)"
|
|
)
|
|
time.sleep(0.1)
|
|
try:
|
|
os.write(fd, str(os.getpid()).encode())
|
|
os.close(fd)
|
|
fd = None
|
|
yield
|
|
finally:
|
|
if fd is not None:
|
|
os.close(fd)
|
|
try:
|
|
lock.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
@contextmanager
|
|
def manifest_lock(vault: Path, *, timeout: float = 10.0, stale_after: float = 60.0):
|
|
"""Advisory lock around a manifest read-modify-write.
|
|
|
|
Parallel ingest agents (``batch-plan`` fan-out) and the Docker server can
|
|
all write one vault's manifest; without this the read-modify-write races
|
|
and a whole entry is silently lost.
|
|
"""
|
|
# ponytail: whole-manifest advisory lock; per-entry locking if batch fan-out ever contends
|
|
with advisory_lock(
|
|
_lock_path(vault),
|
|
timeout=timeout,
|
|
stale_after=stale_after,
|
|
error_cls=ManifestLockTimeout,
|
|
):
|
|
yield
|
|
|
|
|
|
def _write_manifest(vault: Path, manifest: dict) -> None:
|
|
"""Atomically replace the manifest file — no torn reads, no partial file."""
|
|
target = _manifest_path(vault)
|
|
tmp = target.with_name(f".manifest.json.{os.getpid()}.tmp")
|
|
try:
|
|
tmp.write_text(
|
|
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
os.replace(tmp, target) # atomic on POSIX and Windows
|
|
except BaseException:
|
|
try:
|
|
tmp.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def _save_manifest(vault: Path, sources) -> None:
|
|
"""Write *sources* back into the manifest, preserving other top-level keys."""
|
|
with manifest_lock(vault):
|
|
manifest = _load_raw(vault)
|
|
manifest["sources"] = sources
|
|
_write_manifest(vault, manifest)
|
|
|
|
|
|
def _iter_entries(sources) -> Iterator[tuple[str | None, dict]]:
|
|
"""Yield ``(stored_key, entry)`` pairs for either manifest shape.
|
|
|
|
For the dict shape the key is the dict key; for the list shape it is the
|
|
entry's ``path`` (falling back to ``source_id``).
|
|
"""
|
|
if isinstance(sources, dict):
|
|
for key, entry in sources.items():
|
|
if isinstance(entry, dict):
|
|
yield key, entry
|
|
elif isinstance(sources, list):
|
|
for entry in sources:
|
|
if isinstance(entry, dict):
|
|
yield (entry.get("path") or entry.get("source_id")), entry
|
|
|
|
|
|
# Matches "sha256:", "https://", "slack:#..." — anything with an algo/scheme
|
|
# prefix that shouldn't be treated as a filesystem path.
|
|
_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:[^\\/]")
|
|
|
|
# A Windows drive-absolute path ("C:\dir\file", "C:/dir/file"). On POSIX this is
|
|
# not a path at all, so it cannot be resolved and is machine-specific; lint.py
|
|
# recognises the same shape when reporting stored keys. Kept local rather than
|
|
# imported so the runtime cache carries no dependency on the lint module.
|
|
_WINDOWS_ABS_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
|
|
|
|
|
def _strip_algo(value: str | None) -> str:
|
|
"""Drop an optional ``algo:`` prefix so ``sha256:<hex>`` == ``<hex>``."""
|
|
value = value or ""
|
|
return value.split(":", 1)[1] if ":" in value else value
|
|
|
|
|
|
def _format_hash(existing: str | None, new_hex: str) -> str:
|
|
"""Format *new_hex* keeping any ``algo:`` prefix the existing value used."""
|
|
if existing and ":" in existing:
|
|
return f"{existing.split(':', 1)[0]}:{new_hex}"
|
|
return new_hex
|
|
|
|
|
|
def _is_file_key(key: str | None) -> bool:
|
|
"""True if *key* looks like a filesystem path rather than a URL/pseudo-key."""
|
|
return bool(key) and "://" not in key and not _SCHEME_RE.match(key)
|
|
|
|
|
|
def _expand_key(key: str) -> str:
|
|
"""Expand ``~`` and environment variables, but only when the key uses them.
|
|
|
|
A plain vault-relative key like ``Raw/x.pdf`` must survive untouched, so the
|
|
expansion is gated rather than applied unconditionally.
|
|
"""
|
|
if key.startswith("~") or "$" in key:
|
|
return os.path.expandvars(os.path.expanduser(key))
|
|
return key
|
|
|
|
|
|
def resolve_key(key: str | None, vault: Path) -> Path | None:
|
|
"""Normalize a manifest key to an absolute path, or ``None`` for non-file keys.
|
|
|
|
Resolution order (the source key contract v2, see ``llm-wiki/SKILL.md``):
|
|
|
|
1. pseudo-key (``scheme:`` / ``://``) -> ``None`` — an opaque identifier, not
|
|
a filesystem location;
|
|
2. ``~``- or env-var-bearing key -> expand;
|
|
3. absolute path -> use as-is (legacy compatibility);
|
|
4. anything else -> resolve against the vault root.
|
|
"""
|
|
if not _is_file_key(key):
|
|
return None
|
|
path = Path(_expand_key(key))
|
|
return path if path.is_absolute() else (vault / path)
|
|
|
|
|
|
def stored_key(path: Path, vault: Path) -> str | None:
|
|
"""Return the portable manifest key for *path*, or ``None`` if not portable.
|
|
|
|
In-vault sources become vault-relative (``Raw/x.pdf``); sources under
|
|
``$HOME`` become home-relative (``~/.claude/...``); anything else has no
|
|
machine-portable representation and needs an explicit pseudo-key from the
|
|
caller (``repo:``, ``url:``, ``agent:``).
|
|
"""
|
|
p = Path(os.path.abspath(_expand_key(str(path))))
|
|
vault_root = Path(os.path.abspath(_expand_key(str(vault))))
|
|
home_root = Path(os.path.expanduser("~"))
|
|
for root, prefix in ((vault_root, ""), (home_root, "~/")):
|
|
try:
|
|
rel = p.relative_to(root)
|
|
except ValueError:
|
|
continue
|
|
if rel == Path("."):
|
|
return None
|
|
return prefix + rel.as_posix()
|
|
return None
|
|
|
|
|
|
def _same_source(stored_key_value: str | None, query: Path, vault: Path) -> bool:
|
|
"""True if a manifest key refers to the same source as *query*.
|
|
|
|
Matches on the raw string first (covers pseudo-keys), then on the resolved
|
|
absolute form so an absolute query matches a vault-relative or home-relative
|
|
stored key.
|
|
"""
|
|
if not stored_key_value:
|
|
return False
|
|
if str(query) == stored_key_value:
|
|
return True
|
|
stored_path = resolve_key(stored_key_value, vault)
|
|
if stored_path is None:
|
|
return False
|
|
try:
|
|
return stored_path.resolve() == query.resolve()
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _missing_on_disk(key: str | None, vault: Path) -> bool:
|
|
"""True if a filesystem-style manifest key has no file on disk."""
|
|
path = resolve_key(key, vault)
|
|
if path is None:
|
|
return False
|
|
return not path.exists()
|
|
|
|
|
|
def _vault_top_names(vault: Path) -> set[str]:
|
|
"""Names of the vault's top-level entries, or an empty set if unreadable."""
|
|
try:
|
|
return {p.name for p in vault.iterdir()}
|
|
except OSError:
|
|
return set()
|
|
|
|
|
|
def _is_vault_local(key: str | None, vault: Path, top_names: set[str] | None = None) -> bool:
|
|
"""True if a file key names a source that travels with the vault.
|
|
|
|
Vault-local sources travel with the vault, so their absence is a real loss
|
|
(``missing``). A key that is machine-specific *and* points outside this vault
|
|
(``/home/other/wiki/x.md``, ``~/.claude/...``) may simply not exist on the
|
|
machine reading a synced vault, so it is reported under ``unavailable``. An
|
|
absolute or ``~``-relative path that resolves *inside* this vault is still
|
|
vault-local — machine-specific in form, but not in target.
|
|
|
|
A *relative* key resolves lexically inside the vault even when it is really
|
|
relative to some other root — the legacy ingest-root keys such as
|
|
``-Users-x-github/abc.jsonl``. Path shape cannot tell those apart from vault
|
|
keys, so the vault's own topology decides: the first segment must name a real
|
|
top-level entry. That also gives the honest answer for an out-of-vault source
|
|
parked in a namespace the vault does not contain (``external/.hermes/...``):
|
|
unavailable, not missing. When the vault cannot be listed the lexical answer
|
|
stands, so an I/O hiccup never hides a real loss.
|
|
|
|
A key written for another OS — a drive-letter path or one using backslashes
|
|
as separators — cannot be resolved on this host, so it is machine-specific
|
|
(``unavailable``) rather than a vault-relative key.
|
|
"""
|
|
path = resolve_key(key, vault)
|
|
if path is None:
|
|
return False
|
|
raw = str(key)
|
|
try:
|
|
rel = Path(os.path.abspath(str(path))).relative_to(Path(os.path.abspath(str(vault))))
|
|
except ValueError:
|
|
return False
|
|
if rel == Path("."):
|
|
return False
|
|
if os.name != "nt" and ("\\" in raw or _WINDOWS_ABS_RE.match(raw)):
|
|
# A key written for another OS: a backslash is not a separator here and a
|
|
# drive-letter path cannot be resolved, so this is machine-specific — not
|
|
# a vault source whose file went away.
|
|
return False
|
|
if os.path.isabs(raw) or raw.startswith("~") or "$" in raw:
|
|
return True
|
|
if "/" not in raw and "\\" not in raw:
|
|
# A bare filename at the vault root. There is no leading component that
|
|
# could be a foreign root, so this is a vault source by construction —
|
|
# and if it is gone, that is a real vault-local loss. The backslash guard
|
|
# keeps a Windows-form key out of this shortcut on every platform.
|
|
return True
|
|
if top_names is None:
|
|
top_names = _vault_top_names(vault)
|
|
if not top_names:
|
|
return True
|
|
return rel.parts[0] in top_names
|
|
|
|
|
|
def sha256_file(path: Path, chunk: int = 1 << 20) -> str:
|
|
"""Return the hex SHA-256 digest of *path* without loading it all into RAM."""
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as fh:
|
|
while True:
|
|
block = fh.read(chunk)
|
|
if not block:
|
|
break
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def sha256_dir(path: Path) -> str:
|
|
"""Stable SHA-256 over all files in a directory tree (sorted by relative path)."""
|
|
h = hashlib.sha256()
|
|
for fp in sorted(path.rglob("*"), key=lambda p: p.relative_to(path).as_posix()):
|
|
if fp.is_file():
|
|
rel = fp.relative_to(path).as_posix()
|
|
h.update(rel.encode())
|
|
h.update(sha256_file(fp).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def compute_hash(path: Path) -> str:
|
|
if path.is_dir():
|
|
return sha256_dir(path)
|
|
return sha256_file(path)
|
|
|
|
|
|
def _resolved_form(path: Path) -> str | None:
|
|
"""``realpath`` of *path* as a comparable string, or ``None`` if it cannot be resolved.
|
|
|
|
``os.path.normcase`` keeps the comparison equivalent to ``Path`` equality on
|
|
case-insensitive filesystems.
|
|
"""
|
|
try:
|
|
return os.path.normcase(str(path.resolve()))
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def check_sources(vault: Path, source_paths: list[Path]) -> CheckResult:
|
|
"""Classify each source as new / modified / unchanged vs. the manifest.
|
|
|
|
Also reports manifest entries whose source file no longer exists on disk.
|
|
Handles both manifest shapes and compares hashes prefix-insensitively.
|
|
|
|
Every entry and every query is resolved once and matched through an index
|
|
of its forms (raw key, resolved path). Resolving is a ``realpath`` walk —
|
|
one ``lstat`` per path component — so doing it per (entry, query) pair, as
|
|
the first-match scan did, made a 3,000-entry manifest cost about a second
|
|
per queried file. The index keeps the first entry for each form, which is
|
|
what the scan's ``break`` on first hit returned.
|
|
"""
|
|
entries = list(_iter_entries(_load_manifest(vault)))
|
|
result: CheckResult = {
|
|
"new": [], "modified": [], "unchanged": [], "missing": [], "unavailable": []
|
|
}
|
|
|
|
by_form: dict[str, int] = {}
|
|
entry_resolved: list[str | None] = []
|
|
for i, (stored_key, _entry) in enumerate(entries):
|
|
resolved = None
|
|
if stored_key:
|
|
by_form.setdefault(stored_key, i)
|
|
stored_path = resolve_key(stored_key, vault)
|
|
if stored_path is not None:
|
|
resolved = _resolved_form(stored_path)
|
|
if resolved is not None:
|
|
by_form.setdefault(resolved, i)
|
|
entry_resolved.append(resolved)
|
|
|
|
matched: set[int] = set()
|
|
query_forms: set[str] = set()
|
|
for path in source_paths:
|
|
key = str(path)
|
|
forms = [key]
|
|
resolved = _resolved_form(path)
|
|
if resolved is not None:
|
|
forms.append(resolved)
|
|
query_forms.update(forms)
|
|
if not path.exists():
|
|
result["missing"].append(key)
|
|
continue
|
|
current_hash = _strip_algo(compute_hash(path))
|
|
hits = [by_form[f] for f in forms if f in by_form]
|
|
if not hits:
|
|
result["new"].append(key)
|
|
continue
|
|
i = min(hits) # the earliest entry matching by either form — the old scan's first hit
|
|
matched.add(i)
|
|
entry = entries[i][1]
|
|
if _strip_algo(entry.get("content_hash")) != current_hash:
|
|
result["modified"].append(key)
|
|
else:
|
|
result["unchanged"].append(key)
|
|
|
|
# Report manifest entries whose source file no longer exists on disk and
|
|
# that weren't among the scanned paths (in any path form).
|
|
top_names = _vault_top_names(vault)
|
|
for i, (stored_key, _entry) in enumerate(entries):
|
|
if i in matched:
|
|
continue
|
|
if stored_key in query_forms or (entry_resolved[i] is not None and entry_resolved[i] in query_forms):
|
|
continue
|
|
if _missing_on_disk(stored_key, vault):
|
|
bucket = "missing" if _is_vault_local(stored_key, vault, top_names) else "unavailable"
|
|
result[bucket].append(stored_key)
|
|
|
|
return result
|
|
|
|
|
|
def update_source(
|
|
vault: Path,
|
|
source_path: Path,
|
|
*,
|
|
pages_produced: list[str] | None = None,
|
|
key: str | None = None,
|
|
) -> str:
|
|
"""Record the current hash of *source_path* in the manifest. Returns the hash.
|
|
|
|
Edits the manifest in place: matches an existing entry across path forms and
|
|
updates it, otherwise appends a new one — preserving the manifest's shape
|
|
(dict or list), duplicate-path entries, and any skill-written fields.
|
|
|
|
The stored key is normalized to the portable form (vault-relative for
|
|
in-vault sources, ``~``-relative under ``$HOME``) so a synced vault never
|
|
accumulates machine absolute paths. Pass *key* explicitly for sources that
|
|
have no filesystem representation in either form — a pseudo-key such as
|
|
``repo:github.com/owner/name``, ``url:https://...``, or ``agent:claude/<id>``.
|
|
When neither applies, the raw path is kept for backward compatibility and a
|
|
warning is written to stderr, because that entry is not portable across
|
|
machines.
|
|
"""
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
explicit_key = key is not None
|
|
hash_path = source_path
|
|
if explicit_key:
|
|
new_key = key
|
|
else:
|
|
derived = stored_key(source_path, vault)
|
|
if derived is None:
|
|
new_key = str(source_path)
|
|
print(
|
|
f"warning: {source_path} is outside the vault and $HOME, so it has no "
|
|
f"portable key; storing the absolute path. Pass key=<repo:|url:|agent:> "
|
|
f"to keep the vault portable across machines.",
|
|
file=sys.stderr,
|
|
)
|
|
else:
|
|
from obsidian_wiki.provenance import prefer_archive_write_path
|
|
preferred = prefer_archive_write_path(vault, derived)
|
|
if preferred is not None:
|
|
new_key = preferred
|
|
hash_path = vault / preferred
|
|
else:
|
|
new_key = derived
|
|
# Hash outside the lock — hashing a large source tree can take seconds and
|
|
# nothing else in the manifest depends on it.
|
|
current_hash = compute_hash(hash_path)
|
|
|
|
with manifest_lock(vault):
|
|
return _update_source_locked(
|
|
vault, source_path, new_key, explicit_key, current_hash, now, pages_produced,
|
|
)
|
|
|
|
|
|
def _update_source_locked(
|
|
vault: Path,
|
|
source_path: Path,
|
|
new_key: str,
|
|
explicit_key: bool,
|
|
current_hash: str,
|
|
now: str,
|
|
pages_produced: list[str] | None,
|
|
) -> str:
|
|
"""The manifest read-modify-write half of :func:`update_source`."""
|
|
manifest = _load_raw(vault)
|
|
sources = manifest.get("sources")
|
|
|
|
if isinstance(sources, list):
|
|
target: dict | None = None
|
|
for e in sources:
|
|
if isinstance(e, dict) and _same_source(
|
|
e.get("path") or e.get("source_id"), source_path, vault
|
|
):
|
|
target = e
|
|
break
|
|
if target is None:
|
|
target = {"path": new_key}
|
|
sources.append(target)
|
|
elif target.get("path") != new_key and (
|
|
explicit_key or _rekey_stale_raw_to_archive(target.get("path"), new_key)
|
|
):
|
|
# Explicit --key is authoritative; otherwise only stale
|
|
# ``_raw/<name>.md`` keys move onto ``_raw/_archived/…``.
|
|
target["path"] = new_key
|
|
target["content_hash"] = _format_hash(target.get("content_hash"), current_hash)
|
|
target["last_ingested"] = now
|
|
if pages_produced is not None:
|
|
target["pages_produced"] = pages_produced
|
|
else:
|
|
if not isinstance(sources, dict):
|
|
sources = {}
|
|
match_key: str | None = None
|
|
for existing_key in sources:
|
|
if _same_source(existing_key, source_path, vault):
|
|
match_key = existing_key
|
|
break
|
|
if match_key is not None and match_key != new_key and (
|
|
explicit_key or _rekey_stale_raw_to_archive(match_key, new_key)
|
|
):
|
|
# Explicit key wins; otherwise only stale ``_raw/<name>.md`` keys
|
|
# move onto ``_raw/_archived/…`` so other portable keys stay put.
|
|
moved = sources.pop(match_key)
|
|
existing = sources.get(new_key)
|
|
if isinstance(existing, dict):
|
|
moved = {**existing, **moved}
|
|
sources[new_key] = moved
|
|
match_key = new_key
|
|
# A matched legacy entry otherwise keeps its existing key; only new
|
|
# entries use the portable form, so an in-place update never re-keys a
|
|
# dict by surprise (migrate handles legacy conversion).
|
|
manifest_key = match_key if match_key is not None else new_key
|
|
entry = sources.get(manifest_key) if isinstance(sources.get(manifest_key), dict) else {}
|
|
entry["content_hash"] = _format_hash(entry.get("content_hash"), current_hash)
|
|
entry["last_ingested"] = now
|
|
if pages_produced is not None:
|
|
entry["pages_produced"] = pages_produced
|
|
sources[manifest_key] = entry
|
|
|
|
manifest["sources"] = sources
|
|
_write_manifest(vault, manifest)
|
|
return current_hash
|
|
|
|
|
|
def _rekey_stale_raw_to_archive(old_key: str | None, new_key: str) -> bool:
|
|
if not old_key:
|
|
return False
|
|
old = Path(old_key)
|
|
new = Path(new_key)
|
|
return (
|
|
old.parts[:1] == ("_raw",)
|
|
and "_archived" not in old.parts
|
|
and len(new.parts) >= 3
|
|
and new.parts[0] == "_raw"
|
|
and new.parts[1] == "_archived"
|
|
)
|
|
|
|
|
|
def hash_file(path: Path) -> str:
|
|
"""Just compute and return the hash — no manifest I/O."""
|
|
return compute_hash(path)
|
|
|
|
|
|
# --- project (git) staleness ------------------------------------------------
|
|
# `check_sources` answers "did this source file's bytes change?". It cannot
|
|
# answer the same question for a project, because a git repo is not stored as a
|
|
# hashed source: wiki-update records it under `projects` keyed by `source_repo`
|
|
# with a `last_commit_synced` bookmark. Nothing compared that bookmark to HEAD,
|
|
# so a vault could be arbitrarily far behind a repo with no signal at all.
|
|
|
|
_GIT_TIMEOUT = 15
|
|
|
|
|
|
class ProjectStatus(TypedDict, total=False):
|
|
project: str
|
|
commits: int
|
|
last_commit_synced: str
|
|
path: str
|
|
reason: str
|
|
|
|
|
|
class ProjectCheckResult(TypedDict):
|
|
behind: list[ProjectStatus] # checkout has moved past last_commit_synced
|
|
current: list[str] # last_commit_synced == HEAD
|
|
unreachable: list[ProjectStatus] # stored sha not usable (rebase/force-push/gc)
|
|
unsynced: list[str] # entry exists but no last_commit_synced yet
|
|
unavailable: list[ProjectStatus] # no usable checkout on this machine
|
|
|
|
|
|
def _git_in(path: Path, *args: str) -> subprocess.CompletedProcess | None:
|
|
"""Run git in *path*; None when git itself is unusable (absent, hung)."""
|
|
try:
|
|
return subprocess.run(
|
|
["git", "-C", str(path), *args],
|
|
capture_output=True, text=True, timeout=_GIT_TIMEOUT,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return None
|
|
|
|
|
|
def check_projects(vault: Path) -> ProjectCheckResult:
|
|
"""Classify each manifest ``projects`` entry by how far its checkout has
|
|
moved past the recorded ``last_commit_synced``.
|
|
|
|
Mirrors :func:`check_sources`' vocabulary deliberately: a project whose
|
|
checkout is not on this machine is ``unavailable``, not stale.
|
|
``source_cwd_hint`` is a *hint* — the manifest is portable across machines
|
|
by design, so a missing checkout is an absence of evidence, never a finding.
|
|
|
|
Read-only: never runs a git command that can write, and never fails the
|
|
whole call because one entry is malformed.
|
|
"""
|
|
raw = _load_raw(vault).get("projects")
|
|
projects = raw if isinstance(raw, dict) else {}
|
|
result: ProjectCheckResult = {
|
|
"behind": [], "current": [], "unreachable": [], "unsynced": [], "unavailable": []
|
|
}
|
|
|
|
for name in sorted(projects):
|
|
entry = projects[name]
|
|
if not isinstance(entry, dict):
|
|
result["unavailable"].append({"project": name, "reason": "malformed_entry"})
|
|
continue
|
|
|
|
hint = entry.get("source_cwd_hint")
|
|
if not isinstance(hint, str) or not hint.strip():
|
|
result["unavailable"].append({"project": name, "reason": "no_source_cwd_hint"})
|
|
continue
|
|
path = Path(hint.strip()).expanduser()
|
|
if not path.is_dir():
|
|
result["unavailable"].append(
|
|
{"project": name, "reason": "checkout_not_found", "path": str(path)}
|
|
)
|
|
continue
|
|
|
|
# `rev-parse --git-dir` rather than a `.git` directory test: a linked
|
|
# worktree's `.git` is a file, and the hint may point below the root.
|
|
probe = _git_in(path, "rev-parse", "--git-dir")
|
|
if probe is None:
|
|
result["unavailable"].append(
|
|
{"project": name, "reason": "git_unavailable", "path": str(path)}
|
|
)
|
|
continue
|
|
if probe.returncode != 0:
|
|
result["unavailable"].append(
|
|
{"project": name, "reason": "not_a_git_repo", "path": str(path)}
|
|
)
|
|
continue
|
|
|
|
sha = entry.get("last_commit_synced")
|
|
if not isinstance(sha, str) or not sha.strip():
|
|
result["unsynced"].append(name)
|
|
continue
|
|
sha = sha.strip()
|
|
|
|
ancestor = _git_in(path, "merge-base", "--is-ancestor", sha, "HEAD")
|
|
if ancestor is None:
|
|
result["unavailable"].append(
|
|
{"project": name, "reason": "git_unavailable", "path": str(path)}
|
|
)
|
|
continue
|
|
if ancestor.returncode != 0:
|
|
# 1 = resolves but is not an ancestor (history was rewritten);
|
|
# anything else (128) = the object is not in this repo at all.
|
|
result["unreachable"].append({
|
|
"project": name,
|
|
"reason": "not_ancestor" if ancestor.returncode == 1 else "unknown_commit",
|
|
"last_commit_synced": sha,
|
|
"path": str(path),
|
|
})
|
|
continue
|
|
|
|
counted = _git_in(path, "rev-list", "--count", f"{sha}..HEAD")
|
|
if counted is None or counted.returncode != 0:
|
|
result["unavailable"].append(
|
|
{"project": name, "reason": "count_failed", "path": str(path)}
|
|
)
|
|
continue
|
|
try:
|
|
commits = int(counted.stdout.strip())
|
|
except ValueError:
|
|
result["unavailable"].append(
|
|
{"project": name, "reason": "count_unparsable", "path": str(path)}
|
|
)
|
|
continue
|
|
|
|
if commits == 0:
|
|
result["current"].append(name)
|
|
else:
|
|
result["behind"].append({
|
|
"project": name,
|
|
"commits": commits,
|
|
"last_commit_synced": sha,
|
|
"path": str(path),
|
|
})
|
|
|
|
return result
|