feat(wiki): sync upstream v2026.10.4 core maintenance
Selectively port source hashing, locked manifest/memory writes, path-aware link lint and graph analysis, and revision-checked staging from upstream 4f457e154653756b04802d5aeb25c20f42b4770c. Record source hashes and exclusions. Preserve local archive/catalog citations, curated manifest metadata, config location, file-index and MCP behavior. Wire maintenance guidance into four skills without migrating vault contents or changing running services. Validation: 609 tests passed, including existing MCP/file-asset regressions and imported upstream maintenance tests. Read-only vault status left shared metadata hashes unchanged. Wiki MCP remains inactive; orders MCP is active.
This commit is contained in:
@@ -0,0 +1,765 @@
|
||||
"""Content-hash cache for wiki-ingest source tracking.
|
||||
|
||||
Provides a reliable, platform-independent alternative to running `sha256sum`
|
||||
in the skill. The agent calls `obsidian-wiki cache-check` / `cache-update`
|
||||
instead of shelling out to sha256sum and manually parsing .manifest.json.
|
||||
|
||||
Manifest format (.manifest.json in the vault root). Two `sources` shapes are
|
||||
supported, because real vaults contain both:
|
||||
|
||||
1. Dict keyed by path (this module's original format)::
|
||||
|
||||
{"sources": {"<abs-or-rel-path>": {"content_hash": "...", ...}}}
|
||||
|
||||
2. List of entry objects, each carrying its own ``path`` (the shape the
|
||||
wiki-ingest skill writes)::
|
||||
|
||||
{"sources": [{"path": "_raw/foo.md", "content_hash": "sha256:...", ...}]}
|
||||
|
||||
Both shapes are read transparently, and `update_source` edits the manifest
|
||||
*in place* — preserving its shape, any duplicate-path entries, and
|
||||
skill-written fields (``pages_created``, ``size_bytes``, ``source_type``, …).
|
||||
|
||||
Hashes are compared prefix-insensitively: the skill records
|
||||
``"sha256:<hex>"`` while this module computes a bare ``<hex>``, so an optional
|
||||
``algo:`` prefix is stripped from both sides before comparison.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterator, TypedDict
|
||||
|
||||
|
||||
class SourceEntry(TypedDict, total=False):
|
||||
content_hash: str
|
||||
last_ingested: str
|
||||
pages_produced: list[str]
|
||||
|
||||
|
||||
class CheckResult(TypedDict):
|
||||
new: list[str]
|
||||
modified: list[str]
|
||||
unchanged: list[str]
|
||||
missing: list[str] # vault-local source in manifest but no longer on disk
|
||||
unavailable: list[str] # machine-local source absent here (e.g. synced from another host)
|
||||
|
||||
|
||||
def _manifest_path(vault: Path) -> Path:
|
||||
return vault / ".manifest.json"
|
||||
|
||||
|
||||
def _load_raw(vault: Path) -> dict:
|
||||
"""Return the full manifest object (``{}`` if absent/unreadable)."""
|
||||
mp = _manifest_path(vault)
|
||||
if not mp.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(mp.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def _load_manifest(vault: Path):
|
||||
"""Return the raw ``sources`` value — a dict, a list, or ``{}`` if absent.
|
||||
|
||||
Kept for backward compatibility; callers that need shape-agnostic access
|
||||
should use :func:`_iter_entries`.
|
||||
"""
|
||||
return _load_raw(vault).get("sources", {})
|
||||
|
||||
|
||||
def _lock_path(vault: Path) -> Path:
|
||||
return vault / ".manifest.lock"
|
||||
|
||||
|
||||
class AdvisoryLockTimeout(RuntimeError):
|
||||
"""Raised when an advisory lockfile could not be claimed within the timeout."""
|
||||
|
||||
|
||||
class ManifestLockTimeout(AdvisoryLockTimeout):
|
||||
"""Raised when the manifest lock could not be acquired within the timeout."""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def advisory_lock(
|
||||
lock: Path,
|
||||
*,
|
||||
timeout: float = 10.0,
|
||||
stale_after: float = 60.0,
|
||||
error_cls: type = AdvisoryLockTimeout,
|
||||
):
|
||||
"""Claim *lock* as an advisory lockfile for a read-modify-write.
|
||||
|
||||
``O_CREAT | O_EXCL`` is the portable stdlib primitive here — ``fcntl`` is
|
||||
POSIX-only and this ships on Windows. A lockfile older than *stale_after*
|
||||
is assumed to belong to a crashed process and is stolen.
|
||||
|
||||
Shared by the manifest writer and the prose-memory writer
|
||||
(:mod:`obsidian_wiki.memory`) so both serialise the same way.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
fd = None
|
||||
while True:
|
||||
try:
|
||||
fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
break
|
||||
except FileExistsError:
|
||||
try:
|
||||
age = time.time() - lock.stat().st_mtime
|
||||
except FileNotFoundError:
|
||||
continue # holder released it between our open and stat
|
||||
if age > stale_after:
|
||||
# ponytail: last-writer-wins steal; a pid check would narrow the
|
||||
# window if two processes ever race to steal the same stale lock
|
||||
try:
|
||||
lock.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
continue
|
||||
if time.monotonic() >= deadline:
|
||||
raise error_cls(
|
||||
f"could not acquire {lock} within {timeout}s "
|
||||
f"(held for {age:.1f}s; stale after {stale_after}s)"
|
||||
)
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
os.write(fd, str(os.getpid()).encode())
|
||||
os.close(fd)
|
||||
fd = None
|
||||
yield
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
try:
|
||||
lock.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def manifest_lock(vault: Path, *, timeout: float = 10.0, stale_after: float = 60.0):
|
||||
"""Advisory lock around a manifest read-modify-write.
|
||||
|
||||
Parallel ingest agents (``batch-plan`` fan-out) and the Docker server can
|
||||
all write one vault's manifest; without this the read-modify-write races
|
||||
and a whole entry is silently lost.
|
||||
"""
|
||||
# ponytail: whole-manifest advisory lock; per-entry locking if batch fan-out ever contends
|
||||
with advisory_lock(
|
||||
_lock_path(vault),
|
||||
timeout=timeout,
|
||||
stale_after=stale_after,
|
||||
error_cls=ManifestLockTimeout,
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _write_manifest(vault: Path, manifest: dict) -> None:
|
||||
"""Atomically replace the manifest file — no torn reads, no partial file."""
|
||||
target = _manifest_path(vault)
|
||||
tmp = target.with_name(f".manifest.json.{os.getpid()}.tmp")
|
||||
try:
|
||||
tmp.write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
os.replace(tmp, target) # atomic on POSIX and Windows
|
||||
except BaseException:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _save_manifest(vault: Path, sources) -> None:
|
||||
"""Write *sources* back into the manifest, preserving other top-level keys."""
|
||||
with manifest_lock(vault):
|
||||
manifest = _load_raw(vault)
|
||||
manifest["sources"] = sources
|
||||
_write_manifest(vault, manifest)
|
||||
|
||||
|
||||
def _iter_entries(sources) -> Iterator[tuple[str | None, dict]]:
|
||||
"""Yield ``(stored_key, entry)`` pairs for either manifest shape.
|
||||
|
||||
For the dict shape the key is the dict key; for the list shape it is the
|
||||
entry's ``path`` (falling back to ``source_id``).
|
||||
"""
|
||||
if isinstance(sources, dict):
|
||||
for key, entry in sources.items():
|
||||
if isinstance(entry, dict):
|
||||
yield key, entry
|
||||
elif isinstance(sources, list):
|
||||
for entry in sources:
|
||||
if isinstance(entry, dict):
|
||||
yield (entry.get("path") or entry.get("source_id")), entry
|
||||
|
||||
|
||||
# Matches "sha256:", "https://", "slack:#..." — anything with an algo/scheme
|
||||
# prefix that shouldn't be treated as a filesystem path.
|
||||
_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:[^\\/]")
|
||||
|
||||
# A Windows drive-absolute path ("C:\dir\file", "C:/dir/file"). On POSIX this is
|
||||
# not a path at all, so it cannot be resolved and is machine-specific; lint.py
|
||||
# recognises the same shape when reporting stored keys. Kept local rather than
|
||||
# imported so the runtime cache carries no dependency on the lint module.
|
||||
_WINDOWS_ABS_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
def _strip_algo(value: str | None) -> str:
|
||||
"""Drop an optional ``algo:`` prefix so ``sha256:<hex>`` == ``<hex>``."""
|
||||
value = value or ""
|
||||
return value.split(":", 1)[1] if ":" in value else value
|
||||
|
||||
|
||||
def _format_hash(existing: str | None, new_hex: str) -> str:
|
||||
"""Format *new_hex* keeping any ``algo:`` prefix the existing value used."""
|
||||
if existing and ":" in existing:
|
||||
return f"{existing.split(':', 1)[0]}:{new_hex}"
|
||||
return new_hex
|
||||
|
||||
|
||||
def _is_file_key(key: str | None) -> bool:
|
||||
"""True if *key* looks like a filesystem path rather than a URL/pseudo-key."""
|
||||
return bool(key) and "://" not in key and not _SCHEME_RE.match(key)
|
||||
|
||||
|
||||
def _expand_key(key: str) -> str:
|
||||
"""Expand ``~`` and environment variables, but only when the key uses them.
|
||||
|
||||
A plain vault-relative key like ``Raw/x.pdf`` must survive untouched, so the
|
||||
expansion is gated rather than applied unconditionally.
|
||||
"""
|
||||
if key.startswith("~") or "$" in key:
|
||||
return os.path.expandvars(os.path.expanduser(key))
|
||||
return key
|
||||
|
||||
|
||||
def resolve_key(key: str | None, vault: Path) -> Path | None:
|
||||
"""Normalize a manifest key to an absolute path, or ``None`` for non-file keys.
|
||||
|
||||
Resolution order (the source key contract v2, see ``llm-wiki/SKILL.md``):
|
||||
|
||||
1. pseudo-key (``scheme:`` / ``://``) -> ``None`` — an opaque identifier, not
|
||||
a filesystem location;
|
||||
2. ``~``- or env-var-bearing key -> expand;
|
||||
3. absolute path -> use as-is (legacy compatibility);
|
||||
4. anything else -> resolve against the vault root.
|
||||
"""
|
||||
if not _is_file_key(key):
|
||||
return None
|
||||
path = Path(_expand_key(key))
|
||||
return path if path.is_absolute() else (vault / path)
|
||||
|
||||
|
||||
def stored_key(path: Path, vault: Path) -> str | None:
|
||||
"""Return the portable manifest key for *path*, or ``None`` if not portable.
|
||||
|
||||
In-vault sources become vault-relative (``Raw/x.pdf``); sources under
|
||||
``$HOME`` become home-relative (``~/.claude/...``); anything else has no
|
||||
machine-portable representation and needs an explicit pseudo-key from the
|
||||
caller (``repo:``, ``url:``, ``agent:``).
|
||||
"""
|
||||
p = Path(os.path.abspath(_expand_key(str(path))))
|
||||
vault_root = Path(os.path.abspath(_expand_key(str(vault))))
|
||||
home_root = Path(os.path.expanduser("~"))
|
||||
for root, prefix in ((vault_root, ""), (home_root, "~/")):
|
||||
try:
|
||||
rel = p.relative_to(root)
|
||||
except ValueError:
|
||||
continue
|
||||
if rel == Path("."):
|
||||
return None
|
||||
return prefix + rel.as_posix()
|
||||
return None
|
||||
|
||||
|
||||
def _same_source(stored_key_value: str | None, query: Path, vault: Path) -> bool:
|
||||
"""True if a manifest key refers to the same source as *query*.
|
||||
|
||||
Matches on the raw string first (covers pseudo-keys), then on the resolved
|
||||
absolute form so an absolute query matches a vault-relative or home-relative
|
||||
stored key.
|
||||
"""
|
||||
if not stored_key_value:
|
||||
return False
|
||||
if str(query) == stored_key_value:
|
||||
return True
|
||||
stored_path = resolve_key(stored_key_value, vault)
|
||||
if stored_path is None:
|
||||
return False
|
||||
try:
|
||||
return stored_path.resolve() == query.resolve()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _missing_on_disk(key: str | None, vault: Path) -> bool:
|
||||
"""True if a filesystem-style manifest key has no file on disk."""
|
||||
path = resolve_key(key, vault)
|
||||
if path is None:
|
||||
return False
|
||||
return not path.exists()
|
||||
|
||||
|
||||
def _vault_top_names(vault: Path) -> set[str]:
|
||||
"""Names of the vault's top-level entries, or an empty set if unreadable."""
|
||||
try:
|
||||
return {p.name for p in vault.iterdir()}
|
||||
except OSError:
|
||||
return set()
|
||||
|
||||
|
||||
def _is_vault_local(key: str | None, vault: Path, top_names: set[str] | None = None) -> bool:
|
||||
"""True if a file key names a source that travels with the vault.
|
||||
|
||||
Vault-local sources travel with the vault, so their absence is a real loss
|
||||
(``missing``). A key that is machine-specific *and* points outside this vault
|
||||
(``/home/other/wiki/x.md``, ``~/.claude/...``) may simply not exist on the
|
||||
machine reading a synced vault, so it is reported under ``unavailable``. An
|
||||
absolute or ``~``-relative path that resolves *inside* this vault is still
|
||||
vault-local — machine-specific in form, but not in target.
|
||||
|
||||
A *relative* key resolves lexically inside the vault even when it is really
|
||||
relative to some other root — the legacy ingest-root keys such as
|
||||
``-Users-x-github/abc.jsonl``. Path shape cannot tell those apart from vault
|
||||
keys, so the vault's own topology decides: the first segment must name a real
|
||||
top-level entry. That also gives the honest answer for an out-of-vault source
|
||||
parked in a namespace the vault does not contain (``external/.hermes/...``):
|
||||
unavailable, not missing. When the vault cannot be listed the lexical answer
|
||||
stands, so an I/O hiccup never hides a real loss.
|
||||
|
||||
A key written for another OS — a drive-letter path or one using backslashes
|
||||
as separators — cannot be resolved on this host, so it is machine-specific
|
||||
(``unavailable``) rather than a vault-relative key.
|
||||
"""
|
||||
path = resolve_key(key, vault)
|
||||
if path is None:
|
||||
return False
|
||||
raw = str(key)
|
||||
try:
|
||||
rel = Path(os.path.abspath(str(path))).relative_to(Path(os.path.abspath(str(vault))))
|
||||
except ValueError:
|
||||
return False
|
||||
if rel == Path("."):
|
||||
return False
|
||||
if os.name != "nt" and ("\\" in raw or _WINDOWS_ABS_RE.match(raw)):
|
||||
# A key written for another OS: a backslash is not a separator here and a
|
||||
# drive-letter path cannot be resolved, so this is machine-specific — not
|
||||
# a vault source whose file went away.
|
||||
return False
|
||||
if os.path.isabs(raw) or raw.startswith("~") or "$" in raw:
|
||||
return True
|
||||
if "/" not in raw and "\\" not in raw:
|
||||
# A bare filename at the vault root. There is no leading component that
|
||||
# could be a foreign root, so this is a vault source by construction —
|
||||
# and if it is gone, that is a real vault-local loss. The backslash guard
|
||||
# keeps a Windows-form key out of this shortcut on every platform.
|
||||
return True
|
||||
if top_names is None:
|
||||
top_names = _vault_top_names(vault)
|
||||
if not top_names:
|
||||
return True
|
||||
return rel.parts[0] in top_names
|
||||
|
||||
|
||||
def sha256_file(path: Path, chunk: int = 1 << 20) -> str:
|
||||
"""Return the hex SHA-256 digest of *path* without loading it all into RAM."""
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
while True:
|
||||
block = fh.read(chunk)
|
||||
if not block:
|
||||
break
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def sha256_dir(path: Path) -> str:
|
||||
"""Stable SHA-256 over all files in a directory tree (sorted by relative path)."""
|
||||
h = hashlib.sha256()
|
||||
for fp in sorted(path.rglob("*"), key=lambda p: p.relative_to(path).as_posix()):
|
||||
if fp.is_file():
|
||||
rel = fp.relative_to(path).as_posix()
|
||||
h.update(rel.encode())
|
||||
h.update(sha256_file(fp).encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def compute_hash(path: Path) -> str:
|
||||
if path.is_dir():
|
||||
return sha256_dir(path)
|
||||
return sha256_file(path)
|
||||
|
||||
|
||||
def _resolved_form(path: Path) -> str | None:
|
||||
"""``realpath`` of *path* as a comparable string, or ``None`` if it cannot be resolved.
|
||||
|
||||
``os.path.normcase`` keeps the comparison equivalent to ``Path`` equality on
|
||||
case-insensitive filesystems.
|
||||
"""
|
||||
try:
|
||||
return os.path.normcase(str(path.resolve()))
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def check_sources(vault: Path, source_paths: list[Path]) -> CheckResult:
|
||||
"""Classify each source as new / modified / unchanged vs. the manifest.
|
||||
|
||||
Also reports manifest entries whose source file no longer exists on disk.
|
||||
Handles both manifest shapes and compares hashes prefix-insensitively.
|
||||
|
||||
Every entry and every query is resolved once and matched through an index
|
||||
of its forms (raw key, resolved path). Resolving is a ``realpath`` walk —
|
||||
one ``lstat`` per path component — so doing it per (entry, query) pair, as
|
||||
the first-match scan did, made a 3,000-entry manifest cost about a second
|
||||
per queried file. The index keeps the first entry for each form, which is
|
||||
what the scan's ``break`` on first hit returned.
|
||||
"""
|
||||
entries = list(_iter_entries(_load_manifest(vault)))
|
||||
result: CheckResult = {
|
||||
"new": [], "modified": [], "unchanged": [], "missing": [], "unavailable": []
|
||||
}
|
||||
|
||||
by_form: dict[str, int] = {}
|
||||
entry_resolved: list[str | None] = []
|
||||
for i, (stored_key, _entry) in enumerate(entries):
|
||||
resolved = None
|
||||
if stored_key:
|
||||
by_form.setdefault(stored_key, i)
|
||||
stored_path = resolve_key(stored_key, vault)
|
||||
if stored_path is not None:
|
||||
resolved = _resolved_form(stored_path)
|
||||
if resolved is not None:
|
||||
by_form.setdefault(resolved, i)
|
||||
entry_resolved.append(resolved)
|
||||
|
||||
matched: set[int] = set()
|
||||
query_forms: set[str] = set()
|
||||
for path in source_paths:
|
||||
key = str(path)
|
||||
forms = [key]
|
||||
resolved = _resolved_form(path)
|
||||
if resolved is not None:
|
||||
forms.append(resolved)
|
||||
query_forms.update(forms)
|
||||
if not path.exists():
|
||||
result["missing"].append(key)
|
||||
continue
|
||||
current_hash = _strip_algo(compute_hash(path))
|
||||
hits = [by_form[f] for f in forms if f in by_form]
|
||||
if not hits:
|
||||
result["new"].append(key)
|
||||
continue
|
||||
i = min(hits) # the earliest entry matching by either form — the old scan's first hit
|
||||
matched.add(i)
|
||||
entry = entries[i][1]
|
||||
if _strip_algo(entry.get("content_hash")) != current_hash:
|
||||
result["modified"].append(key)
|
||||
else:
|
||||
result["unchanged"].append(key)
|
||||
|
||||
# Report manifest entries whose source file no longer exists on disk and
|
||||
# that weren't among the scanned paths (in any path form).
|
||||
top_names = _vault_top_names(vault)
|
||||
for i, (stored_key, _entry) in enumerate(entries):
|
||||
if i in matched:
|
||||
continue
|
||||
if stored_key in query_forms or (entry_resolved[i] is not None and entry_resolved[i] in query_forms):
|
||||
continue
|
||||
if _missing_on_disk(stored_key, vault):
|
||||
bucket = "missing" if _is_vault_local(stored_key, vault, top_names) else "unavailable"
|
||||
result[bucket].append(stored_key)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def update_source(
|
||||
vault: Path,
|
||||
source_path: Path,
|
||||
*,
|
||||
pages_produced: list[str] | None = None,
|
||||
key: str | None = None,
|
||||
) -> str:
|
||||
"""Record the current hash of *source_path* in the manifest. Returns the hash.
|
||||
|
||||
Edits the manifest in place: matches an existing entry across path forms and
|
||||
updates it, otherwise appends a new one — preserving the manifest's shape
|
||||
(dict or list), duplicate-path entries, and any skill-written fields.
|
||||
|
||||
The stored key is normalized to the portable form (vault-relative for
|
||||
in-vault sources, ``~``-relative under ``$HOME``) so a synced vault never
|
||||
accumulates machine absolute paths. Pass *key* explicitly for sources that
|
||||
have no filesystem representation in either form — a pseudo-key such as
|
||||
``repo:github.com/owner/name``, ``url:https://...``, or ``agent:claude/<id>``.
|
||||
When neither applies, the raw path is kept for backward compatibility and a
|
||||
warning is written to stderr, because that entry is not portable across
|
||||
machines.
|
||||
"""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
explicit_key = key is not None
|
||||
hash_path = source_path
|
||||
if explicit_key:
|
||||
new_key = key
|
||||
else:
|
||||
derived = stored_key(source_path, vault)
|
||||
if derived is None:
|
||||
new_key = str(source_path)
|
||||
print(
|
||||
f"warning: {source_path} is outside the vault and $HOME, so it has no "
|
||||
f"portable key; storing the absolute path. Pass key=<repo:|url:|agent:> "
|
||||
f"to keep the vault portable across machines.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
from obsidian_wiki.provenance import prefer_archive_write_path
|
||||
preferred = prefer_archive_write_path(vault, derived)
|
||||
if preferred is not None:
|
||||
new_key = preferred
|
||||
hash_path = vault / preferred
|
||||
else:
|
||||
new_key = derived
|
||||
# Hash outside the lock — hashing a large source tree can take seconds and
|
||||
# nothing else in the manifest depends on it.
|
||||
current_hash = compute_hash(hash_path)
|
||||
|
||||
with manifest_lock(vault):
|
||||
return _update_source_locked(
|
||||
vault, source_path, new_key, explicit_key, current_hash, now, pages_produced,
|
||||
)
|
||||
|
||||
|
||||
def _update_source_locked(
|
||||
vault: Path,
|
||||
source_path: Path,
|
||||
new_key: str,
|
||||
explicit_key: bool,
|
||||
current_hash: str,
|
||||
now: str,
|
||||
pages_produced: list[str] | None,
|
||||
) -> str:
|
||||
"""The manifest read-modify-write half of :func:`update_source`."""
|
||||
manifest = _load_raw(vault)
|
||||
sources = manifest.get("sources")
|
||||
|
||||
if isinstance(sources, list):
|
||||
target: dict | None = None
|
||||
for e in sources:
|
||||
if isinstance(e, dict) and _same_source(
|
||||
e.get("path") or e.get("source_id"), source_path, vault
|
||||
):
|
||||
target = e
|
||||
break
|
||||
if target is None:
|
||||
target = {"path": new_key}
|
||||
sources.append(target)
|
||||
elif target.get("path") != new_key and (
|
||||
explicit_key or _rekey_stale_raw_to_archive(target.get("path"), new_key)
|
||||
):
|
||||
# Explicit --key is authoritative; otherwise only stale
|
||||
# ``_raw/<name>.md`` keys move onto ``_raw/_archived/…``.
|
||||
target["path"] = new_key
|
||||
target["content_hash"] = _format_hash(target.get("content_hash"), current_hash)
|
||||
target["last_ingested"] = now
|
||||
if pages_produced is not None:
|
||||
target["pages_produced"] = pages_produced
|
||||
else:
|
||||
if not isinstance(sources, dict):
|
||||
sources = {}
|
||||
match_key: str | None = None
|
||||
for existing_key in sources:
|
||||
if _same_source(existing_key, source_path, vault):
|
||||
match_key = existing_key
|
||||
break
|
||||
if match_key is not None and match_key != new_key and (
|
||||
explicit_key or _rekey_stale_raw_to_archive(match_key, new_key)
|
||||
):
|
||||
# Explicit key wins; otherwise only stale ``_raw/<name>.md`` keys
|
||||
# move onto ``_raw/_archived/…`` so other portable keys stay put.
|
||||
moved = sources.pop(match_key)
|
||||
existing = sources.get(new_key)
|
||||
if isinstance(existing, dict):
|
||||
moved = {**existing, **moved}
|
||||
sources[new_key] = moved
|
||||
match_key = new_key
|
||||
# A matched legacy entry otherwise keeps its existing key; only new
|
||||
# entries use the portable form, so an in-place update never re-keys a
|
||||
# dict by surprise (migrate handles legacy conversion).
|
||||
manifest_key = match_key if match_key is not None else new_key
|
||||
entry = sources.get(manifest_key) if isinstance(sources.get(manifest_key), dict) else {}
|
||||
entry["content_hash"] = _format_hash(entry.get("content_hash"), current_hash)
|
||||
entry["last_ingested"] = now
|
||||
if pages_produced is not None:
|
||||
entry["pages_produced"] = pages_produced
|
||||
sources[manifest_key] = entry
|
||||
|
||||
manifest["sources"] = sources
|
||||
_write_manifest(vault, manifest)
|
||||
return current_hash
|
||||
|
||||
|
||||
def _rekey_stale_raw_to_archive(old_key: str | None, new_key: str) -> bool:
|
||||
if not old_key:
|
||||
return False
|
||||
old = Path(old_key)
|
||||
new = Path(new_key)
|
||||
return (
|
||||
old.parts[:1] == ("_raw",)
|
||||
and "_archived" not in old.parts
|
||||
and len(new.parts) >= 3
|
||||
and new.parts[0] == "_raw"
|
||||
and new.parts[1] == "_archived"
|
||||
)
|
||||
|
||||
|
||||
def hash_file(path: Path) -> str:
|
||||
"""Just compute and return the hash — no manifest I/O."""
|
||||
return compute_hash(path)
|
||||
|
||||
|
||||
# --- project (git) staleness ------------------------------------------------
|
||||
# `check_sources` answers "did this source file's bytes change?". It cannot
|
||||
# answer the same question for a project, because a git repo is not stored as a
|
||||
# hashed source: wiki-update records it under `projects` keyed by `source_repo`
|
||||
# with a `last_commit_synced` bookmark. Nothing compared that bookmark to HEAD,
|
||||
# so a vault could be arbitrarily far behind a repo with no signal at all.
|
||||
|
||||
_GIT_TIMEOUT = 15
|
||||
|
||||
|
||||
class ProjectStatus(TypedDict, total=False):
|
||||
project: str
|
||||
commits: int
|
||||
last_commit_synced: str
|
||||
path: str
|
||||
reason: str
|
||||
|
||||
|
||||
class ProjectCheckResult(TypedDict):
|
||||
behind: list[ProjectStatus] # checkout has moved past last_commit_synced
|
||||
current: list[str] # last_commit_synced == HEAD
|
||||
unreachable: list[ProjectStatus] # stored sha not usable (rebase/force-push/gc)
|
||||
unsynced: list[str] # entry exists but no last_commit_synced yet
|
||||
unavailable: list[ProjectStatus] # no usable checkout on this machine
|
||||
|
||||
|
||||
def _git_in(path: Path, *args: str) -> subprocess.CompletedProcess | None:
|
||||
"""Run git in *path*; None when git itself is unusable (absent, hung)."""
|
||||
try:
|
||||
return subprocess.run(
|
||||
["git", "-C", str(path), *args],
|
||||
capture_output=True, text=True, timeout=_GIT_TIMEOUT,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
|
||||
|
||||
def check_projects(vault: Path) -> ProjectCheckResult:
|
||||
"""Classify each manifest ``projects`` entry by how far its checkout has
|
||||
moved past the recorded ``last_commit_synced``.
|
||||
|
||||
Mirrors :func:`check_sources`' vocabulary deliberately: a project whose
|
||||
checkout is not on this machine is ``unavailable``, not stale.
|
||||
``source_cwd_hint`` is a *hint* — the manifest is portable across machines
|
||||
by design, so a missing checkout is an absence of evidence, never a finding.
|
||||
|
||||
Read-only: never runs a git command that can write, and never fails the
|
||||
whole call because one entry is malformed.
|
||||
"""
|
||||
raw = _load_raw(vault).get("projects")
|
||||
projects = raw if isinstance(raw, dict) else {}
|
||||
result: ProjectCheckResult = {
|
||||
"behind": [], "current": [], "unreachable": [], "unsynced": [], "unavailable": []
|
||||
}
|
||||
|
||||
for name in sorted(projects):
|
||||
entry = projects[name]
|
||||
if not isinstance(entry, dict):
|
||||
result["unavailable"].append({"project": name, "reason": "malformed_entry"})
|
||||
continue
|
||||
|
||||
hint = entry.get("source_cwd_hint")
|
||||
if not isinstance(hint, str) or not hint.strip():
|
||||
result["unavailable"].append({"project": name, "reason": "no_source_cwd_hint"})
|
||||
continue
|
||||
path = Path(hint.strip()).expanduser()
|
||||
if not path.is_dir():
|
||||
result["unavailable"].append(
|
||||
{"project": name, "reason": "checkout_not_found", "path": str(path)}
|
||||
)
|
||||
continue
|
||||
|
||||
# `rev-parse --git-dir` rather than a `.git` directory test: a linked
|
||||
# worktree's `.git` is a file, and the hint may point below the root.
|
||||
probe = _git_in(path, "rev-parse", "--git-dir")
|
||||
if probe is None:
|
||||
result["unavailable"].append(
|
||||
{"project": name, "reason": "git_unavailable", "path": str(path)}
|
||||
)
|
||||
continue
|
||||
if probe.returncode != 0:
|
||||
result["unavailable"].append(
|
||||
{"project": name, "reason": "not_a_git_repo", "path": str(path)}
|
||||
)
|
||||
continue
|
||||
|
||||
sha = entry.get("last_commit_synced")
|
||||
if not isinstance(sha, str) or not sha.strip():
|
||||
result["unsynced"].append(name)
|
||||
continue
|
||||
sha = sha.strip()
|
||||
|
||||
ancestor = _git_in(path, "merge-base", "--is-ancestor", sha, "HEAD")
|
||||
if ancestor is None:
|
||||
result["unavailable"].append(
|
||||
{"project": name, "reason": "git_unavailable", "path": str(path)}
|
||||
)
|
||||
continue
|
||||
if ancestor.returncode != 0:
|
||||
# 1 = resolves but is not an ancestor (history was rewritten);
|
||||
# anything else (128) = the object is not in this repo at all.
|
||||
result["unreachable"].append({
|
||||
"project": name,
|
||||
"reason": "not_ancestor" if ancestor.returncode == 1 else "unknown_commit",
|
||||
"last_commit_synced": sha,
|
||||
"path": str(path),
|
||||
})
|
||||
continue
|
||||
|
||||
counted = _git_in(path, "rev-list", "--count", f"{sha}..HEAD")
|
||||
if counted is None or counted.returncode != 0:
|
||||
result["unavailable"].append(
|
||||
{"project": name, "reason": "count_failed", "path": str(path)}
|
||||
)
|
||||
continue
|
||||
try:
|
||||
commits = int(counted.stdout.strip())
|
||||
except ValueError:
|
||||
result["unavailable"].append(
|
||||
{"project": name, "reason": "count_unparsable", "path": str(path)}
|
||||
)
|
||||
continue
|
||||
|
||||
if commits == 0:
|
||||
result["current"].append(name)
|
||||
else:
|
||||
result["behind"].append({
|
||||
"project": name,
|
||||
"commits": commits,
|
||||
"last_commit_synced": sha,
|
||||
"path": str(path),
|
||||
})
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user