Codex
2183404966
Expose authoritative ingest phases and supervisor-backed process liveness without semantic activity inference. Require deterministic registry, manifest, archive, file-index, and declared-output verification before an agent-successful job can finish succeeded.\n\nPreserve legacy state values, derive defaults for older job records, keep verification outside the manager lock, and retain partial outputs on verification failure.\n\nValidated with the full unittest suite (205 tests).
237 lines
7.5 KiB
Python
237 lines
7.5 KiB
Python
"""Deterministic post-run verification for remote MCP ingest jobs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from obsidian_wiki.file_assets import db_path_for_vault, registry_path_for_vault
|
|
|
|
|
|
SHA256_RE = re.compile(r"^(?:sha256:)?([0-9a-fA-F]{64})$")
|
|
DECLARED_PATH_FIELDS = (
|
|
"related_pages",
|
|
"catalog_paths",
|
|
"pages_created",
|
|
"pages_updated",
|
|
)
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def verify_ingest_outputs(
|
|
*,
|
|
vault_path: str | Path,
|
|
uploads: list[dict[str, Any]],
|
|
job_started_at: str | None,
|
|
archive_sources: bool,
|
|
) -> dict[str, Any]:
|
|
"""Verify durable ingest artifacts without trusting agent-reported success."""
|
|
|
|
vault = Path(vault_path).expanduser().resolve()
|
|
registry_records, registry_warnings = _load_registry(vault)
|
|
manifest_sources, manifest_warnings = _load_manifest_sources(vault)
|
|
failures: list[dict[str, str]] = []
|
|
warnings = [*registry_warnings, *manifest_warnings]
|
|
outputs = {
|
|
"source_count": len(uploads),
|
|
"registry_count": 0,
|
|
"archive_count": 0,
|
|
"manifest_count": 0,
|
|
"indexed_count": 0,
|
|
}
|
|
|
|
indexed_digests, index_warning = _load_indexed_digests(vault)
|
|
if index_warning:
|
|
warnings.append(index_warning)
|
|
|
|
for upload in uploads:
|
|
upload_id = str(upload.get("upload_id") or "")
|
|
digest = _normalize_digest(upload.get("sha256"))
|
|
if digest is None:
|
|
failures.append({"code": "invalid_upload_sha256", "upload_id": upload_id})
|
|
continue
|
|
source_id = f"sha256:{digest}"
|
|
|
|
registry_record = next(
|
|
(
|
|
record
|
|
for record in registry_records
|
|
if str(record.get("source_id") or "").lower() == source_id
|
|
),
|
|
None,
|
|
)
|
|
if registry_record is None:
|
|
failures.append({"code": "registry_missing", "upload_id": upload_id})
|
|
else:
|
|
outputs["registry_count"] += 1
|
|
|
|
if archive_sources:
|
|
archive_path = str((registry_record or {}).get("archive_path") or "")
|
|
if not archive_path or not _vault_path_exists(vault, archive_path):
|
|
failures.append({"code": "archive_missing", "upload_id": upload_id})
|
|
else:
|
|
outputs["archive_count"] += 1
|
|
|
|
upload_path = str(upload.get("path") or "")
|
|
manifest_record = _matching_manifest_record(
|
|
manifest_sources,
|
|
upload_path=upload_path,
|
|
source_id=source_id,
|
|
job_started_at=job_started_at,
|
|
)
|
|
if manifest_record is None:
|
|
failures.append({"code": "manifest_missing", "upload_id": upload_id})
|
|
else:
|
|
outputs["manifest_count"] += 1
|
|
|
|
if digest in indexed_digests:
|
|
outputs["indexed_count"] += 1
|
|
else:
|
|
failures.append({"code": "index_missing", "upload_id": upload_id})
|
|
|
|
declared_paths = _declared_paths(registry_record, manifest_record)
|
|
for declared_path in declared_paths:
|
|
if not _vault_path_exists(vault, declared_path):
|
|
failures.append({"code": "declared_path_missing", "upload_id": upload_id})
|
|
break
|
|
|
|
return {
|
|
"status": "passed" if not failures else "failed",
|
|
"checked_at": utc_now(),
|
|
"failures": failures,
|
|
"warnings": warnings,
|
|
"outputs": outputs,
|
|
}
|
|
|
|
|
|
def _normalize_digest(value: Any) -> str | None:
|
|
match = SHA256_RE.fullmatch(str(value or "").strip())
|
|
return match.group(1).lower() if match else None
|
|
|
|
|
|
def _load_registry(vault: Path) -> tuple[list[dict[str, Any]], list[str]]:
|
|
path = registry_path_for_vault(vault)
|
|
if not path.is_file():
|
|
return [], []
|
|
records: list[dict[str, Any]] = []
|
|
malformed = 0
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
malformed += 1
|
|
continue
|
|
if isinstance(record, dict):
|
|
records.append(record)
|
|
warnings = ["source registry contains malformed records"] if malformed else []
|
|
return records, warnings
|
|
|
|
|
|
def _load_manifest_sources(vault: Path) -> tuple[dict[str, Any], list[str]]:
|
|
path = vault / ".manifest.json"
|
|
if not path.is_file():
|
|
return {}, []
|
|
try:
|
|
manifest = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return {}, ["manifest could not be parsed"]
|
|
sources = manifest.get("sources") if isinstance(manifest, dict) else None
|
|
return (sources if isinstance(sources, dict) else {}), []
|
|
|
|
|
|
def _load_indexed_digests(vault: Path) -> tuple[set[str], str | None]:
|
|
db_path = db_path_for_vault(vault)
|
|
if not db_path.is_file():
|
|
return set(), None
|
|
try:
|
|
with sqlite3.connect(str(db_path)) as conn:
|
|
rows = conn.execute("SELECT sha256 FROM file_assets").fetchall()
|
|
except sqlite3.Error:
|
|
return set(), "file asset index could not be queried"
|
|
return {
|
|
digest
|
|
for row in rows
|
|
if row and (digest := _normalize_digest(row[0])) is not None
|
|
}, None
|
|
|
|
|
|
def _matching_manifest_record(
|
|
sources: dict[str, Any],
|
|
*,
|
|
upload_path: str,
|
|
source_id: str,
|
|
job_started_at: str | None,
|
|
) -> dict[str, Any] | None:
|
|
for source_path, value in sources.items():
|
|
if not isinstance(value, dict):
|
|
continue
|
|
original_path = str(value.get("original_path") or "")
|
|
if str(source_path) != upload_path and original_path != upload_path:
|
|
continue
|
|
hashes = {
|
|
str(value.get("content_hash") or "").lower(),
|
|
str(value.get("source_id") or "").lower(),
|
|
}
|
|
if source_id not in hashes:
|
|
continue
|
|
if not _is_fresh(value.get("ingested_at"), job_started_at):
|
|
continue
|
|
return value
|
|
return None
|
|
|
|
|
|
def _is_fresh(ingested_at: Any, job_started_at: str | None) -> bool:
|
|
if not job_started_at:
|
|
return bool(ingested_at)
|
|
ingested = _parse_timestamp(ingested_at)
|
|
started = _parse_timestamp(job_started_at)
|
|
return ingested is not None and started is not None and ingested >= started
|
|
|
|
|
|
def _parse_timestamp(value: Any) -> datetime | None:
|
|
text = str(value or "").strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
|
|
def _declared_paths(*records: dict[str, Any] | None) -> set[str]:
|
|
paths: set[str] = set()
|
|
for record in records:
|
|
if not isinstance(record, dict):
|
|
continue
|
|
for field in DECLARED_PATH_FIELDS:
|
|
value = record.get(field)
|
|
if isinstance(value, str):
|
|
paths.add(value)
|
|
elif isinstance(value, list):
|
|
paths.update(str(item) for item in value if str(item).strip())
|
|
return paths
|
|
|
|
|
|
def _vault_path_exists(vault: Path, relative_path: str) -> bool:
|
|
candidate = Path(relative_path)
|
|
if candidate.is_absolute():
|
|
return False
|
|
resolved = (vault / candidate).resolve()
|
|
try:
|
|
resolved.relative_to(vault)
|
|
except ValueError:
|
|
return False
|
|
return resolved.exists()
|