feat(mcp): add verified ingest job phases

Expose authoritative ingest phases and supervisor-backed process liveness without semantic activity inference. Require deterministic registry, manifest, archive, file-index, and declared-output verification before an agent-successful job can finish succeeded.\n\nPreserve legacy state values, derive defaults for older job records, keep verification outside the manager lock, and retain partial outputs on verification failure.\n\nValidated with the full unittest suite (205 tests).
This commit is contained in:
Codex
2026-07-16 14:26:58 +00:00
parent a2b4bc393b
commit 2183404966
8 changed files with 888 additions and 10 deletions
+149 -6
View File
@@ -21,6 +21,7 @@ from threading import RLock
from typing import Any
from obsidian_wiki.file_assets import clean_config_value, db_path_for_vault, registry_path_for_vault
from obsidian_wiki.mcp_ingest_verification import verify_ingest_outputs
from obsidian_wiki.mcp_jobs import ProcessOutcome, ProcessSupervisor, extract_final_answer, parse_marked_result
DEFAULT_INGEST_MAX_FILE_BYTES = 100 * 1024 * 1024
@@ -533,16 +534,20 @@ class IngestManager:
job_id = uuid.uuid4().hex
job_dir = self.jobs_root / job_id
job_dir.mkdir(parents=True, exist_ok=False)
created_at = utc_now()
job = {
"job_id": job_id,
"state": "queued",
"phase": "queued",
"phase_started_at": created_at,
"verification": {"status": "not_run"},
"upload_ids": upload_ids,
"bundle": bundle,
"files": [self._public_upload(upload, expose_path=True) for upload in uploads],
"note": self._safe_optional_text(note),
"queue_sequence": time.time_ns(),
"created_at": utc_now(),
"updated_at": utc_now(),
"created_at": created_at,
"updated_at": created_at,
"started_at": None,
"finished_at": None,
"exit_code": None,
@@ -590,6 +595,10 @@ class IngestManager:
return {
"job_id": status["job_id"],
"state": status["state"],
"phase": status["phase"],
"phase_started_at": status["phase_started_at"],
"process_alive": status["process_alive"],
"verification": status["verification"],
"summary": self._summary_for_status(status),
"files": status["files"],
"exit_code": status.get("exit_code"),
@@ -602,10 +611,21 @@ class IngestManager:
job = self._load_job(job_id)
if job["state"] == "queued":
job["state"] = "cancelled"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return self._public_job(job)
if job["state"] == "running" and self._job_phase(job) == "preflight":
job["state"] = "cancelled"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["error"] = "cancelled during OpenCode readiness preflight"
self._write_job(job)
return self._public_job(job)
if job["state"] == "running":
self._supervisor.cancel(job_id)
return self._public_job(self._load_job(job_id))
@@ -627,6 +647,8 @@ class IngestManager:
for job in self._iter_jobs():
if job.get("state") == "queued":
job["state"] = "cancelled"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["error"] = "ingest manager is shutting down"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
@@ -731,10 +753,35 @@ class IngestManager:
return
def _launch_job_locked(self, job: dict[str, Any]) -> bool:
readiness = self.check_opencode_readiness()
phase_started_at = utc_now()
job["state"] = "running"
job["phase"] = "preflight"
job["phase_started_at"] = phase_started_at
job["started_at"] = job.get("started_at") or phase_started_at
job["updated_at"] = phase_started_at
self._write_job(job)
self._lock.release()
try:
try:
readiness = self.check_opencode_readiness()
except Exception as exc:
readiness = {
"ok": False,
"static_preflight": False,
"runtime_smoke_test": "not_run",
"warnings": [],
"errors": [f"{type(exc).__name__}: {exc}"],
}
finally:
self._lock.acquire()
job = self._load_job(job["job_id"])
if job.get("state") != "running" or self._job_phase(job) != "preflight":
return False
job["preflight"] = readiness
if not readiness["ok"]:
job["state"] = "failed"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["error"] = "OpenCode readiness failed: " + "; ".join(readiness["errors"])
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
@@ -754,8 +801,6 @@ class IngestManager:
f"MCP ingest {job['job_id']}",
prompt,
]
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
try:
@@ -770,6 +815,8 @@ class IngestManager:
)
except (OSError, RuntimeError, ValueError) as exc:
job["state"] = "failed"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["error"] = str(exc)
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
@@ -778,6 +825,9 @@ class IngestManager:
current = self._load_job(job["job_id"])
if current.get("state") == "running":
current["pid"] = process.pid
current["phase"] = "agent_running"
current["phase_started_at"] = utc_now()
current["updated_at"] = utc_now()
self._write_job(current)
return True
@@ -789,6 +839,8 @@ class IngestManager:
return
if job.get("state") == "running":
job["state"] = "failed"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["exit_code"] = outcome.returncode
job["error"] = f"ingest completion handling failed: {type(error).__name__}: {error}"
job["finished_at"] = utc_now()
@@ -797,6 +849,7 @@ class IngestManager:
self._start_next_job_locked()
def _handle_process_outcome(self, outcome: ProcessOutcome) -> None:
verification_inputs: dict[str, Any] | None = None
with self._lock:
try:
job = self._load_job(outcome.job_id)
@@ -811,7 +864,6 @@ class IngestManager:
failure_marker=INGEST_FAILURE_MARKER,
)
job["exit_code"] = outcome.returncode
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["result_text"] = result_text
if outcome.cancelled:
@@ -827,8 +879,57 @@ class IngestManager:
job["state"] = "failed"
job["error"] = protocol_error
else:
job["phase"] = "verifying"
job["phase_started_at"] = utc_now()
job["verification"] = {"status": "running"}
job["error"] = ""
verification_inputs = {
"vault_path": self.vault,
"uploads": [dict(item) for item in job["files"]],
"job_started_at": job.get("started_at"),
"archive_sources": parse_bool(
os.environ.get("WIKI_ARCHIVE_SOURCES"),
default=False,
),
}
if verification_inputs is None:
job["finished_at"] = utc_now()
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
self._write_job(job)
if verification_inputs is None:
self._start_next_job_locked()
return
try:
verification = verify_ingest_outputs(**verification_inputs)
except Exception as exc:
verification = {
"status": "failed",
"checked_at": utc_now(),
"failures": [{"code": "verification_error"}],
"warnings": [f"{type(exc).__name__}: {exc}"],
"outputs": {"source_count": len(verification_inputs["uploads"])},
}
with self._lock:
try:
job = self._load_job(outcome.job_id)
except IngestError:
return
if job.get("state") != "running" or self._job_phase(job) != "verifying":
return
job["verification"] = verification
job["finished_at"] = utc_now()
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["updated_at"] = utc_now()
if verification.get("status") == "passed":
job["state"] = "succeeded"
job["error"] = ""
else:
job["state"] = "failed"
job["error"] = "deterministic ingest verification failed"
self._write_job(job)
self._start_next_job_locked()
@@ -961,6 +1062,9 @@ class IngestManager:
for job in self._iter_jobs():
if job.get("state") in {"queued", "running"}:
job["state"] = "failed"
job["phase"] = "complete"
job["phase_started_at"] = utc_now()
job["verification"] = {"status": "not_run"}
job["error"] = "interrupted by MCP service restart"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
@@ -1110,6 +1214,29 @@ class IngestManager:
seen.add(key)
return duplicates
def _job_phase(self, job: dict[str, Any]) -> str:
phase = str(job.get("phase") or "")
if phase in {"queued", "preflight", "agent_running", "verifying", "complete"}:
return phase
state = str(job.get("state") or "")
if state == "queued":
return "queued"
if state == "running":
return "agent_running"
return "complete"
def _job_verification(self, job: dict[str, Any]) -> dict[str, Any]:
verification = job.get("verification")
if not isinstance(verification, dict):
return {"status": "not_run"}
status = str(verification.get("status") or "not_run")
if status not in {"not_run", "running", "passed", "failed"}:
status = "not_run"
return {
**self._redact_public_value(verification),
"status": status,
}
def _public_job(self, job: dict[str, Any]) -> dict[str, Any]:
output = str(job.get("result_text") or extract_final_answer(job["log_path"]))
if output.startswith((INGEST_SUCCESS_MARKER, INGEST_FAILURE_MARKER)):
@@ -1120,9 +1247,25 @@ class IngestManager:
)
output = public_text or error
output = self._redact_paths(output)
phase = self._job_phase(job)
phase_started_at = job.get("phase_started_at")
if not phase_started_at:
if phase == "queued":
phase_started_at = job.get("created_at")
elif phase == "agent_running":
phase_started_at = job.get("started_at") or job.get("updated_at")
else:
phase_started_at = job.get("finished_at") or job.get("updated_at")
return {
"job_id": job["job_id"],
"state": job["state"],
"phase": phase,
"phase_started_at": phase_started_at,
"process_alive": (
phase == "agent_running"
and self._supervisor.is_running(str(job["job_id"]))
),
"verification": self._job_verification(job),
"bundle": self._public_bundle(job["bundle"]) if job.get("bundle") else None,
"files": [self._public_upload(file_meta) for file_meta in job["files"]],
"created_at": job["created_at"],
+236
View File
@@ -0,0 +1,236 @@
"""Deterministic post-run verification for remote MCP ingest jobs."""
from __future__ import annotations
import json
import re
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from obsidian_wiki.file_assets import db_path_for_vault, registry_path_for_vault
SHA256_RE = re.compile(r"^(?:sha256:)?([0-9a-fA-F]{64})$")
DECLARED_PATH_FIELDS = (
"related_pages",
"catalog_paths",
"pages_created",
"pages_updated",
)
def utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def verify_ingest_outputs(
*,
vault_path: str | Path,
uploads: list[dict[str, Any]],
job_started_at: str | None,
archive_sources: bool,
) -> dict[str, Any]:
"""Verify durable ingest artifacts without trusting agent-reported success."""
vault = Path(vault_path).expanduser().resolve()
registry_records, registry_warnings = _load_registry(vault)
manifest_sources, manifest_warnings = _load_manifest_sources(vault)
failures: list[dict[str, str]] = []
warnings = [*registry_warnings, *manifest_warnings]
outputs = {
"source_count": len(uploads),
"registry_count": 0,
"archive_count": 0,
"manifest_count": 0,
"indexed_count": 0,
}
indexed_digests, index_warning = _load_indexed_digests(vault)
if index_warning:
warnings.append(index_warning)
for upload in uploads:
upload_id = str(upload.get("upload_id") or "")
digest = _normalize_digest(upload.get("sha256"))
if digest is None:
failures.append({"code": "invalid_upload_sha256", "upload_id": upload_id})
continue
source_id = f"sha256:{digest}"
registry_record = next(
(
record
for record in registry_records
if str(record.get("source_id") or "").lower() == source_id
),
None,
)
if registry_record is None:
failures.append({"code": "registry_missing", "upload_id": upload_id})
else:
outputs["registry_count"] += 1
if archive_sources:
archive_path = str((registry_record or {}).get("archive_path") or "")
if not archive_path or not _vault_path_exists(vault, archive_path):
failures.append({"code": "archive_missing", "upload_id": upload_id})
else:
outputs["archive_count"] += 1
upload_path = str(upload.get("path") or "")
manifest_record = _matching_manifest_record(
manifest_sources,
upload_path=upload_path,
source_id=source_id,
job_started_at=job_started_at,
)
if manifest_record is None:
failures.append({"code": "manifest_missing", "upload_id": upload_id})
else:
outputs["manifest_count"] += 1
if digest in indexed_digests:
outputs["indexed_count"] += 1
else:
failures.append({"code": "index_missing", "upload_id": upload_id})
declared_paths = _declared_paths(registry_record, manifest_record)
for declared_path in declared_paths:
if not _vault_path_exists(vault, declared_path):
failures.append({"code": "declared_path_missing", "upload_id": upload_id})
break
return {
"status": "passed" if not failures else "failed",
"checked_at": utc_now(),
"failures": failures,
"warnings": warnings,
"outputs": outputs,
}
def _normalize_digest(value: Any) -> str | None:
match = SHA256_RE.fullmatch(str(value or "").strip())
return match.group(1).lower() if match else None
def _load_registry(vault: Path) -> tuple[list[dict[str, Any]], list[str]]:
path = registry_path_for_vault(vault)
if not path.is_file():
return [], []
records: list[dict[str, Any]] = []
malformed = 0
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
record = json.loads(line)
except json.JSONDecodeError:
malformed += 1
continue
if isinstance(record, dict):
records.append(record)
warnings = ["source registry contains malformed records"] if malformed else []
return records, warnings
def _load_manifest_sources(vault: Path) -> tuple[dict[str, Any], list[str]]:
path = vault / ".manifest.json"
if not path.is_file():
return {}, []
try:
manifest = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}, ["manifest could not be parsed"]
sources = manifest.get("sources") if isinstance(manifest, dict) else None
return (sources if isinstance(sources, dict) else {}), []
def _load_indexed_digests(vault: Path) -> tuple[set[str], str | None]:
db_path = db_path_for_vault(vault)
if not db_path.is_file():
return set(), None
try:
with sqlite3.connect(str(db_path)) as conn:
rows = conn.execute("SELECT sha256 FROM file_assets").fetchall()
except sqlite3.Error:
return set(), "file asset index could not be queried"
return {
digest
for row in rows
if row and (digest := _normalize_digest(row[0])) is not None
}, None
def _matching_manifest_record(
sources: dict[str, Any],
*,
upload_path: str,
source_id: str,
job_started_at: str | None,
) -> dict[str, Any] | None:
for source_path, value in sources.items():
if not isinstance(value, dict):
continue
original_path = str(value.get("original_path") or "")
if str(source_path) != upload_path and original_path != upload_path:
continue
hashes = {
str(value.get("content_hash") or "").lower(),
str(value.get("source_id") or "").lower(),
}
if source_id not in hashes:
continue
if not _is_fresh(value.get("ingested_at"), job_started_at):
continue
return value
return None
def _is_fresh(ingested_at: Any, job_started_at: str | None) -> bool:
if not job_started_at:
return bool(ingested_at)
ingested = _parse_timestamp(ingested_at)
started = _parse_timestamp(job_started_at)
return ingested is not None and started is not None and ingested >= started
def _parse_timestamp(value: Any) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def _declared_paths(*records: dict[str, Any] | None) -> set[str]:
paths: set[str] = set()
for record in records:
if not isinstance(record, dict):
continue
for field in DECLARED_PATH_FIELDS:
value = record.get(field)
if isinstance(value, str):
paths.add(value)
elif isinstance(value, list):
paths.update(str(item) for item in value if str(item).strip())
return paths
def _vault_path_exists(vault: Path, relative_path: str) -> bool:
candidate = Path(relative_path)
if candidate.is_absolute():
return False
resolved = (vault / candidate).resolve()
try:
resolved.relative_to(vault)
except ValueError:
return False
return resolved.exists()
+8
View File
@@ -382,6 +382,14 @@ def create_server(
"preferred_text_tool": "ingest_text_note" if enable_ingest else None,
"supports_bundle_upload": enable_ingest,
"supports_body_text": enable_ingest,
"authoritative_job_phases": (
["queued", "preflight", "agent_running", "verifying", "complete"]
if enable_ingest
else []
),
"process_alive_source": "process_supervisor" if enable_ingest else None,
"supports_post_run_verification": enable_ingest,
"semantic_activity_inference": False,
"multipart_optional_fields": (
[
"body_text",