feat(mcp): add verified ingest job phases
Expose authoritative ingest phases and supervisor-backed process liveness without semantic activity inference. Require deterministic registry, manifest, archive, file-index, and declared-output verification before an agent-successful job can finish succeeded.\n\nPreserve legacy state values, derive defaults for older job records, keep verification outside the manager lock, and retain partial outputs on verification failure.\n\nValidated with the full unittest suite (205 tests).
This commit is contained in:
+149
-6
@@ -21,6 +21,7 @@ from threading import RLock
|
||||
from typing import Any
|
||||
|
||||
from obsidian_wiki.file_assets import clean_config_value, db_path_for_vault, registry_path_for_vault
|
||||
from obsidian_wiki.mcp_ingest_verification import verify_ingest_outputs
|
||||
from obsidian_wiki.mcp_jobs import ProcessOutcome, ProcessSupervisor, extract_final_answer, parse_marked_result
|
||||
|
||||
DEFAULT_INGEST_MAX_FILE_BYTES = 100 * 1024 * 1024
|
||||
@@ -533,16 +534,20 @@ class IngestManager:
|
||||
job_id = uuid.uuid4().hex
|
||||
job_dir = self.jobs_root / job_id
|
||||
job_dir.mkdir(parents=True, exist_ok=False)
|
||||
created_at = utc_now()
|
||||
job = {
|
||||
"job_id": job_id,
|
||||
"state": "queued",
|
||||
"phase": "queued",
|
||||
"phase_started_at": created_at,
|
||||
"verification": {"status": "not_run"},
|
||||
"upload_ids": upload_ids,
|
||||
"bundle": bundle,
|
||||
"files": [self._public_upload(upload, expose_path=True) for upload in uploads],
|
||||
"note": self._safe_optional_text(note),
|
||||
"queue_sequence": time.time_ns(),
|
||||
"created_at": utc_now(),
|
||||
"updated_at": utc_now(),
|
||||
"created_at": created_at,
|
||||
"updated_at": created_at,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"exit_code": None,
|
||||
@@ -590,6 +595,10 @@ class IngestManager:
|
||||
return {
|
||||
"job_id": status["job_id"],
|
||||
"state": status["state"],
|
||||
"phase": status["phase"],
|
||||
"phase_started_at": status["phase_started_at"],
|
||||
"process_alive": status["process_alive"],
|
||||
"verification": status["verification"],
|
||||
"summary": self._summary_for_status(status),
|
||||
"files": status["files"],
|
||||
"exit_code": status.get("exit_code"),
|
||||
@@ -602,10 +611,21 @@ class IngestManager:
|
||||
job = self._load_job(job_id)
|
||||
if job["state"] == "queued":
|
||||
job["state"] = "cancelled"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
return self._public_job(job)
|
||||
if job["state"] == "running" and self._job_phase(job) == "preflight":
|
||||
job["state"] = "cancelled"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
job["error"] = "cancelled during OpenCode readiness preflight"
|
||||
self._write_job(job)
|
||||
return self._public_job(job)
|
||||
if job["state"] == "running":
|
||||
self._supervisor.cancel(job_id)
|
||||
return self._public_job(self._load_job(job_id))
|
||||
@@ -627,6 +647,8 @@ class IngestManager:
|
||||
for job in self._iter_jobs():
|
||||
if job.get("state") == "queued":
|
||||
job["state"] = "cancelled"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["error"] = "ingest manager is shutting down"
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
@@ -731,10 +753,35 @@ class IngestManager:
|
||||
return
|
||||
|
||||
def _launch_job_locked(self, job: dict[str, Any]) -> bool:
|
||||
readiness = self.check_opencode_readiness()
|
||||
phase_started_at = utc_now()
|
||||
job["state"] = "running"
|
||||
job["phase"] = "preflight"
|
||||
job["phase_started_at"] = phase_started_at
|
||||
job["started_at"] = job.get("started_at") or phase_started_at
|
||||
job["updated_at"] = phase_started_at
|
||||
self._write_job(job)
|
||||
self._lock.release()
|
||||
try:
|
||||
try:
|
||||
readiness = self.check_opencode_readiness()
|
||||
except Exception as exc:
|
||||
readiness = {
|
||||
"ok": False,
|
||||
"static_preflight": False,
|
||||
"runtime_smoke_test": "not_run",
|
||||
"warnings": [],
|
||||
"errors": [f"{type(exc).__name__}: {exc}"],
|
||||
}
|
||||
finally:
|
||||
self._lock.acquire()
|
||||
job = self._load_job(job["job_id"])
|
||||
if job.get("state") != "running" or self._job_phase(job) != "preflight":
|
||||
return False
|
||||
job["preflight"] = readiness
|
||||
if not readiness["ok"]:
|
||||
job["state"] = "failed"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["error"] = "OpenCode readiness failed: " + "; ".join(readiness["errors"])
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
@@ -754,8 +801,6 @@ class IngestManager:
|
||||
f"MCP ingest {job['job_id']}",
|
||||
prompt,
|
||||
]
|
||||
job["state"] = "running"
|
||||
job["started_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
try:
|
||||
@@ -770,6 +815,8 @@ class IngestManager:
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
job["state"] = "failed"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["error"] = str(exc)
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
@@ -778,6 +825,9 @@ class IngestManager:
|
||||
current = self._load_job(job["job_id"])
|
||||
if current.get("state") == "running":
|
||||
current["pid"] = process.pid
|
||||
current["phase"] = "agent_running"
|
||||
current["phase_started_at"] = utc_now()
|
||||
current["updated_at"] = utc_now()
|
||||
self._write_job(current)
|
||||
return True
|
||||
|
||||
@@ -789,6 +839,8 @@ class IngestManager:
|
||||
return
|
||||
if job.get("state") == "running":
|
||||
job["state"] = "failed"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["exit_code"] = outcome.returncode
|
||||
job["error"] = f"ingest completion handling failed: {type(error).__name__}: {error}"
|
||||
job["finished_at"] = utc_now()
|
||||
@@ -797,6 +849,7 @@ class IngestManager:
|
||||
self._start_next_job_locked()
|
||||
|
||||
def _handle_process_outcome(self, outcome: ProcessOutcome) -> None:
|
||||
verification_inputs: dict[str, Any] | None = None
|
||||
with self._lock:
|
||||
try:
|
||||
job = self._load_job(outcome.job_id)
|
||||
@@ -811,7 +864,6 @@ class IngestManager:
|
||||
failure_marker=INGEST_FAILURE_MARKER,
|
||||
)
|
||||
job["exit_code"] = outcome.returncode
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
job["result_text"] = result_text
|
||||
if outcome.cancelled:
|
||||
@@ -827,8 +879,57 @@ class IngestManager:
|
||||
job["state"] = "failed"
|
||||
job["error"] = protocol_error
|
||||
else:
|
||||
job["phase"] = "verifying"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["verification"] = {"status": "running"}
|
||||
job["error"] = ""
|
||||
verification_inputs = {
|
||||
"vault_path": self.vault,
|
||||
"uploads": [dict(item) for item in job["files"]],
|
||||
"job_started_at": job.get("started_at"),
|
||||
"archive_sources": parse_bool(
|
||||
os.environ.get("WIKI_ARCHIVE_SOURCES"),
|
||||
default=False,
|
||||
),
|
||||
}
|
||||
if verification_inputs is None:
|
||||
job["finished_at"] = utc_now()
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
if verification_inputs is None:
|
||||
self._start_next_job_locked()
|
||||
return
|
||||
|
||||
try:
|
||||
verification = verify_ingest_outputs(**verification_inputs)
|
||||
except Exception as exc:
|
||||
verification = {
|
||||
"status": "failed",
|
||||
"checked_at": utc_now(),
|
||||
"failures": [{"code": "verification_error"}],
|
||||
"warnings": [f"{type(exc).__name__}: {exc}"],
|
||||
"outputs": {"source_count": len(verification_inputs["uploads"])},
|
||||
}
|
||||
|
||||
with self._lock:
|
||||
try:
|
||||
job = self._load_job(outcome.job_id)
|
||||
except IngestError:
|
||||
return
|
||||
if job.get("state") != "running" or self._job_phase(job) != "verifying":
|
||||
return
|
||||
job["verification"] = verification
|
||||
job["finished_at"] = utc_now()
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
if verification.get("status") == "passed":
|
||||
job["state"] = "succeeded"
|
||||
job["error"] = ""
|
||||
else:
|
||||
job["state"] = "failed"
|
||||
job["error"] = "deterministic ingest verification failed"
|
||||
self._write_job(job)
|
||||
self._start_next_job_locked()
|
||||
|
||||
@@ -961,6 +1062,9 @@ class IngestManager:
|
||||
for job in self._iter_jobs():
|
||||
if job.get("state") in {"queued", "running"}:
|
||||
job["state"] = "failed"
|
||||
job["phase"] = "complete"
|
||||
job["phase_started_at"] = utc_now()
|
||||
job["verification"] = {"status": "not_run"}
|
||||
job["error"] = "interrupted by MCP service restart"
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
@@ -1110,6 +1214,29 @@ class IngestManager:
|
||||
seen.add(key)
|
||||
return duplicates
|
||||
|
||||
def _job_phase(self, job: dict[str, Any]) -> str:
|
||||
phase = str(job.get("phase") or "")
|
||||
if phase in {"queued", "preflight", "agent_running", "verifying", "complete"}:
|
||||
return phase
|
||||
state = str(job.get("state") or "")
|
||||
if state == "queued":
|
||||
return "queued"
|
||||
if state == "running":
|
||||
return "agent_running"
|
||||
return "complete"
|
||||
|
||||
def _job_verification(self, job: dict[str, Any]) -> dict[str, Any]:
|
||||
verification = job.get("verification")
|
||||
if not isinstance(verification, dict):
|
||||
return {"status": "not_run"}
|
||||
status = str(verification.get("status") or "not_run")
|
||||
if status not in {"not_run", "running", "passed", "failed"}:
|
||||
status = "not_run"
|
||||
return {
|
||||
**self._redact_public_value(verification),
|
||||
"status": status,
|
||||
}
|
||||
|
||||
def _public_job(self, job: dict[str, Any]) -> dict[str, Any]:
|
||||
output = str(job.get("result_text") or extract_final_answer(job["log_path"]))
|
||||
if output.startswith((INGEST_SUCCESS_MARKER, INGEST_FAILURE_MARKER)):
|
||||
@@ -1120,9 +1247,25 @@ class IngestManager:
|
||||
)
|
||||
output = public_text or error
|
||||
output = self._redact_paths(output)
|
||||
phase = self._job_phase(job)
|
||||
phase_started_at = job.get("phase_started_at")
|
||||
if not phase_started_at:
|
||||
if phase == "queued":
|
||||
phase_started_at = job.get("created_at")
|
||||
elif phase == "agent_running":
|
||||
phase_started_at = job.get("started_at") or job.get("updated_at")
|
||||
else:
|
||||
phase_started_at = job.get("finished_at") or job.get("updated_at")
|
||||
return {
|
||||
"job_id": job["job_id"],
|
||||
"state": job["state"],
|
||||
"phase": phase,
|
||||
"phase_started_at": phase_started_at,
|
||||
"process_alive": (
|
||||
phase == "agent_running"
|
||||
and self._supervisor.is_running(str(job["job_id"]))
|
||||
),
|
||||
"verification": self._job_verification(job),
|
||||
"bundle": self._public_bundle(job["bundle"]) if job.get("bundle") else None,
|
||||
"files": [self._public_upload(file_meta) for file_meta in job["files"]],
|
||||
"created_at": job["created_at"],
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Deterministic post-run verification for remote MCP ingest jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from obsidian_wiki.file_assets import db_path_for_vault, registry_path_for_vault
|
||||
|
||||
|
||||
SHA256_RE = re.compile(r"^(?:sha256:)?([0-9a-fA-F]{64})$")
|
||||
DECLARED_PATH_FIELDS = (
|
||||
"related_pages",
|
||||
"catalog_paths",
|
||||
"pages_created",
|
||||
"pages_updated",
|
||||
)
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def verify_ingest_outputs(
|
||||
*,
|
||||
vault_path: str | Path,
|
||||
uploads: list[dict[str, Any]],
|
||||
job_started_at: str | None,
|
||||
archive_sources: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Verify durable ingest artifacts without trusting agent-reported success."""
|
||||
|
||||
vault = Path(vault_path).expanduser().resolve()
|
||||
registry_records, registry_warnings = _load_registry(vault)
|
||||
manifest_sources, manifest_warnings = _load_manifest_sources(vault)
|
||||
failures: list[dict[str, str]] = []
|
||||
warnings = [*registry_warnings, *manifest_warnings]
|
||||
outputs = {
|
||||
"source_count": len(uploads),
|
||||
"registry_count": 0,
|
||||
"archive_count": 0,
|
||||
"manifest_count": 0,
|
||||
"indexed_count": 0,
|
||||
}
|
||||
|
||||
indexed_digests, index_warning = _load_indexed_digests(vault)
|
||||
if index_warning:
|
||||
warnings.append(index_warning)
|
||||
|
||||
for upload in uploads:
|
||||
upload_id = str(upload.get("upload_id") or "")
|
||||
digest = _normalize_digest(upload.get("sha256"))
|
||||
if digest is None:
|
||||
failures.append({"code": "invalid_upload_sha256", "upload_id": upload_id})
|
||||
continue
|
||||
source_id = f"sha256:{digest}"
|
||||
|
||||
registry_record = next(
|
||||
(
|
||||
record
|
||||
for record in registry_records
|
||||
if str(record.get("source_id") or "").lower() == source_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if registry_record is None:
|
||||
failures.append({"code": "registry_missing", "upload_id": upload_id})
|
||||
else:
|
||||
outputs["registry_count"] += 1
|
||||
|
||||
if archive_sources:
|
||||
archive_path = str((registry_record or {}).get("archive_path") or "")
|
||||
if not archive_path or not _vault_path_exists(vault, archive_path):
|
||||
failures.append({"code": "archive_missing", "upload_id": upload_id})
|
||||
else:
|
||||
outputs["archive_count"] += 1
|
||||
|
||||
upload_path = str(upload.get("path") or "")
|
||||
manifest_record = _matching_manifest_record(
|
||||
manifest_sources,
|
||||
upload_path=upload_path,
|
||||
source_id=source_id,
|
||||
job_started_at=job_started_at,
|
||||
)
|
||||
if manifest_record is None:
|
||||
failures.append({"code": "manifest_missing", "upload_id": upload_id})
|
||||
else:
|
||||
outputs["manifest_count"] += 1
|
||||
|
||||
if digest in indexed_digests:
|
||||
outputs["indexed_count"] += 1
|
||||
else:
|
||||
failures.append({"code": "index_missing", "upload_id": upload_id})
|
||||
|
||||
declared_paths = _declared_paths(registry_record, manifest_record)
|
||||
for declared_path in declared_paths:
|
||||
if not _vault_path_exists(vault, declared_path):
|
||||
failures.append({"code": "declared_path_missing", "upload_id": upload_id})
|
||||
break
|
||||
|
||||
return {
|
||||
"status": "passed" if not failures else "failed",
|
||||
"checked_at": utc_now(),
|
||||
"failures": failures,
|
||||
"warnings": warnings,
|
||||
"outputs": outputs,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_digest(value: Any) -> str | None:
|
||||
match = SHA256_RE.fullmatch(str(value or "").strip())
|
||||
return match.group(1).lower() if match else None
|
||||
|
||||
|
||||
def _load_registry(vault: Path) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
path = registry_path_for_vault(vault)
|
||||
if not path.is_file():
|
||||
return [], []
|
||||
records: list[dict[str, Any]] = []
|
||||
malformed = 0
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
malformed += 1
|
||||
continue
|
||||
if isinstance(record, dict):
|
||||
records.append(record)
|
||||
warnings = ["source registry contains malformed records"] if malformed else []
|
||||
return records, warnings
|
||||
|
||||
|
||||
def _load_manifest_sources(vault: Path) -> tuple[dict[str, Any], list[str]]:
|
||||
path = vault / ".manifest.json"
|
||||
if not path.is_file():
|
||||
return {}, []
|
||||
try:
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}, ["manifest could not be parsed"]
|
||||
sources = manifest.get("sources") if isinstance(manifest, dict) else None
|
||||
return (sources if isinstance(sources, dict) else {}), []
|
||||
|
||||
|
||||
def _load_indexed_digests(vault: Path) -> tuple[set[str], str | None]:
|
||||
db_path = db_path_for_vault(vault)
|
||||
if not db_path.is_file():
|
||||
return set(), None
|
||||
try:
|
||||
with sqlite3.connect(str(db_path)) as conn:
|
||||
rows = conn.execute("SELECT sha256 FROM file_assets").fetchall()
|
||||
except sqlite3.Error:
|
||||
return set(), "file asset index could not be queried"
|
||||
return {
|
||||
digest
|
||||
for row in rows
|
||||
if row and (digest := _normalize_digest(row[0])) is not None
|
||||
}, None
|
||||
|
||||
|
||||
def _matching_manifest_record(
|
||||
sources: dict[str, Any],
|
||||
*,
|
||||
upload_path: str,
|
||||
source_id: str,
|
||||
job_started_at: str | None,
|
||||
) -> dict[str, Any] | None:
|
||||
for source_path, value in sources.items():
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
original_path = str(value.get("original_path") or "")
|
||||
if str(source_path) != upload_path and original_path != upload_path:
|
||||
continue
|
||||
hashes = {
|
||||
str(value.get("content_hash") or "").lower(),
|
||||
str(value.get("source_id") or "").lower(),
|
||||
}
|
||||
if source_id not in hashes:
|
||||
continue
|
||||
if not _is_fresh(value.get("ingested_at"), job_started_at):
|
||||
continue
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _is_fresh(ingested_at: Any, job_started_at: str | None) -> bool:
|
||||
if not job_started_at:
|
||||
return bool(ingested_at)
|
||||
ingested = _parse_timestamp(ingested_at)
|
||||
started = _parse_timestamp(job_started_at)
|
||||
return ingested is not None and started is not None and ingested >= started
|
||||
|
||||
|
||||
def _parse_timestamp(value: Any) -> datetime | None:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _declared_paths(*records: dict[str, Any] | None) -> set[str]:
|
||||
paths: set[str] = set()
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
for field in DECLARED_PATH_FIELDS:
|
||||
value = record.get(field)
|
||||
if isinstance(value, str):
|
||||
paths.add(value)
|
||||
elif isinstance(value, list):
|
||||
paths.update(str(item) for item in value if str(item).strip())
|
||||
return paths
|
||||
|
||||
|
||||
def _vault_path_exists(vault: Path, relative_path: str) -> bool:
|
||||
candidate = Path(relative_path)
|
||||
if candidate.is_absolute():
|
||||
return False
|
||||
resolved = (vault / candidate).resolve()
|
||||
try:
|
||||
resolved.relative_to(vault)
|
||||
except ValueError:
|
||||
return False
|
||||
return resolved.exists()
|
||||
@@ -382,6 +382,14 @@ def create_server(
|
||||
"preferred_text_tool": "ingest_text_note" if enable_ingest else None,
|
||||
"supports_bundle_upload": enable_ingest,
|
||||
"supports_body_text": enable_ingest,
|
||||
"authoritative_job_phases": (
|
||||
["queued", "preflight", "agent_running", "verifying", "complete"]
|
||||
if enable_ingest
|
||||
else []
|
||||
),
|
||||
"process_alive_source": "process_supervisor" if enable_ingest else None,
|
||||
"supports_post_run_verification": enable_ingest,
|
||||
"semantic_activity_inference": False,
|
||||
"multipart_optional_fields": (
|
||||
[
|
||||
"body_text",
|
||||
|
||||
Reference in New Issue
Block a user