feat(mcp): harden remote assets and job execution

Gate source-file downloads and verify registry paths and content hashes. Stream multipart uploads with bounded limits, supervise OpenCode jobs with explicit result markers and timeouts, and preserve managed endpoint security settings. Document the trusted-client boundary and add focused regression coverage.

Validation: PYTHONPATH=<staged-tree> python -m unittest discover -s tests (182 tests)
This commit is contained in:
Codex
2026-07-09 21:44:48 +00:00
parent d47bb2c549
commit 7eb9b987ab
16 changed files with 3709 additions and 500 deletions
+173 -60
View File
@@ -15,17 +15,21 @@ from threading import RLock
from typing import Any
from obsidian_wiki.file_assets import clean_config_value
from obsidian_wiki.mcp_jobs import ProcessOutcome, ProcessSupervisor, extract_final_answer, parse_marked_result
DEFAULT_QUERY_MAX_CHARS = 8000
DEFAULT_OPENCODE_PATH = "opencode"
DEFAULT_QUERY_OPENCODE_MODEL = "openai/gpt-5.5"
DEFAULT_QUERY_OPENCODE_VARIANT = "minimal"
DEFAULT_OPENCODE_SMOKE_TIMEOUT_SECONDS = 120
DEFAULT_QUERY_JOB_TIMEOUT_SECONDS = 5 * 60
QUERY_MODES = {"normal", "index_only"}
OPENCODE_REQUIRED_RUN_FLAGS = ("--format", "--auto", "--dir", "--title")
OPENCODE_SMOKE_SUCCESS = "WIKI_QUERY_SKILL_LOADED"
OPENCODE_SMOKE_FAILURE = "SKILL_LOAD_FAILED"
QUERY_SUCCESS_MARKER = "MCP_QUERY_OK"
QUERY_FAILURE_MARKER = "MCP_QUERY_FAILED"
TERMINAL_JOB_STATES = {"succeeded", "failed", "cancelled"}
@@ -56,6 +60,7 @@ class QueryManager:
opencode_model: str | None = DEFAULT_QUERY_OPENCODE_MODEL,
opencode_variant: str | None = DEFAULT_QUERY_OPENCODE_VARIANT,
max_query_chars: int = DEFAULT_QUERY_MAX_CHARS,
job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
) -> None:
self.vault = Path(clean_config_value(str(vault_path))).expanduser().resolve()
self.project_root = Path(clean_config_value(str(project_root))).expanduser().resolve()
@@ -69,10 +74,15 @@ class QueryManager:
self.opencode_model = clean_config_value(str(opencode_model or "")).strip()
self.opencode_variant = clean_config_value(str(opencode_variant or "")).strip()
self.max_query_chars = max_query_chars
self.job_timeout_seconds = max(1, int(job_timeout_seconds))
self.jobs_root = self.runtime_root / "jobs"
self._lock = RLock()
self._processes: dict[str, subprocess.Popen[Any]] = {}
self._closing = False
self.jobs_root.mkdir(parents=True, exist_ok=True)
self._supervisor = ProcessSupervisor(
self.runtime_root,
popen_factory=lambda *args, **kwargs: subprocess.Popen(*args, **kwargs),
)
self._mark_interrupted_jobs()
def start_query_job(
@@ -86,6 +96,8 @@ class QueryManager:
text = self._validate_query(query)
query_mode = self._validate_mode(mode)
with self._lock:
if self._closing:
raise QueryError("query manager is closed")
job_id = uuid.uuid4().hex
job_dir = self.jobs_root / job_id
job_dir.mkdir(parents=True, exist_ok=False)
@@ -96,6 +108,7 @@ class QueryManager:
"mode": query_mode,
"public_only": bool(public_only),
"note": self._safe_optional_text(note),
"queue_sequence": time.time_ns(),
"created_at": utc_now(),
"updated_at": utc_now(),
"started_at": None,
@@ -108,7 +121,7 @@ class QueryManager:
}
self._write_job(job)
self._start_next_job_locked()
return self.get_query_job_status(job_id)
return self.get_query_job_status(job_id)
def check_opencode_readiness(self, *, run_smoke_test: bool = False) -> dict[str, Any]:
result: dict[str, Any] = {
@@ -132,15 +145,14 @@ class QueryManager:
return result
def get_query_job_status(self, job_id: str) -> dict[str, Any]:
self.refresh_jobs()
with self._lock:
self.refresh_jobs()
job = self._load_job(job_id)
return self._public_job(job)
def get_query_job_result(self, job_id: str) -> dict[str, Any]:
with self._lock:
status = self.get_query_job_status(job_id)
return self._query_result_from_status(status)
status = self.get_query_job_status(job_id)
return self._query_result_from_status(status)
def ask_wiki(
self,
@@ -179,33 +191,32 @@ class QueryManager:
self._write_job(job)
return self._public_job(job)
if job["state"] == "running":
process = self._processes.get(job_id)
if process is not None:
process.terminate()
job["state"] = "cancelled"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["exit_code"] = -15
self._write_job(job)
self._processes.pop(job_id, None)
self._start_next_job_locked()
return self._public_job(job)
self._supervisor.cancel(job_id)
return self._public_job(self._load_job(job_id))
return self._public_job(job)
def refresh_jobs(self) -> None:
self._supervisor.refresh()
def close(self) -> None:
with self._lock:
for job_id, process in list(self._processes.items()):
code = process.poll()
if code is None:
continue
job = self._load_job(job_id)
job["state"] = "succeeded" if code == 0 else "failed"
job["exit_code"] = code
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
self._processes.pop(job_id, None)
self._start_next_job_locked()
if self._closing:
cancel_queued = False
else:
self._closing = True
cancel_queued = True
try:
if cancel_queued:
with self._lock:
for job in self._iter_jobs():
if job.get("state") == "queued":
job["state"] = "cancelled"
job["error"] = "query manager is shutting down"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
finally:
self._supervisor.close()
def cleanup_runtime(self, *, max_age_days: int = 30) -> dict[str, Any]:
if max_age_days <= 0:
@@ -216,8 +227,8 @@ class QueryManager:
"max_age_days": max_age_days,
"old_jobs_deleted": 0,
}
self.refresh_jobs()
with self._lock:
self.refresh_jobs()
for job in self._iter_jobs():
if job.get("state") not in TERMINAL_JOB_STATES:
continue
@@ -232,15 +243,26 @@ class QueryManager:
return result
def _start_next_job_locked(self) -> None:
if any(job.get("state") == "running" for job in self._iter_jobs()):
if self._closing:
return
queued = [job for job in self._iter_jobs() if job.get("state") == "queued"]
if not queued:
return
queued.sort(key=lambda job: job["created_at"])
self._launch_job_locked(queued[0])
while True:
jobs = self._iter_jobs()
if any(job.get("state") == "running" for job in jobs):
return
queued = [job for job in jobs if job.get("state") == "queued"]
if not queued:
return
queued.sort(
key=lambda job: (
job.get("created_at", ""),
int(job.get("queue_sequence") or 0),
job.get("job_id", ""),
)
)
if self._launch_job_locked(queued[0]):
return
def _launch_job_locked(self, job: dict[str, Any]) -> None:
def _launch_job_locked(self, job: dict[str, Any]) -> bool:
readiness = self.check_opencode_readiness()
job["preflight"] = readiness
if not readiness["ok"]:
@@ -249,12 +271,12 @@ class QueryManager:
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return
return False
args = [
self.opencode_path,
"run",
"--format",
"default",
"json",
"--auto",
*self._opencode_model_args(),
"--dir",
@@ -263,31 +285,83 @@ class QueryManager:
f"MCP query {job['job_id']}",
self._build_prompt(job),
]
log_path = Path(job["log_path"])
log_path.parent.mkdir(parents=True, exist_ok=True)
log_file = log_path.open("ab")
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
try:
process = subprocess.Popen(
process = self._supervisor.start(
job["job_id"],
args,
cwd=str(self.project_root),
stdout=log_file,
stderr=subprocess.STDOUT,
log_path=job["log_path"],
timeout_seconds=self.job_timeout_seconds,
on_complete=self._handle_process_outcome,
on_callback_error=self._handle_process_callback_error,
)
except OSError as exc:
log_file.close()
except (OSError, RuntimeError, ValueError) as exc:
job["state"] = "failed"
job["error"] = str(exc)
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
job["pid"] = process.pid
self._processes[job["job_id"]] = process
self._write_job(job)
log_file.close()
return False
current = self._load_job(job["job_id"])
if current.get("state") == "running":
current["pid"] = process.pid
self._write_job(current)
return True
def _handle_process_callback_error(self, outcome: ProcessOutcome, error: Exception) -> None:
with self._lock:
try:
job = self._load_job(outcome.job_id)
except QueryError:
return
if job.get("state") == "running":
job["state"] = "failed"
job["exit_code"] = outcome.returncode
job["error"] = f"query completion handling failed: {type(error).__name__}: {error}"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
self._start_next_job_locked()
def _handle_process_outcome(self, outcome: ProcessOutcome) -> None:
with self._lock:
try:
job = self._load_job(outcome.job_id)
except QueryError:
return
if job.get("state") != "running":
return
final_text = extract_final_answer(job["log_path"])
protocol_ok, answer, protocol_error = parse_marked_result(
final_text,
success_marker=QUERY_SUCCESS_MARKER,
failure_marker=QUERY_FAILURE_MARKER,
)
job["exit_code"] = outcome.returncode
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["result_text"] = answer
if outcome.cancelled:
job["state"] = "cancelled"
job["error"] = outcome.error
elif outcome.timed_out:
job["state"] = "failed"
job["error"] = f"OpenCode query exceeded {self.job_timeout_seconds} seconds"
elif outcome.returncode != 0:
job["state"] = "failed"
job["error"] = outcome.error or protocol_error or f"OpenCode exited with code {outcome.returncode}"
elif not protocol_ok:
job["state"] = "failed"
job["error"] = protocol_error
else:
job["state"] = "succeeded"
job["error"] = ""
self._write_job(job)
self._start_next_job_locked()
def _build_prompt(self, job: dict[str, Any]) -> str:
mode = "index-only" if job.get("mode") == "index_only" else "normal"
@@ -313,6 +387,8 @@ class QueryManager:
"Business SQL must follow the schema guardrails: only SELECT/WITH SELECT against documented structured tables, never raw_json, raw_snapshots, writes, DDL, PRAGMA, or sync/config operations.",
"If Visibility is public only, do not use business facts; continue with filtered wiki retrieval.",
"Return the final synthesized answer with wiki citations when available.",
f"If the query succeeds, the final response must start with {QUERY_SUCCESS_MARKER} on its own line, followed by the answer.",
f"If it fails, the final response must start with {QUERY_FAILURE_MARKER} on its own line, followed by the reason.",
"",
f"Optional context note: {note}",
]
@@ -348,6 +424,15 @@ class QueryManager:
self._write_job(job)
def _public_job(self, job: dict[str, Any]) -> dict[str, Any]:
output = str(job.get("result_text") or extract_final_answer(job["log_path"]))
if output.startswith((QUERY_SUCCESS_MARKER, QUERY_FAILURE_MARKER)):
_ok, public_text, error = parse_marked_result(
output,
success_marker=QUERY_SUCCESS_MARKER,
failure_marker=QUERY_FAILURE_MARKER,
)
output = public_text or error
output = self._redact_paths(output)
return {
"job_id": job["job_id"],
"state": job["state"],
@@ -359,12 +444,40 @@ class QueryManager:
"started_at": job.get("started_at"),
"finished_at": job.get("finished_at"),
"exit_code": job.get("exit_code"),
"pid": job.get("pid"),
"error": job.get("error", ""),
"preflight": job.get("preflight"),
"log_tail": self._read_tail(Path(job["log_path"])),
"error": self._redact_paths(str(job.get("error", ""))),
"preflight": self._public_preflight(job.get("preflight")),
"log_tail": output,
}
def _public_preflight(self, preflight: dict[str, Any] | None) -> dict[str, Any] | None:
if preflight is None:
return None
return {
key: self._redact_public_value(value)
for key, value in preflight.items()
if key != "checked_paths"
}
def _redact_public_value(self, value: Any) -> Any:
if isinstance(value, dict):
return {key: self._redact_public_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [self._redact_public_value(item) for item in value]
if isinstance(value, str):
return self._redact_paths(value)
return value
def _redact_paths(self, value: str) -> str:
text = value
for path, label in (
(self.runtime_root, "<runtime>"),
(self.vault, "<vault>"),
(self.project_root, "<project>"),
(Path.home(), "<home>"),
):
text = text.replace(str(path), label)
return text
def _query_result_from_status(self, status: dict[str, Any]) -> dict[str, Any]:
output_tail = status.get("log_tail", "")
answer = output_tail.strip()
@@ -542,7 +655,7 @@ class QueryManager:
self.opencode_path,
"run",
"--format",
"default",
"json",
"--auto",
*self._opencode_model_args(),
"--dir",