feat(mcp): harden remote assets and job execution
Gate source-file downloads and verify registry paths and content hashes. Stream multipart uploads with bounded limits, supervise OpenCode jobs with explicit result markers and timeouts, and preserve managed endpoint security settings. Document the trusted-client boundary and add focused regression coverage. Validation: PYTHONPATH=<staged-tree> python -m unittest discover -s tests (182 tests)
This commit is contained in:
+173
-60
@@ -15,17 +15,21 @@ from threading import RLock
|
||||
from typing import Any
|
||||
|
||||
from obsidian_wiki.file_assets import clean_config_value
|
||||
from obsidian_wiki.mcp_jobs import ProcessOutcome, ProcessSupervisor, extract_final_answer, parse_marked_result
|
||||
|
||||
DEFAULT_QUERY_MAX_CHARS = 8000
|
||||
DEFAULT_OPENCODE_PATH = "opencode"
|
||||
DEFAULT_QUERY_OPENCODE_MODEL = "openai/gpt-5.5"
|
||||
DEFAULT_QUERY_OPENCODE_VARIANT = "minimal"
|
||||
DEFAULT_OPENCODE_SMOKE_TIMEOUT_SECONDS = 120
|
||||
DEFAULT_QUERY_JOB_TIMEOUT_SECONDS = 5 * 60
|
||||
|
||||
QUERY_MODES = {"normal", "index_only"}
|
||||
OPENCODE_REQUIRED_RUN_FLAGS = ("--format", "--auto", "--dir", "--title")
|
||||
OPENCODE_SMOKE_SUCCESS = "WIKI_QUERY_SKILL_LOADED"
|
||||
OPENCODE_SMOKE_FAILURE = "SKILL_LOAD_FAILED"
|
||||
QUERY_SUCCESS_MARKER = "MCP_QUERY_OK"
|
||||
QUERY_FAILURE_MARKER = "MCP_QUERY_FAILED"
|
||||
TERMINAL_JOB_STATES = {"succeeded", "failed", "cancelled"}
|
||||
|
||||
|
||||
@@ -56,6 +60,7 @@ class QueryManager:
|
||||
opencode_model: str | None = DEFAULT_QUERY_OPENCODE_MODEL,
|
||||
opencode_variant: str | None = DEFAULT_QUERY_OPENCODE_VARIANT,
|
||||
max_query_chars: int = DEFAULT_QUERY_MAX_CHARS,
|
||||
job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
self.vault = Path(clean_config_value(str(vault_path))).expanduser().resolve()
|
||||
self.project_root = Path(clean_config_value(str(project_root))).expanduser().resolve()
|
||||
@@ -69,10 +74,15 @@ class QueryManager:
|
||||
self.opencode_model = clean_config_value(str(opencode_model or "")).strip()
|
||||
self.opencode_variant = clean_config_value(str(opencode_variant or "")).strip()
|
||||
self.max_query_chars = max_query_chars
|
||||
self.job_timeout_seconds = max(1, int(job_timeout_seconds))
|
||||
self.jobs_root = self.runtime_root / "jobs"
|
||||
self._lock = RLock()
|
||||
self._processes: dict[str, subprocess.Popen[Any]] = {}
|
||||
self._closing = False
|
||||
self.jobs_root.mkdir(parents=True, exist_ok=True)
|
||||
self._supervisor = ProcessSupervisor(
|
||||
self.runtime_root,
|
||||
popen_factory=lambda *args, **kwargs: subprocess.Popen(*args, **kwargs),
|
||||
)
|
||||
self._mark_interrupted_jobs()
|
||||
|
||||
def start_query_job(
|
||||
@@ -86,6 +96,8 @@ class QueryManager:
|
||||
text = self._validate_query(query)
|
||||
query_mode = self._validate_mode(mode)
|
||||
with self._lock:
|
||||
if self._closing:
|
||||
raise QueryError("query manager is closed")
|
||||
job_id = uuid.uuid4().hex
|
||||
job_dir = self.jobs_root / job_id
|
||||
job_dir.mkdir(parents=True, exist_ok=False)
|
||||
@@ -96,6 +108,7 @@ class QueryManager:
|
||||
"mode": query_mode,
|
||||
"public_only": bool(public_only),
|
||||
"note": self._safe_optional_text(note),
|
||||
"queue_sequence": time.time_ns(),
|
||||
"created_at": utc_now(),
|
||||
"updated_at": utc_now(),
|
||||
"started_at": None,
|
||||
@@ -108,7 +121,7 @@ class QueryManager:
|
||||
}
|
||||
self._write_job(job)
|
||||
self._start_next_job_locked()
|
||||
return self.get_query_job_status(job_id)
|
||||
return self.get_query_job_status(job_id)
|
||||
|
||||
def check_opencode_readiness(self, *, run_smoke_test: bool = False) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
@@ -132,15 +145,14 @@ class QueryManager:
|
||||
return result
|
||||
|
||||
def get_query_job_status(self, job_id: str) -> dict[str, Any]:
|
||||
self.refresh_jobs()
|
||||
with self._lock:
|
||||
self.refresh_jobs()
|
||||
job = self._load_job(job_id)
|
||||
return self._public_job(job)
|
||||
|
||||
def get_query_job_result(self, job_id: str) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
status = self.get_query_job_status(job_id)
|
||||
return self._query_result_from_status(status)
|
||||
status = self.get_query_job_status(job_id)
|
||||
return self._query_result_from_status(status)
|
||||
|
||||
def ask_wiki(
|
||||
self,
|
||||
@@ -179,33 +191,32 @@ class QueryManager:
|
||||
self._write_job(job)
|
||||
return self._public_job(job)
|
||||
if job["state"] == "running":
|
||||
process = self._processes.get(job_id)
|
||||
if process is not None:
|
||||
process.terminate()
|
||||
job["state"] = "cancelled"
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
job["exit_code"] = -15
|
||||
self._write_job(job)
|
||||
self._processes.pop(job_id, None)
|
||||
self._start_next_job_locked()
|
||||
return self._public_job(job)
|
||||
self._supervisor.cancel(job_id)
|
||||
return self._public_job(self._load_job(job_id))
|
||||
return self._public_job(job)
|
||||
|
||||
def refresh_jobs(self) -> None:
|
||||
self._supervisor.refresh()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
for job_id, process in list(self._processes.items()):
|
||||
code = process.poll()
|
||||
if code is None:
|
||||
continue
|
||||
job = self._load_job(job_id)
|
||||
job["state"] = "succeeded" if code == 0 else "failed"
|
||||
job["exit_code"] = code
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
self._processes.pop(job_id, None)
|
||||
self._start_next_job_locked()
|
||||
if self._closing:
|
||||
cancel_queued = False
|
||||
else:
|
||||
self._closing = True
|
||||
cancel_queued = True
|
||||
try:
|
||||
if cancel_queued:
|
||||
with self._lock:
|
||||
for job in self._iter_jobs():
|
||||
if job.get("state") == "queued":
|
||||
job["state"] = "cancelled"
|
||||
job["error"] = "query manager is shutting down"
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
finally:
|
||||
self._supervisor.close()
|
||||
|
||||
def cleanup_runtime(self, *, max_age_days: int = 30) -> dict[str, Any]:
|
||||
if max_age_days <= 0:
|
||||
@@ -216,8 +227,8 @@ class QueryManager:
|
||||
"max_age_days": max_age_days,
|
||||
"old_jobs_deleted": 0,
|
||||
}
|
||||
self.refresh_jobs()
|
||||
with self._lock:
|
||||
self.refresh_jobs()
|
||||
for job in self._iter_jobs():
|
||||
if job.get("state") not in TERMINAL_JOB_STATES:
|
||||
continue
|
||||
@@ -232,15 +243,26 @@ class QueryManager:
|
||||
return result
|
||||
|
||||
def _start_next_job_locked(self) -> None:
|
||||
if any(job.get("state") == "running" for job in self._iter_jobs()):
|
||||
if self._closing:
|
||||
return
|
||||
queued = [job for job in self._iter_jobs() if job.get("state") == "queued"]
|
||||
if not queued:
|
||||
return
|
||||
queued.sort(key=lambda job: job["created_at"])
|
||||
self._launch_job_locked(queued[0])
|
||||
while True:
|
||||
jobs = self._iter_jobs()
|
||||
if any(job.get("state") == "running" for job in jobs):
|
||||
return
|
||||
queued = [job for job in jobs if job.get("state") == "queued"]
|
||||
if not queued:
|
||||
return
|
||||
queued.sort(
|
||||
key=lambda job: (
|
||||
job.get("created_at", ""),
|
||||
int(job.get("queue_sequence") or 0),
|
||||
job.get("job_id", ""),
|
||||
)
|
||||
)
|
||||
if self._launch_job_locked(queued[0]):
|
||||
return
|
||||
|
||||
def _launch_job_locked(self, job: dict[str, Any]) -> None:
|
||||
def _launch_job_locked(self, job: dict[str, Any]) -> bool:
|
||||
readiness = self.check_opencode_readiness()
|
||||
job["preflight"] = readiness
|
||||
if not readiness["ok"]:
|
||||
@@ -249,12 +271,12 @@ class QueryManager:
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
return
|
||||
return False
|
||||
args = [
|
||||
self.opencode_path,
|
||||
"run",
|
||||
"--format",
|
||||
"default",
|
||||
"json",
|
||||
"--auto",
|
||||
*self._opencode_model_args(),
|
||||
"--dir",
|
||||
@@ -263,31 +285,83 @@ class QueryManager:
|
||||
f"MCP query {job['job_id']}",
|
||||
self._build_prompt(job),
|
||||
]
|
||||
log_path = Path(job["log_path"])
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_file = log_path.open("ab")
|
||||
job["state"] = "running"
|
||||
job["started_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
process = self._supervisor.start(
|
||||
job["job_id"],
|
||||
args,
|
||||
cwd=str(self.project_root),
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
log_path=job["log_path"],
|
||||
timeout_seconds=self.job_timeout_seconds,
|
||||
on_complete=self._handle_process_outcome,
|
||||
on_callback_error=self._handle_process_callback_error,
|
||||
)
|
||||
except OSError as exc:
|
||||
log_file.close()
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
job["state"] = "failed"
|
||||
job["error"] = str(exc)
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
return
|
||||
job["state"] = "running"
|
||||
job["started_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
job["pid"] = process.pid
|
||||
self._processes[job["job_id"]] = process
|
||||
self._write_job(job)
|
||||
log_file.close()
|
||||
return False
|
||||
current = self._load_job(job["job_id"])
|
||||
if current.get("state") == "running":
|
||||
current["pid"] = process.pid
|
||||
self._write_job(current)
|
||||
return True
|
||||
|
||||
def _handle_process_callback_error(self, outcome: ProcessOutcome, error: Exception) -> None:
|
||||
with self._lock:
|
||||
try:
|
||||
job = self._load_job(outcome.job_id)
|
||||
except QueryError:
|
||||
return
|
||||
if job.get("state") == "running":
|
||||
job["state"] = "failed"
|
||||
job["exit_code"] = outcome.returncode
|
||||
job["error"] = f"query completion handling failed: {type(error).__name__}: {error}"
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
self._write_job(job)
|
||||
self._start_next_job_locked()
|
||||
|
||||
def _handle_process_outcome(self, outcome: ProcessOutcome) -> None:
|
||||
with self._lock:
|
||||
try:
|
||||
job = self._load_job(outcome.job_id)
|
||||
except QueryError:
|
||||
return
|
||||
if job.get("state") != "running":
|
||||
return
|
||||
final_text = extract_final_answer(job["log_path"])
|
||||
protocol_ok, answer, protocol_error = parse_marked_result(
|
||||
final_text,
|
||||
success_marker=QUERY_SUCCESS_MARKER,
|
||||
failure_marker=QUERY_FAILURE_MARKER,
|
||||
)
|
||||
job["exit_code"] = outcome.returncode
|
||||
job["finished_at"] = utc_now()
|
||||
job["updated_at"] = utc_now()
|
||||
job["result_text"] = answer
|
||||
if outcome.cancelled:
|
||||
job["state"] = "cancelled"
|
||||
job["error"] = outcome.error
|
||||
elif outcome.timed_out:
|
||||
job["state"] = "failed"
|
||||
job["error"] = f"OpenCode query exceeded {self.job_timeout_seconds} seconds"
|
||||
elif outcome.returncode != 0:
|
||||
job["state"] = "failed"
|
||||
job["error"] = outcome.error or protocol_error or f"OpenCode exited with code {outcome.returncode}"
|
||||
elif not protocol_ok:
|
||||
job["state"] = "failed"
|
||||
job["error"] = protocol_error
|
||||
else:
|
||||
job["state"] = "succeeded"
|
||||
job["error"] = ""
|
||||
self._write_job(job)
|
||||
self._start_next_job_locked()
|
||||
|
||||
def _build_prompt(self, job: dict[str, Any]) -> str:
|
||||
mode = "index-only" if job.get("mode") == "index_only" else "normal"
|
||||
@@ -313,6 +387,8 @@ class QueryManager:
|
||||
"Business SQL must follow the schema guardrails: only SELECT/WITH SELECT against documented structured tables, never raw_json, raw_snapshots, writes, DDL, PRAGMA, or sync/config operations.",
|
||||
"If Visibility is public only, do not use business facts; continue with filtered wiki retrieval.",
|
||||
"Return the final synthesized answer with wiki citations when available.",
|
||||
f"If the query succeeds, the final response must start with {QUERY_SUCCESS_MARKER} on its own line, followed by the answer.",
|
||||
f"If it fails, the final response must start with {QUERY_FAILURE_MARKER} on its own line, followed by the reason.",
|
||||
"",
|
||||
f"Optional context note: {note}",
|
||||
]
|
||||
@@ -348,6 +424,15 @@ class QueryManager:
|
||||
self._write_job(job)
|
||||
|
||||
def _public_job(self, job: dict[str, Any]) -> dict[str, Any]:
|
||||
output = str(job.get("result_text") or extract_final_answer(job["log_path"]))
|
||||
if output.startswith((QUERY_SUCCESS_MARKER, QUERY_FAILURE_MARKER)):
|
||||
_ok, public_text, error = parse_marked_result(
|
||||
output,
|
||||
success_marker=QUERY_SUCCESS_MARKER,
|
||||
failure_marker=QUERY_FAILURE_MARKER,
|
||||
)
|
||||
output = public_text or error
|
||||
output = self._redact_paths(output)
|
||||
return {
|
||||
"job_id": job["job_id"],
|
||||
"state": job["state"],
|
||||
@@ -359,12 +444,40 @@ class QueryManager:
|
||||
"started_at": job.get("started_at"),
|
||||
"finished_at": job.get("finished_at"),
|
||||
"exit_code": job.get("exit_code"),
|
||||
"pid": job.get("pid"),
|
||||
"error": job.get("error", ""),
|
||||
"preflight": job.get("preflight"),
|
||||
"log_tail": self._read_tail(Path(job["log_path"])),
|
||||
"error": self._redact_paths(str(job.get("error", ""))),
|
||||
"preflight": self._public_preflight(job.get("preflight")),
|
||||
"log_tail": output,
|
||||
}
|
||||
|
||||
def _public_preflight(self, preflight: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if preflight is None:
|
||||
return None
|
||||
return {
|
||||
key: self._redact_public_value(value)
|
||||
for key, value in preflight.items()
|
||||
if key != "checked_paths"
|
||||
}
|
||||
|
||||
def _redact_public_value(self, value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {key: self._redact_public_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [self._redact_public_value(item) for item in value]
|
||||
if isinstance(value, str):
|
||||
return self._redact_paths(value)
|
||||
return value
|
||||
|
||||
def _redact_paths(self, value: str) -> str:
|
||||
text = value
|
||||
for path, label in (
|
||||
(self.runtime_root, "<runtime>"),
|
||||
(self.vault, "<vault>"),
|
||||
(self.project_root, "<project>"),
|
||||
(Path.home(), "<home>"),
|
||||
):
|
||||
text = text.replace(str(path), label)
|
||||
return text
|
||||
|
||||
def _query_result_from_status(self, status: dict[str, Any]) -> dict[str, Any]:
|
||||
output_tail = status.get("log_tail", "")
|
||||
answer = output_tail.strip()
|
||||
@@ -542,7 +655,7 @@ class QueryManager:
|
||||
self.opencode_path,
|
||||
"run",
|
||||
"--format",
|
||||
"default",
|
||||
"json",
|
||||
"--auto",
|
||||
*self._opencode_model_args(),
|
||||
"--dir",
|
||||
|
||||
Reference in New Issue
Block a user