feat(mcp): harden remote assets and job execution

Gate source-file downloads and verify registry paths and content hashes. Stream multipart uploads with bounded limits, supervise OpenCode jobs with explicit result markers and timeouts, and preserve managed endpoint security settings. Document the trusted-client boundary and add focused regression coverage.

Validation: PYTHONPATH=<staged-tree> python -m unittest discover -s tests (182 tests)
This commit is contained in:
Codex
2026-07-09 21:44:48 +00:00
parent d47bb2c549
commit 7eb9b987ab
16 changed files with 3709 additions and 500 deletions
+23 -1
View File
@@ -40,10 +40,13 @@ OBSIDIAN_BUSINESS_FACTS_DB=
#
# Run with: obsidian-wiki-mcp
# Install extra: pip install "obsidian-wiki[mcp]"
# The MCP server serves only safe engineering source files by default.
# Metadata/search is limited to safe engineering source files. Downloading the
# underlying source bytes is a separate opt-in and defaults off for new managed
# installs; metadata tools remain available when it is disabled.
OBSIDIAN_MCP_BOT_PROFILE=bjd-general-bot
OBSIDIAN_MCP_MAX_FILE_BYTES=26214400
OBSIDIAN_MCP_EXPOSE_PATHS=false
OBSIDIAN_MCP_ALLOW_SOURCE_FILES=false
# Optional remote Streamable HTTP MCP mode.
# Run locally:
@@ -57,10 +60,22 @@ OBSIDIAN_MCP_HOST=127.0.0.1
OBSIDIAN_MCP_PORT=8765
OBSIDIAN_MCP_PATH=/mcp
OBSIDIAN_MCP_ENABLED_TOOLS=file-assets
# Optional public URLs shown to remote agents in capabilities/config output.
# Set PUBLIC_BASE_URL when /mcp is behind DNS/tunnel/reverse proxy.
OBSIDIAN_MCP_PUBLIC_BASE_URL=
# Set only when the ingest upload endpoint has a different public URL.
OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=
# Comma-separated HTTP Host and Origin allowlists. Managed service installs
# derive these from their bind/public URLs; set them explicitly for manual HTTP
# deployments, especially behind a tunnel or reverse proxy.
OBSIDIAN_MCP_ALLOWED_HOSTS=
OBSIDIAN_MCP_ALLOWED_ORIGINS=
# Optional write-capable MCP ingest tools. Default is disabled.
# When enabled, remote MCP clients sharing this token can upload files and
# trigger a local `opencode run --auto` wiki-ingest job for those uploads.
# The prompt entrypoint is fixed, but `--auto` has no enforced shell sandbox.
# Enable ingest/query only for trusted clients behind a trusted network boundary.
# After enabling, call the MCP tool `check_ingest_opencode_readiness` before
# the first upload; set run_smoke_test=true to verify OpenCode can load the
# project wiki-ingest skill without editing files.
@@ -74,9 +89,15 @@ OBSIDIAN_MCP_ENABLED_TOOLS=file-assets
# Prefer OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,ingest for new configs. This
# legacy flag is still read when OBSIDIAN_MCP_ENABLED_TOOLS is absent.
OBSIDIAN_MCP_ENABLE_INGEST=0
# Multipart files are streamed to spooled storage. The parser enforces the
# per-file cap while consuming each part.
OBSIDIAN_MCP_INGEST_MAX_FILE_BYTES=104857600
OBSIDIAN_MCP_INGEST_MAX_CHUNK_BYTES=4194304
OBSIDIAN_MCP_INGEST_MAX_FILES_PER_JOB=10
# 0 derives the total request cap from max-file-bytes * max-files plus multipart
# overhead. A positive value sets an additional, stricter total request cap.
OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES=0
OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS=1800
OBSIDIAN_MCP_OPENCODE_PATH=opencode
OBSIDIAN_MCP_INGEST_OPENCODE_PATH=opencode
OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5
@@ -94,6 +115,7 @@ OBSIDIAN_MCP_INGEST_RUNTIME_ROOT=.runtime/mcp-ingest
OBSIDIAN_MCP_QUERY_OPENCODE_MODEL=openai/gpt-5.5
OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT=minimal
OBSIDIAN_MCP_QUERY_MAX_CHARS=8000
OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS=300
OBSIDIAN_MCP_QUERY_RUNTIME_ROOT=.runtime/mcp-query
# Wiki categories (directories created in the vault)
+10 -8
View File
@@ -391,7 +391,7 @@ Configure your agent to run `obsidian-wiki-mcp` as a stdio MCP server. The serve
- `get_mcp_capabilities` — tells remote agents which tool groups are enabled and which tool to use first.
- `asset://source/sha256/{digest}` — binary resource for an allowed archived file.
By default the MCP layer is stricter than local CLI lookup: it only serves engineering files such as drawings, 3D/STEP models, specifications, product parameters, and test reports. Customer-shareable engineering deliverables, such as PDF customer drawings, are allowed for source-file lookup when the active profile already allows technical engineering data. It blocks raw orders, prices, email correspondence, complaint evidence, claims, personal data, and files missing from `_meta/source-registry.jsonl`. Use `OBSIDIAN_MCP_BOT_PROFILE`, `OBSIDIAN_MCP_MAX_FILE_BYTES`, and `OBSIDIAN_MCP_EXPOSE_PATHS` to tune the runtime behavior.
By default the MCP layer is stricter than local CLI lookup: it only returns metadata for engineering files such as drawings, 3D/STEP models, specifications, product parameters, and test reports. Customer-shareable engineering deliverables, such as PDF customer drawings, are eligible for lookup when the active profile already allows technical engineering data. It blocks raw orders, prices, email correspondence, complaint evidence, claims, personal data, and files missing from `_meta/source-registry.jsonl`. Downloading the underlying `asset://source/...` bytes is a separate opt-in and defaults off for new managed installs; set `OBSIDIAN_MCP_ALLOW_SOURCE_FILES=true` (or pass `--allow-source-files`) only for trusted clients. Use `OBSIDIAN_MCP_BOT_PROFILE`, `OBSIDIAN_MCP_MAX_FILE_BYTES`, and `OBSIDIAN_MCP_EXPOSE_PATHS` to tune the remaining runtime behavior.
Optional: run the same MCP server over Streamable HTTP for a remote MCP client:
@@ -428,7 +428,7 @@ For long-running access from another machine, use the managed user service:
scripts/obsidian-wiki-ops.py mcp start
```
`scripts/obsidian-wiki-ops.py mcp start` creates `.runtime/obsidian-wiki-mcp.env` with `0600` permissions if it does not exist, writes `~/.config/systemd/user/obsidian-wiki-mcp.service`, starts the service, and prints the remote URL, `Authorization` header, enabled tool groups, config file path, and client JSON to copy into another agent. The user service restarts on failure and sets `MemoryMax=2G`. When `ingest` is enabled, it also prints the `/mcp-ingest/uploads` file upload endpoint. It serves the configured `OBSIDIAN_VAULT_PATH` by default; pass `--vault /path/to/vault` to override. The service defaults to `0.0.0.0:8765/mcp` because it is intended for remote clients, but it still requires the bearer token. Prefer SSH tunnel, Tailscale, WireGuard, or another trusted network boundary instead of directly exposing the port to the public internet.
`scripts/obsidian-wiki-ops.py mcp start` creates `.runtime/obsidian-wiki-mcp.env` with `0600` permissions if it does not exist, writes `~/.config/systemd/user/obsidian-wiki-mcp.service`, starts the service, and prints the remote URL, `Authorization` header, enabled tool groups, config file path, and client JSON to copy into another agent. The managed endpoint values (bind host/port/path, public URLs, and transport allowlists) live in that env file and are preserved across later starts and token rotations. The user service restarts on failure and sets `MemoryMax=2G`. When `ingest` is enabled, it also prints the `/mcp-ingest/uploads` file upload endpoint. It serves the configured `OBSIDIAN_VAULT_PATH` by default; pass `--vault /path/to/vault` to override. The service defaults to `0.0.0.0:8765/mcp` because it is intended for remote clients, but it still requires the bearer token. Managed installs derive `OBSIDIAN_MCP_ALLOWED_HOSTS` and `OBSIDIAN_MCP_ALLOWED_ORIGINS` from their bind/public URLs; review them when using custom DNS, a tunnel, or a reverse proxy. Prefer SSH tunnel, Tailscale, WireGuard, or another trusted network boundary instead of directly exposing the port to the public internet.
Customize the first startup if needed:
@@ -471,7 +471,7 @@ This adds write-capable MCP tools to the same `wiki-file-assets` server and bear
- `start_ingest_job` — queues a local `opencode run --auto --format json` job with a fixed `/wiki-ingest` prompt for the completed uploads.
- `get_ingest_job_status`, `get_ingest_job_result`, `cancel_ingest_job` — inspect or cancel the local job.
The managed service config sets the OpenCode model server-side. By default ingest uses `OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5` and `OBSIDIAN_MCP_INGEST_OPENCODE_VARIANT=high` for stronger file processing. It also writes `OBSIDIAN_MCP_PUBLIC_BASE_URL` from the detected remote address so ticket responses can include a complete upload URL; set it manually when the service is behind a tunnel, reverse proxy, or custom DNS name. Change these values in `.runtime/obsidian-wiki-mcp.env` on the server if needed; remote MCP clients do not choose the model per call.
The managed service config sets the OpenCode model server-side. By default ingest uses `OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5`, `OBSIDIAN_MCP_INGEST_OPENCODE_VARIANT=high`, and an `OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS=1800` deadline. It also writes `OBSIDIAN_MCP_PUBLIC_BASE_URL` from the detected remote address so ticket responses can include a complete upload URL; set it manually when the service is behind a tunnel, reverse proxy, or custom DNS name. If the upload endpoint is exposed through a different public URL than `/mcp`, set `OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL` explicitly and it will override the derived upload URL in capabilities, upload instructions, tickets, and `show-config`. Change these values in `.runtime/obsidian-wiki-mcp.env` on the server if needed; remote MCP clients do not choose the model per call.
For short text thoughts, notes, findings, or pasted context, call `ingest_text_note(text, title?, project?, doc_type_hint?, note?, start_job?)`. It uses `doc_type_hint="thought"` by default and returns the created upload plus the queued job when `start_job=true`.
@@ -486,11 +486,13 @@ curl -H "X-Obsidian-Wiki-Upload-Token: <upload-token>" \
<upload_url>
```
The endpoint accepts repeated `files` fields plus optional `project`, `doc_type_hint`, `note`, and `start_job` fields. It still accepts the same bearer token as `/mcp` for local/admin compatibility, but remote agents should prefer single-use upload tickets. Upload tickets are consumed only by authorized `POST` uploads, not by accidental `GET`/wrong-method requests. The upload route enforces configured `--allowed-host` and `--allowed-origin` checks, and rejects oversized request bodies with HTTP `413` before parsing multipart content. Base64 file upload is intentionally not exposed as an MCP tool because large JSON arguments are fragile in many agent tool-call implementations. Duplicate detection is exact sha256 matching only; it does not replace human review for near-duplicate or renamed source files.
The endpoint accepts repeated `files` fields plus optional `project`, `doc_type_hint`, `note`, and `start_job` fields. Multipart files are parsed as streams into spooled storage instead of buffering the whole request in memory, and `OBSIDIAN_MCP_INGEST_MAX_FILE_BYTES` is enforced while each file part is being parsed. `OBSIDIAN_MCP_INGEST_MAX_FILES_PER_JOB` limits the count. With `OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES=0`, the route derives its total request-body cap from per-file size times maximum file count plus multipart overhead; a positive value adds a stricter total cap. The stricter upload-ticket limits still apply. Ticket-bound `project`, `doc_type_hint`, and `note` values cannot be replaced by form data, and a ticket created with `start_job=false` cannot be escalated to start a job by sending `start_job=true` in the form. Oversized requests return HTTP `413`. The endpoint still accepts the same bearer token as `/mcp` for local/admin compatibility, but remote agents should prefer single-use upload tickets. Upload tickets are consumed by authorized `POST` attempts, including malformed multipart or oversized uploads, but not by accidental `GET`/wrong-method requests. When an upload-ticket POST fails after the ticket is consumed, the JSON error includes `ticket_consumed: true` and `retry_action: "create_new_upload_ticket"`, so the remote agent should request a fresh ticket before retrying. The upload route enforces configured `--allowed-host` and `--allowed-origin` checks. Base64 file upload is intentionally not exposed as an MCP tool because large JSON arguments are fragile in many agent tool-call implementations. Duplicate detection is exact sha256 matching only; it does not replace human review for near-duplicate or renamed source files.
Run `check_ingest_opencode_readiness` before the first remote upload. Use `run_smoke_test: true` when you want OpenCode itself to prove it can load the project `wiki-ingest` skill; otherwise the static preflight is also run automatically before every `start_ingest_job`.
Uploads and text notes land under `.runtime/mcp-ingest/uploads/`, outside the Obsidian vault, and are treated only as temporary source staging. Job state and logs live under `.runtime/mcp-ingest/jobs/`. Remote upload and job responses expose upload/job ids, hashes, states, and log tails, but omit local runtime file paths; the internal job record still keeps paths so the fixed local OpenCode prompt can ingest only the uploaded files. `get_mcp_capabilities` includes `server_version`, `capabilities_schema_version`, `tools_version`, file-index freshness, ingest queue state, and known limitations so remote agents can detect stale client instructions. The MCP server refuses to enable ingest if `OBSIDIAN_MCP_INGEST_RUNTIME_ROOT` points inside the vault; let `wiki-ingest` decide how to archive sources into `_sources/archive/`, update registries, and rebuild the file index. The ingest prompt is generated by the MCP service, requires OpenCode to load `wiki-ingest`, and restricts OpenCode to the uploaded file list; clients cannot pass arbitrary shell commands through this interface. Because enabling ingest gives the remote token write access to the vault through local OpenCode, keep it on a trusted network and rotate the token if it has been shared too broadly.
Uploads and text notes land under `.runtime/mcp-ingest/uploads/`, outside the Obsidian vault, and are treated only as temporary source staging. Job state and logs live under `.runtime/mcp-ingest/jobs/`. Remote upload and job responses expose upload/job ids, hashes, states, and bounded result text, but omit local runtime file paths; the internal job record still keeps paths so the fixed local OpenCode prompt can ingest only the uploaded files. Completion is accepted only from the final JSON text event containing `MCP_INGEST_OK`; `MCP_INGEST_FAILED`, a missing marker, a nonzero exit, timeout, or cancellation marks the job failed/cancelled. `get_mcp_capabilities` includes `server_version`, `schema_version`/`capabilities_schema_version`, `tools_version`, file-index freshness, ingest queue state, and known limitations so remote agents can detect stale client instructions. Its `recommended_first_steps` only names currently enabled tool groups. The MCP server refuses to enable ingest if `OBSIDIAN_MCP_INGEST_RUNTIME_ROOT` points inside the vault; let `wiki-ingest` decide how to archive sources into `_sources/archive/`, update registries, and rebuild the file index.
Ingest and query launch OpenCode with a fixed prompt and `--auto`, but the MCP service does not enforce a shell sandbox and prompt restrictions are not a security boundary. There is no direct arbitrary-prompt or arbitrary-shell MCP tool, yet a client able to influence vault/upload content must still be trusted. Keep these tool groups behind a trusted network boundary and rotate the token if it has been shared too broadly.
To let a remote MCP client ask synthesized questions against the compiled wiki, enable the `query` tool group:
@@ -504,12 +506,12 @@ This adds read-oriented OpenCode-backed query tools to the same `wiki-file-asset
- `check_query_opencode_readiness` — verifies the local OpenCode binary, required run flags, project `wiki-query`/`llm-wiki` skills, and optionally runs a no-write smoke test.
- `ask_wiki` — preferred one-call query path for remote agents. It starts the local job, polls briefly, and returns either a structured answer or a `job_id` to continue polling.
- `start_wiki_query` — queues a local `opencode run --auto --format default` job with a fixed `/wiki-query` prompt. Parameters: `query`, `mode` (`normal` or `index_only`), `public_only` (default `false`), and optional `note`.
- `start_wiki_query` — queues a local `opencode run --auto --format json` job with a fixed `/wiki-query` prompt. Parameters: `query`, `mode` (`normal` or `index_only`), `public_only` (default `false`), and optional `note`.
- `get_wiki_query_status`, `get_wiki_query_result`, `cancel_wiki_query` — inspect, retrieve, or cancel the local query job.
The managed service config sets query to `OBSIDIAN_MCP_QUERY_OPENCODE_MODEL=openai/gpt-5.5` and `OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT=minimal` by default, prioritizing response speed over maximum reasoning depth. Tune the server env file if a different balance is needed.
The managed service config sets query to `OBSIDIAN_MCP_QUERY_OPENCODE_MODEL=openai/gpt-5.5`, `OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT=minimal`, and `OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS=300` by default, prioritizing response speed over maximum reasoning depth. Tune the server env file if a different balance is needed.
`public_only` defaults to `false`, so the answer may use the same full vault visibility that a local operator sees. Set `public_only=true` when the remote answer must exclude `visibility/internal` and `visibility/pii` pages. Job state and logs live under `.runtime/mcp-query/jobs/`, outside the vault; remote query status/result responses include log tails but omit local log file paths. Query results keep the legacy `answer` field and also include `answer_markdown`, `citations`, `used_wiki`, `used_business_facts`, `data_freshness`, `warnings`, and `confidence` so agents can parse the response without scraping the log tail. A normal `wiki-query` log entry is allowed. The generated prompt forbids ingest/update/rebuild workflows, writes, sync/config/timer operations, and arbitrary shell execution; it only allows the `obsidian-wiki business schema/template/sql` read-only commands needed by the wiki-query business facts route. Hybrid business + wiki questions may run a wiki context track and a business facts track, using parallel subagents only when the host supports them and otherwise running sequentially. When `public_only=true`, business facts remain disabled.
`public_only` defaults to `false`, so the answer may use the same full vault visibility that a local operator sees. Set `public_only=true` when the remote answer must exclude `visibility/internal` and `visibility/pii` pages. Job state and logs live under `.runtime/mcp-query/jobs/`, outside the vault; remote query status/result responses expose bounded, path-redacted text and omit local log paths. The final JSON text event must contain `MCP_QUERY_OK`; `MCP_QUERY_FAILED`, a missing marker, a nonzero exit, timeout, or cancellation cannot be reported as a successful answer. Query results keep the legacy `answer` field and also include `answer_markdown`, `citations`, `used_wiki`, `used_business_facts`, `data_freshness`, `warnings`, and `confidence` so agents can parse the response without scraping raw logs. A normal `wiki-query` log entry is allowed. The generated prompt instructs the agent to avoid ingest/update/rebuild workflows, writes, sync/config/timer operations, and arbitrary shell execution; it only permits the `obsidian-wiki business schema/template/sql` read-only commands needed by the wiki-query business facts route, but these prompt restrictions do not replace the trusted-client boundary described above. Hybrid business + wiki questions may run a wiki context track and a business facts track, using parallel subagents only when the host supports them and otherwise running sequentially. When `public_only=true`, business facts remain disabled.
Clean old MCP runtime staging data with the default 30-day retention window:
+2 -2
View File
@@ -480,7 +480,7 @@ def search_assets(
query: str,
*,
db_path: Path | None = None,
limit: int = 10,
limit: int | None = 10,
doc_type: str | None = None,
source_type: str | None = None,
product_line: str | None = None,
@@ -527,7 +527,7 @@ def search_assets(
rows.append(record)
rows.sort(key=lambda item: (item["score"], item.get("updated_at") or ""), reverse=True)
return rows[:limit]
return rows if limit is None else rows[:limit]
def get_asset_by_file_id(
+36 -10
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import hashlib
import json
import mimetypes
import os
@@ -137,11 +138,12 @@ def search_file_assets(
vault = _vault_path(vault_path)
registry = load_registry_by_file_id(vault)
profile = _load_profile(vault, profile_name)
public_limit = min(max(limit, 1), 100)
rows = search_assets(
vault,
query,
db_path=db_path,
limit=max(limit * 3, limit),
limit=None,
doc_type=doc_type,
source_type=source_type,
product_line=product_line,
@@ -160,7 +162,7 @@ def search_file_assets(
if not ok:
continue
allowed.append(public_asset_record({**record, **row}, expose_paths=expose_paths))
if len(allowed) >= limit:
if len(allowed) >= public_limit:
break
return allowed
@@ -197,8 +199,11 @@ def read_file_asset_resource(
db_path: Path | None = None,
profile_name: str = DEFAULT_PROFILE_NAME,
max_file_bytes: int | None = None,
allow_source_files: bool = False,
) -> dict[str, Any]:
file_id = file_id_from_asset_uri(uri)
if not allow_source_files:
raise AssetAccessError("source file reads are disabled")
vault = _vault_path(vault_path)
row = get_asset_by_file_id(vault, file_id, db_path=db_path, read_only=True)
if row is None:
@@ -215,13 +220,27 @@ def read_file_asset_resource(
if not ok:
raise AssetAccessError(f"file asset blocked: {reason}")
path = safe_asset_path(vault, str(row.get("archive_path") or ""))
registry_archive_path = str(registry_record.get("archive_path") or "")
db_archive_path = str(row.get("archive_path") or "")
path = safe_asset_path(vault, registry_archive_path)
db_asset_path = safe_asset_path(vault, db_archive_path)
if path != db_asset_path:
raise AssetAccessError(
"file asset archive_path mismatch between source registry and index"
)
if not path.is_file():
raise AssetNotFoundError(f"archived file missing: {row.get('archive_path')}")
raise AssetNotFoundError(f"archived file missing: {registry_archive_path}")
limit = DEFAULT_MAX_FILE_BYTES if max_file_bytes is None else max_file_bytes
size = path.stat().st_size
if limit > 0 and size > limit:
raise AssetTooLargeError(f"file exceeds max size: {size} > {limit}")
content = path.read_bytes()
size = len(content)
if limit > 0 and size > limit:
raise AssetTooLargeError(f"file exceeds max size: {size} > {limit}")
digest = hashlib.sha256(content).hexdigest()
if digest != file_id.removeprefix("sha256:"):
raise AssetAccessError(f"file asset content hash mismatch: {file_id}")
name = str(row.get("original_name") or path.name)
return {
"file_id": file_id,
@@ -229,7 +248,7 @@ def read_file_asset_resource(
"name": name,
"mime_type": str(row.get("mime_type") or mimetypes.guess_type(name)[0] or "application/octet-stream"),
"size_bytes": size,
"bytes": path.read_bytes(),
"bytes": content,
}
@@ -283,7 +302,7 @@ def record_allowed_for_mcp(
return False, f"business category {category} blocked"
if profile is None:
return True, "allowed"
return False, "bot profile not found"
decision = source_decision(profile_name, _source_file_profile(profile, record), record)
if not decision.include:
return False, decision.reason
@@ -323,6 +342,7 @@ def mcp_runtime_options(
profile_name: str | None = None,
max_file_bytes: int | None = None,
expose_paths: bool | None = None,
allow_source_files: bool | None = None,
) -> dict[str, Any]:
return {
"profile_name": profile_name or config_value("OBSIDIAN_MCP_BOT_PROFILE", DEFAULT_PROFILE_NAME),
@@ -330,14 +350,20 @@ def mcp_runtime_options(
if max_file_bytes is None
else max_file_bytes,
"expose_paths": _env_flag("OBSIDIAN_MCP_EXPOSE_PATHS", False) if expose_paths is None else expose_paths,
"allow_source_files": _env_flag("OBSIDIAN_MCP_ALLOW_SOURCE_FILES", False)
if allow_source_files is None
else allow_source_files,
}
def _load_profile(vault: Path, profile_name: str) -> dict[str, Any] | None:
def _load_profile(vault: Path, profile_name: str) -> dict[str, Any]:
try:
return load_profiles(vault).get(profile_name)
except FileNotFoundError:
return None
profile = load_profiles(vault).get(profile_name)
except FileNotFoundError as exc:
raise AssetAccessError(f"bot profile not found: {profile_name}") from exc
if not isinstance(profile, dict):
raise AssetAccessError(f"bot profile not found: {profile_name}")
return profile
def _source_file_profile(profile: dict[str, Any], record: dict[str, Any]) -> dict[str, Any]:
+292 -82
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import base64
import binascii
import hashlib
import io
import json
import os
import re
@@ -20,21 +21,27 @@ from threading import RLock
from typing import Any
from obsidian_wiki.file_assets import clean_config_value, db_path_for_vault, registry_path_for_vault
from obsidian_wiki.mcp_jobs import ProcessOutcome, ProcessSupervisor, extract_final_answer, parse_marked_result
DEFAULT_INGEST_MAX_FILE_BYTES = 100 * 1024 * 1024
DEFAULT_INGEST_MAX_CHUNK_BYTES = 4 * 1024 * 1024
DEFAULT_INGEST_MAX_FILES_PER_JOB = 10
DEFAULT_INGEST_MAX_REQUEST_BYTES = 0
DEFAULT_UPLOAD_TICKET_SECONDS = 30 * 60
MAX_UPLOAD_TICKET_SECONDS = 24 * 60 * 60
DEFAULT_OPENCODE_PATH = "opencode"
DEFAULT_INGEST_OPENCODE_MODEL = "openai/gpt-5.5"
DEFAULT_INGEST_OPENCODE_VARIANT = "high"
DEFAULT_OPENCODE_SMOKE_TIMEOUT_SECONDS = 120
DEFAULT_INGEST_JOB_TIMEOUT_SECONDS = 30 * 60
STREAM_COPY_CHUNK_BYTES = 1024 * 1024
SHA256_RE = re.compile(r"^(?:sha256:)?([0-9a-fA-F]{64})$")
OPENCODE_REQUIRED_RUN_FLAGS = ("--format", "--auto", "--dir", "--title")
OPENCODE_SMOKE_SUCCESS = "WIKI_INGEST_SKILL_LOADED"
OPENCODE_SMOKE_FAILURE = "SKILL_LOAD_FAILED"
INGEST_SUCCESS_MARKER = "MCP_INGEST_OK"
INGEST_FAILURE_MARKER = "MCP_INGEST_FAILED"
TERMINAL_JOB_STATES = {"succeeded", "failed", "cancelled"}
@@ -73,6 +80,8 @@ class IngestManager:
max_file_bytes: int = DEFAULT_INGEST_MAX_FILE_BYTES,
max_chunk_bytes: int = DEFAULT_INGEST_MAX_CHUNK_BYTES,
max_files_per_job: int = DEFAULT_INGEST_MAX_FILES_PER_JOB,
max_request_bytes: int = DEFAULT_INGEST_MAX_REQUEST_BYTES,
job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
) -> None:
self.vault = Path(clean_config_value(str(vault_path))).expanduser().resolve()
self.project_root = Path(clean_config_value(str(project_root))).expanduser().resolve()
@@ -88,14 +97,20 @@ class IngestManager:
self.max_file_bytes = max_file_bytes
self.max_chunk_bytes = max_chunk_bytes
self.max_files_per_job = max_files_per_job
self.max_request_bytes = max(0, int(max_request_bytes))
self.job_timeout_seconds = max(1, int(job_timeout_seconds))
self.upload_root = self.runtime_root / "uploads"
self.jobs_root = self.runtime_root / "jobs"
self.upload_ticket_root = self.runtime_root / "upload-tickets"
self._lock = RLock()
self._processes: dict[str, subprocess.Popen[Any]] = {}
self._closing = False
self.upload_root.mkdir(parents=True, exist_ok=True)
self.jobs_root.mkdir(parents=True, exist_ok=True)
self.upload_ticket_root.mkdir(parents=True, exist_ok=True)
self._supervisor = ProcessSupervisor(
self.runtime_root,
popen_factory=lambda *args, **kwargs: subprocess.Popen(*args, **kwargs),
)
self._mark_interrupted_jobs()
def create_upload(
@@ -143,23 +158,80 @@ class IngestManager:
doc_type_hint: str | None = None,
) -> dict[str, Any]:
data = bytes(payload)
digest = hashlib.sha256(data).hexdigest()
return self.create_completed_upload_from_stream(
file_name=file_name,
stream=io.BytesIO(data),
size_bytes=len(data),
project=project,
doc_type_hint=doc_type_hint,
)
def create_completed_upload_from_stream(
self,
file_name: str,
stream: Any,
*,
size_bytes: int | None = None,
project: str | None = None,
doc_type_hint: str | None = None,
) -> dict[str, Any]:
name = self._validate_file_name(file_name)
declared_size = None if size_bytes is None else int(size_bytes)
if declared_size is not None and declared_size < 0:
raise IngestError("size_bytes must be non-negative")
if declared_size is not None and self.max_file_bytes > 0 and declared_size > self.max_file_bytes:
raise IngestError(f"file exceeds max upload size: {declared_size} > {self.max_file_bytes}")
with self._lock:
upload = self.create_upload(
file_name=file_name,
size_bytes=len(data),
sha256=digest,
project=project,
doc_type_hint=doc_type_hint,
)
meta = self._load_upload(upload["upload_id"])
path = Path(meta["path"])
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
meta["received_bytes"] = len(data)
meta["updated_at"] = utc_now()
self._write_json(self._upload_meta_path(path.parent), meta)
return self.complete_upload(upload["upload_id"])
upload_id = uuid.uuid4().hex
month = datetime.now(timezone.utc).strftime("%Y-%m")
upload_dir = self.upload_root / month / upload_id
upload_dir.mkdir(parents=True, exist_ok=False)
path = upload_dir / name
received = 0
hasher = hashlib.sha256()
try:
with path.open("xb") as destination:
while True:
chunk = stream.read(STREAM_COPY_CHUNK_BYTES)
if not chunk:
break
if not isinstance(chunk, (bytes, bytearray, memoryview)):
raise IngestError("upload stream must return bytes")
data = bytes(chunk)
received += len(data)
if self.max_file_bytes > 0 and received > self.max_file_bytes:
raise IngestError(
f"file exceeds max upload size: {received} > {self.max_file_bytes}"
)
if declared_size is not None and received > declared_size:
raise IngestError(
f"upload size mismatch: received more than declared {declared_size} bytes"
)
destination.write(data)
hasher.update(data)
if declared_size is not None and received != declared_size:
raise IngestError(f"upload size mismatch: {received} != {declared_size}")
now = utc_now()
meta = {
"upload_id": upload_id,
"state": "complete",
"file_name": name,
"size_bytes": received,
"received_bytes": received,
"sha256": hasher.hexdigest(),
"project": self._safe_optional_text(project),
"doc_type_hint": self._safe_optional_text(doc_type_hint),
"path": str(path),
"created_at": now,
"completed_at": now,
"updated_at": now,
}
self._write_json(self._upload_meta_path(upload_dir), meta)
return self._public_upload(meta)
except Exception:
shutil.rmtree(upload_dir, ignore_errors=True)
raise
def create_upload_ticket(
self,
@@ -312,16 +384,53 @@ class IngestManager:
self._write_json(self._upload_meta_path(path.parent), meta)
return self._public_upload(meta)
def discard_upload(self, upload_id: str) -> bool:
with self._lock:
if any(upload_id in job.get("upload_ids", []) for job in self._iter_jobs()):
return False
meta = self._load_upload(upload_id)
upload_dir = Path(meta["path"]).parent.resolve()
if not path_is_relative_to(upload_dir, self.upload_root.resolve()):
raise IngestError(f"upload path escapes runtime root: {upload_id}")
shutil.rmtree(upload_dir)
return True
def start_ingest_job(self, upload_ids: list[str], note: str | None = None) -> dict[str, Any]:
with self._lock:
if self._closing:
raise IngestError("ingest manager is closed")
if not upload_ids:
raise IngestError("upload_ids must not be empty")
if len(set(upload_ids)) != len(upload_ids):
raise IngestError("upload_ids must be unique")
if len(upload_ids) > self.max_files_per_job:
raise IngestError(f"too many files for one ingest job: {len(upload_ids)} > {self.max_files_per_job}")
uploads = [self._load_upload(upload_id) for upload_id in upload_ids]
for upload in uploads:
if upload["state"] != "complete":
raise IngestError(f"upload is not complete: {upload['upload_id']}")
requested_upload_ids = tuple(upload_ids)
claimed_jobs = [
job
for job in self._iter_jobs()
if job.get("state") in {"queued", "running", "succeeded"}
and set(upload_ids).intersection(job.get("upload_ids", []))
]
for claimed in claimed_jobs:
if tuple(claimed.get("upload_ids", [])) == requested_upload_ids:
return self._public_job(claimed)
if claimed_jobs:
claimed_ids = sorted(
{
upload_id
for job in claimed_jobs
for upload_id in job.get("upload_ids", [])
if upload_id in upload_ids
}
)
raise IngestError(
"uploads are already assigned to another ingest job: " + ", ".join(claimed_ids)
)
job_id = uuid.uuid4().hex
job_dir = self.jobs_root / job_id
job_dir.mkdir(parents=True, exist_ok=False)
@@ -331,6 +440,7 @@ class IngestManager:
"upload_ids": upload_ids,
"files": [self._public_upload(upload, expose_path=True) for upload in uploads],
"note": self._safe_optional_text(note),
"queue_sequence": time.time_ns(),
"created_at": utc_now(),
"updated_at": utc_now(),
"started_at": None,
@@ -343,7 +453,7 @@ class IngestManager:
}
self._write_job(job)
self._start_next_job_locked()
return self.get_job_status(job_id)
return self.get_job_status(job_id)
def check_opencode_readiness(self, *, run_smoke_test: bool = False) -> dict[str, Any]:
result: dict[str, Any] = {
@@ -367,8 +477,8 @@ class IngestManager:
return result
def get_job_status(self, job_id: str) -> dict[str, Any]:
self.refresh_jobs()
with self._lock:
self.refresh_jobs()
job = self._load_job(job_id)
return self._public_job(job)
@@ -376,17 +486,16 @@ class IngestManager:
return self.get_job_status(job_id)
def get_ingest_job_result(self, job_id: str) -> dict[str, Any]:
with self._lock:
status = self.get_job_status(job_id)
return {
"job_id": status["job_id"],
"state": status["state"],
"summary": self._summary_for_status(status),
"files": status["files"],
"exit_code": status.get("exit_code"),
"opencode_output_tail": status.get("log_tail", ""),
"error": status.get("error", ""),
}
status = self.get_job_status(job_id)
return {
"job_id": status["job_id"],
"state": status["state"],
"summary": self._summary_for_status(status),
"files": status["files"],
"exit_code": status.get("exit_code"),
"opencode_output_tail": status.get("log_tail", ""),
"error": status.get("error", ""),
}
def cancel_ingest_job(self, job_id: str) -> dict[str, Any]:
with self._lock:
@@ -398,33 +507,32 @@ class IngestManager:
self._write_job(job)
return self._public_job(job)
if job["state"] == "running":
process = self._processes.get(job_id)
if process is not None:
process.terminate()
job["state"] = "cancelled"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["exit_code"] = -15
self._write_job(job)
self._processes.pop(job_id, None)
self._start_next_job_locked()
return self._public_job(job)
self._supervisor.cancel(job_id)
return self._public_job(self._load_job(job_id))
return self._public_job(job)
def refresh_jobs(self) -> None:
self._supervisor.refresh()
def close(self) -> None:
with self._lock:
for job_id, process in list(self._processes.items()):
code = process.poll()
if code is None:
continue
job = self._load_job(job_id)
job["state"] = "succeeded" if code == 0 else "failed"
job["exit_code"] = code
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
self._processes.pop(job_id, None)
self._start_next_job_locked()
if self._closing:
cancel_queued = False
else:
self._closing = True
cancel_queued = True
try:
if cancel_queued:
with self._lock:
for job in self._iter_jobs():
if job.get("state") == "queued":
job["state"] = "cancelled"
job["error"] = "ingest manager is shutting down"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
finally:
self._supervisor.close()
def cleanup_runtime(self, *, max_age_days: int = 30) -> dict[str, Any]:
if max_age_days <= 0:
@@ -437,6 +545,7 @@ class IngestManager:
"old_uploads_deleted": 0,
"old_jobs_deleted": 0,
}
self.refresh_jobs()
with self._lock:
active_upload_ids = {
str(upload_id)
@@ -486,16 +595,26 @@ class IngestManager:
return result
def _start_next_job_locked(self) -> None:
if any(job.get("state") == "running" for job in self._iter_jobs()):
if self._closing:
return
queued = [job for job in self._iter_jobs() if job.get("state") == "queued"]
if not queued:
return
queued.sort(key=lambda job: job["created_at"])
job = queued[0]
self._launch_job_locked(job)
while True:
jobs = self._iter_jobs()
if any(job.get("state") == "running" for job in jobs):
return
queued = [job for job in jobs if job.get("state") == "queued"]
if not queued:
return
queued.sort(
key=lambda job: (
job.get("created_at", ""),
int(job.get("queue_sequence") or 0),
job.get("job_id", ""),
)
)
if self._launch_job_locked(queued[0]):
return
def _launch_job_locked(self, job: dict[str, Any]) -> None:
def _launch_job_locked(self, job: dict[str, Any]) -> bool:
readiness = self.check_opencode_readiness()
job["preflight"] = readiness
if not readiness["ok"]:
@@ -504,7 +623,7 @@ class IngestManager:
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return
return False
prompt = self._build_prompt(job)
args = [
self.opencode_path,
@@ -519,31 +638,83 @@ class IngestManager:
f"MCP ingest {job['job_id']}",
prompt,
]
log_path = Path(job["log_path"])
log_path.parent.mkdir(parents=True, exist_ok=True)
log_file = log_path.open("ab")
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
try:
process = subprocess.Popen(
process = self._supervisor.start(
job["job_id"],
args,
cwd=str(self.project_root),
stdout=log_file,
stderr=subprocess.STDOUT,
log_path=job["log_path"],
timeout_seconds=self.job_timeout_seconds,
on_complete=self._handle_process_outcome,
on_callback_error=self._handle_process_callback_error,
)
except OSError as exc:
log_file.close()
except (OSError, RuntimeError, ValueError) as exc:
job["state"] = "failed"
job["error"] = str(exc)
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
job["pid"] = process.pid
self._processes[job["job_id"]] = process
self._write_job(job)
log_file.close()
return False
current = self._load_job(job["job_id"])
if current.get("state") == "running":
current["pid"] = process.pid
self._write_job(current)
return True
def _handle_process_callback_error(self, outcome: ProcessOutcome, error: Exception) -> None:
with self._lock:
try:
job = self._load_job(outcome.job_id)
except IngestError:
return
if job.get("state") == "running":
job["state"] = "failed"
job["exit_code"] = outcome.returncode
job["error"] = f"ingest completion handling failed: {type(error).__name__}: {error}"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
self._start_next_job_locked()
def _handle_process_outcome(self, outcome: ProcessOutcome) -> None:
with self._lock:
try:
job = self._load_job(outcome.job_id)
except IngestError:
return
if job.get("state") != "running":
return
final_text = extract_final_answer(job["log_path"])
protocol_ok, result_text, protocol_error = parse_marked_result(
final_text,
success_marker=INGEST_SUCCESS_MARKER,
failure_marker=INGEST_FAILURE_MARKER,
)
job["exit_code"] = outcome.returncode
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["result_text"] = result_text
if outcome.cancelled:
job["state"] = "cancelled"
job["error"] = outcome.error
elif outcome.timed_out:
job["state"] = "failed"
job["error"] = f"OpenCode ingest exceeded {self.job_timeout_seconds} seconds"
elif outcome.returncode != 0:
job["state"] = "failed"
job["error"] = outcome.error or protocol_error or f"OpenCode exited with code {outcome.returncode}"
elif not protocol_ok:
job["state"] = "failed"
job["error"] = protocol_error
else:
job["state"] = "succeeded"
job["error"] = ""
self._write_job(job)
self._start_next_job_locked()
def _build_prompt(self, job: dict[str, Any]) -> str:
file_lines = []
@@ -582,6 +753,8 @@ class IngestManager:
"",
"After ingesting, archive sources into the vault source archive, update source registry/catalog/wiki metadata as normal, and run `obsidian-wiki file-index rebuild`.",
"Return a concise success or failure summary, including whether the file asset index was refreshed.",
f"If ingest succeeds, the final response must start with {INGEST_SUCCESS_MARKER} on its own line, followed by the summary.",
f"If ingest fails, the final response must start with {INGEST_FAILURE_MARKER} on its own line, followed by the reason.",
]
)
@@ -752,6 +925,15 @@ class IngestManager:
return duplicates
def _public_job(self, job: dict[str, Any]) -> dict[str, Any]:
output = str(job.get("result_text") or extract_final_answer(job["log_path"]))
if output.startswith((INGEST_SUCCESS_MARKER, INGEST_FAILURE_MARKER)):
_ok, public_text, error = parse_marked_result(
output,
success_marker=INGEST_SUCCESS_MARKER,
failure_marker=INGEST_FAILURE_MARKER,
)
output = public_text or error
output = self._redact_paths(output)
return {
"job_id": job["job_id"],
"state": job["state"],
@@ -760,12 +942,40 @@ class IngestManager:
"started_at": job.get("started_at"),
"finished_at": job.get("finished_at"),
"exit_code": job.get("exit_code"),
"pid": job.get("pid"),
"error": job.get("error", ""),
"preflight": job.get("preflight"),
"log_tail": self._read_tail(Path(job["log_path"])),
"error": self._redact_paths(str(job.get("error", ""))),
"preflight": self._public_preflight(job.get("preflight")),
"log_tail": output,
}
def _public_preflight(self, preflight: dict[str, Any] | None) -> dict[str, Any] | None:
if preflight is None:
return None
return {
key: self._redact_public_value(value)
for key, value in preflight.items()
if key != "checked_paths"
}
def _redact_public_value(self, value: Any) -> Any:
if isinstance(value, dict):
return {key: self._redact_public_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [self._redact_public_value(item) for item in value]
if isinstance(value, str):
return self._redact_paths(value)
return value
def _redact_paths(self, value: str) -> str:
text = value
for path, label in (
(self.runtime_root, "<runtime>"),
(self.vault, "<vault>"),
(self.project_root, "<project>"),
(Path.home(), "<home>"),
):
text = text.replace(str(path), label)
return text
def _tree_mtime(self, path: Path) -> float:
if not path.exists():
return time.time()
+331
View File
@@ -0,0 +1,331 @@
"""Shared process supervision for OpenCode-backed MCP jobs."""
from __future__ import annotations
import json
import os
import signal
import subprocess
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
class RuntimeLeaseError(RuntimeError):
"""Raised when another manager already owns a runtime directory."""
@dataclass(frozen=True)
class ProcessOutcome:
job_id: str
returncode: int
cancelled: bool = False
timed_out: bool = False
error: str = ""
@dataclass
class _ProcessEntry:
job_id: str
process: subprocess.Popen[Any]
deadline: float | None
on_complete: Callable[[ProcessOutcome], None]
on_callback_error: Callable[[ProcessOutcome, Exception], None] | None = None
completing: bool = False
class _RuntimeLease:
def __init__(self, runtime_root: Path) -> None:
runtime_root.mkdir(parents=True, exist_ok=True)
self.path = runtime_root / ".manager.lock"
self.handle = self.path.open("a+b")
try:
self._lock()
except OSError as exc:
self.handle.close()
raise RuntimeLeaseError(f"MCP runtime is already in use: {runtime_root}") from exc
def _lock(self) -> None:
if os.name == "nt": # pragma: no cover - exercised on Windows
import msvcrt
self.handle.seek(0)
if self.handle.read(1) == b"":
self.handle.write(b"0")
self.handle.flush()
self.handle.seek(0)
msvcrt.locking(self.handle.fileno(), msvcrt.LK_NBLCK, 1)
return
import fcntl
fcntl.flock(self.handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
def close(self) -> None:
if self.handle.closed:
return
try:
if os.name == "nt": # pragma: no cover - exercised on Windows
import msvcrt
self.handle.seek(0)
msvcrt.locking(self.handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(self.handle.fileno(), fcntl.LOCK_UN)
finally:
self.handle.close()
class ProcessSupervisor:
"""Own subprocesses, enforce deadlines, and deliver one completion callback."""
def __init__(
self,
runtime_root: str | Path,
*,
popen_factory: Callable[..., subprocess.Popen[Any]] = subprocess.Popen,
poll_interval_seconds: float = 0.1,
) -> None:
self.runtime_root = Path(runtime_root).expanduser().resolve()
self._lease = _RuntimeLease(self.runtime_root)
self._popen_factory = popen_factory
self._poll_interval = max(0.01, float(poll_interval_seconds))
self._entries: dict[str, _ProcessEntry] = {}
self._lock = threading.RLock()
self._condition = threading.Condition(self._lock)
self._callbacks_in_progress = 0
self._wake = threading.Event()
self._closed = False
self._thread = threading.Thread(target=self._monitor, name="mcp-job-supervisor", daemon=True)
self._thread.start()
def start(
self,
job_id: str,
args: list[str],
*,
cwd: str | Path,
log_path: str | Path,
timeout_seconds: float,
on_complete: Callable[[ProcessOutcome], None],
on_callback_error: Callable[[ProcessOutcome, Exception], None] | None = None,
) -> subprocess.Popen[Any]:
with self._lock:
if self._closed:
raise RuntimeError("process supervisor is closed")
if job_id in self._entries:
raise ValueError(f"job is already supervised: {job_id}")
log = Path(log_path)
log.parent.mkdir(parents=True, exist_ok=True)
log_file = log.open("ab")
kwargs: dict[str, Any] = {
"cwd": str(cwd),
"stdout": log_file,
"stderr": subprocess.STDOUT,
}
if os.name == "nt": # pragma: no cover - exercised on Windows
kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
else:
kwargs["start_new_session"] = True
try:
process = self._popen_factory(args, **kwargs)
finally:
log_file.close()
deadline = time.monotonic() + float(timeout_seconds) if timeout_seconds > 0 else None
self._entries[job_id] = _ProcessEntry(
job_id,
process,
deadline,
on_complete,
on_callback_error,
)
self._wake.set()
return process
def is_running(self, job_id: str | None = None) -> bool:
with self._lock:
if job_id is None:
return bool(self._entries)
return job_id in self._entries
def refresh(self) -> None:
for entry in self._snapshot():
code = entry.process.poll()
if code is not None:
self._complete_naturally(entry.job_id, code)
elif entry.deadline is not None and time.monotonic() >= entry.deadline:
self._complete_stopped(entry.job_id, timed_out=True, grace_seconds=1.0)
deadline = time.monotonic() + 1.0
with self._condition:
while self._callbacks_in_progress and time.monotonic() < deadline:
self._condition.wait(deadline - time.monotonic())
def cancel(self, job_id: str, *, grace_seconds: float = 5.0) -> bool:
return self._complete_stopped(job_id, cancelled=True, grace_seconds=grace_seconds)
def close(self) -> None:
with self._lock:
if self._closed:
return
self._closed = True
job_ids = list(self._entries)
for job_id in job_ids:
self._complete_stopped(job_id, cancelled=True, grace_seconds=0.2)
self._wake.set()
if threading.current_thread() is not self._thread:
self._thread.join(timeout=1)
self._lease.close()
def _snapshot(self) -> list[_ProcessEntry]:
with self._lock:
return [entry for entry in self._entries.values() if not entry.completing]
def _begin_completion(self, job_id: str) -> _ProcessEntry | None:
with self._lock:
entry = self._entries.get(job_id)
if entry is None or entry.completing:
return None
entry.completing = True
self._callbacks_in_progress += 1
return entry
def _complete_naturally(self, job_id: str, returncode: int) -> bool:
entry = self._begin_completion(job_id)
if entry is None:
return False
self._finish(entry, ProcessOutcome(job_id=job_id, returncode=int(returncode)))
return True
def _complete_stopped(
self,
job_id: str,
*,
cancelled: bool = False,
timed_out: bool = False,
grace_seconds: float,
) -> bool:
entry = self._begin_completion(job_id)
if entry is None:
return False
returncode, error = self._terminate_process(entry.process, grace_seconds=grace_seconds)
self._finish(
entry,
ProcessOutcome(
job_id=job_id,
returncode=returncode,
cancelled=cancelled,
timed_out=timed_out,
error=error,
),
)
return True
def _terminate_process(self, process: subprocess.Popen[Any], *, grace_seconds: float) -> tuple[int, str]:
error = ""
if process.poll() is None:
try:
if os.name != "nt":
os.killpg(process.pid, signal.SIGTERM)
else: # pragma: no cover - exercised on Windows
process.terminate()
except (OSError, ProcessLookupError):
try:
process.terminate()
except OSError as exc:
error = str(exc)
try:
process.wait(timeout=max(0, grace_seconds))
except subprocess.TimeoutExpired:
try:
if os.name != "nt":
os.killpg(process.pid, signal.SIGKILL)
else: # pragma: no cover - exercised on Windows
process.kill()
except (OSError, ProcessLookupError):
try:
process.kill()
except OSError as exc:
error = error or str(exc)
try:
process.wait(timeout=1)
except subprocess.TimeoutExpired:
error = error or "process did not exit after SIGKILL"
code = process.poll()
return (int(code) if code is not None else -9), error
def _finish(self, entry: _ProcessEntry, outcome: ProcessOutcome) -> None:
with self._lock:
current = self._entries.get(entry.job_id)
if current is not entry:
return
self._entries.pop(entry.job_id, None)
try:
entry.on_complete(outcome)
except Exception as exc:
if entry.on_callback_error is not None:
try:
entry.on_callback_error(outcome, exc)
except Exception:
pass
finally:
with self._condition:
self._callbacks_in_progress -= 1
self._condition.notify_all()
self._wake.set()
def _monitor(self) -> None:
while True:
with self._lock:
if self._closed and not self._entries:
return
self.refresh()
self._wake.wait(self._poll_interval)
self._wake.clear()
def extract_final_answer(path: str | Path, *, limit: int = 12000) -> str:
"""Extract only assistant text events, preferring the final-answer phase."""
log_path = Path(path)
if not log_path.is_file():
return ""
last_text = ""
final_text = ""
with log_path.open("r", encoding="utf-8", errors="replace") as handle:
for raw in handle:
try:
event = json.loads(raw)
except json.JSONDecodeError:
continue
if event.get("type") != "text":
continue
part = event.get("part") or {}
text = str(part.get("text") or "")
if not text:
continue
last_text = text
phase = ((part.get("metadata") or {}).get("openai") or {}).get("phase")
if phase == "final_answer":
final_text = text
text = final_text or last_text
if limit <= 0 or len(text) <= limit:
return text if limit > 0 else ""
first_line, separator, remainder = text.partition("\n")
if not separator or len(first_line) + 1 >= limit:
return text[:limit]
tail_size = limit - len(first_line) - 1
return first_line + "\n" + remainder[-tail_size:]
def parse_marked_result(text: str, *, success_marker: str, failure_marker: str) -> tuple[bool, str, str]:
"""Return protocol success, public text, and an error string."""
value = str(text or "").strip()
first, _, remainder = value.partition("\n")
if first.strip() == success_marker:
return True, remainder.strip(), ""
if first.strip() == failure_marker:
error = remainder.strip() or f"OpenCode returned {failure_marker}"
return False, "", error
return False, "", "OpenCode final answer did not include the required result marker"
+173 -60
View File
@@ -15,17 +15,21 @@ from threading import RLock
from typing import Any
from obsidian_wiki.file_assets import clean_config_value
from obsidian_wiki.mcp_jobs import ProcessOutcome, ProcessSupervisor, extract_final_answer, parse_marked_result
DEFAULT_QUERY_MAX_CHARS = 8000
DEFAULT_OPENCODE_PATH = "opencode"
DEFAULT_QUERY_OPENCODE_MODEL = "openai/gpt-5.5"
DEFAULT_QUERY_OPENCODE_VARIANT = "minimal"
DEFAULT_OPENCODE_SMOKE_TIMEOUT_SECONDS = 120
DEFAULT_QUERY_JOB_TIMEOUT_SECONDS = 5 * 60
QUERY_MODES = {"normal", "index_only"}
OPENCODE_REQUIRED_RUN_FLAGS = ("--format", "--auto", "--dir", "--title")
OPENCODE_SMOKE_SUCCESS = "WIKI_QUERY_SKILL_LOADED"
OPENCODE_SMOKE_FAILURE = "SKILL_LOAD_FAILED"
QUERY_SUCCESS_MARKER = "MCP_QUERY_OK"
QUERY_FAILURE_MARKER = "MCP_QUERY_FAILED"
TERMINAL_JOB_STATES = {"succeeded", "failed", "cancelled"}
@@ -56,6 +60,7 @@ class QueryManager:
opencode_model: str | None = DEFAULT_QUERY_OPENCODE_MODEL,
opencode_variant: str | None = DEFAULT_QUERY_OPENCODE_VARIANT,
max_query_chars: int = DEFAULT_QUERY_MAX_CHARS,
job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
) -> None:
self.vault = Path(clean_config_value(str(vault_path))).expanduser().resolve()
self.project_root = Path(clean_config_value(str(project_root))).expanduser().resolve()
@@ -69,10 +74,15 @@ class QueryManager:
self.opencode_model = clean_config_value(str(opencode_model or "")).strip()
self.opencode_variant = clean_config_value(str(opencode_variant or "")).strip()
self.max_query_chars = max_query_chars
self.job_timeout_seconds = max(1, int(job_timeout_seconds))
self.jobs_root = self.runtime_root / "jobs"
self._lock = RLock()
self._processes: dict[str, subprocess.Popen[Any]] = {}
self._closing = False
self.jobs_root.mkdir(parents=True, exist_ok=True)
self._supervisor = ProcessSupervisor(
self.runtime_root,
popen_factory=lambda *args, **kwargs: subprocess.Popen(*args, **kwargs),
)
self._mark_interrupted_jobs()
def start_query_job(
@@ -86,6 +96,8 @@ class QueryManager:
text = self._validate_query(query)
query_mode = self._validate_mode(mode)
with self._lock:
if self._closing:
raise QueryError("query manager is closed")
job_id = uuid.uuid4().hex
job_dir = self.jobs_root / job_id
job_dir.mkdir(parents=True, exist_ok=False)
@@ -96,6 +108,7 @@ class QueryManager:
"mode": query_mode,
"public_only": bool(public_only),
"note": self._safe_optional_text(note),
"queue_sequence": time.time_ns(),
"created_at": utc_now(),
"updated_at": utc_now(),
"started_at": None,
@@ -108,7 +121,7 @@ class QueryManager:
}
self._write_job(job)
self._start_next_job_locked()
return self.get_query_job_status(job_id)
return self.get_query_job_status(job_id)
def check_opencode_readiness(self, *, run_smoke_test: bool = False) -> dict[str, Any]:
result: dict[str, Any] = {
@@ -132,15 +145,14 @@ class QueryManager:
return result
def get_query_job_status(self, job_id: str) -> dict[str, Any]:
self.refresh_jobs()
with self._lock:
self.refresh_jobs()
job = self._load_job(job_id)
return self._public_job(job)
def get_query_job_result(self, job_id: str) -> dict[str, Any]:
with self._lock:
status = self.get_query_job_status(job_id)
return self._query_result_from_status(status)
status = self.get_query_job_status(job_id)
return self._query_result_from_status(status)
def ask_wiki(
self,
@@ -179,33 +191,32 @@ class QueryManager:
self._write_job(job)
return self._public_job(job)
if job["state"] == "running":
process = self._processes.get(job_id)
if process is not None:
process.terminate()
job["state"] = "cancelled"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["exit_code"] = -15
self._write_job(job)
self._processes.pop(job_id, None)
self._start_next_job_locked()
return self._public_job(job)
self._supervisor.cancel(job_id)
return self._public_job(self._load_job(job_id))
return self._public_job(job)
def refresh_jobs(self) -> None:
self._supervisor.refresh()
def close(self) -> None:
with self._lock:
for job_id, process in list(self._processes.items()):
code = process.poll()
if code is None:
continue
job = self._load_job(job_id)
job["state"] = "succeeded" if code == 0 else "failed"
job["exit_code"] = code
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
self._processes.pop(job_id, None)
self._start_next_job_locked()
if self._closing:
cancel_queued = False
else:
self._closing = True
cancel_queued = True
try:
if cancel_queued:
with self._lock:
for job in self._iter_jobs():
if job.get("state") == "queued":
job["state"] = "cancelled"
job["error"] = "query manager is shutting down"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
finally:
self._supervisor.close()
def cleanup_runtime(self, *, max_age_days: int = 30) -> dict[str, Any]:
if max_age_days <= 0:
@@ -216,8 +227,8 @@ class QueryManager:
"max_age_days": max_age_days,
"old_jobs_deleted": 0,
}
self.refresh_jobs()
with self._lock:
self.refresh_jobs()
for job in self._iter_jobs():
if job.get("state") not in TERMINAL_JOB_STATES:
continue
@@ -232,15 +243,26 @@ class QueryManager:
return result
def _start_next_job_locked(self) -> None:
if any(job.get("state") == "running" for job in self._iter_jobs()):
if self._closing:
return
queued = [job for job in self._iter_jobs() if job.get("state") == "queued"]
if not queued:
return
queued.sort(key=lambda job: job["created_at"])
self._launch_job_locked(queued[0])
while True:
jobs = self._iter_jobs()
if any(job.get("state") == "running" for job in jobs):
return
queued = [job for job in jobs if job.get("state") == "queued"]
if not queued:
return
queued.sort(
key=lambda job: (
job.get("created_at", ""),
int(job.get("queue_sequence") or 0),
job.get("job_id", ""),
)
)
if self._launch_job_locked(queued[0]):
return
def _launch_job_locked(self, job: dict[str, Any]) -> None:
def _launch_job_locked(self, job: dict[str, Any]) -> bool:
readiness = self.check_opencode_readiness()
job["preflight"] = readiness
if not readiness["ok"]:
@@ -249,12 +271,12 @@ class QueryManager:
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return
return False
args = [
self.opencode_path,
"run",
"--format",
"default",
"json",
"--auto",
*self._opencode_model_args(),
"--dir",
@@ -263,31 +285,83 @@ class QueryManager:
f"MCP query {job['job_id']}",
self._build_prompt(job),
]
log_path = Path(job["log_path"])
log_path.parent.mkdir(parents=True, exist_ok=True)
log_file = log_path.open("ab")
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
try:
process = subprocess.Popen(
process = self._supervisor.start(
job["job_id"],
args,
cwd=str(self.project_root),
stdout=log_file,
stderr=subprocess.STDOUT,
log_path=job["log_path"],
timeout_seconds=self.job_timeout_seconds,
on_complete=self._handle_process_outcome,
on_callback_error=self._handle_process_callback_error,
)
except OSError as exc:
log_file.close()
except (OSError, RuntimeError, ValueError) as exc:
job["state"] = "failed"
job["error"] = str(exc)
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
return
job["state"] = "running"
job["started_at"] = utc_now()
job["updated_at"] = utc_now()
job["pid"] = process.pid
self._processes[job["job_id"]] = process
self._write_job(job)
log_file.close()
return False
current = self._load_job(job["job_id"])
if current.get("state") == "running":
current["pid"] = process.pid
self._write_job(current)
return True
def _handle_process_callback_error(self, outcome: ProcessOutcome, error: Exception) -> None:
with self._lock:
try:
job = self._load_job(outcome.job_id)
except QueryError:
return
if job.get("state") == "running":
job["state"] = "failed"
job["exit_code"] = outcome.returncode
job["error"] = f"query completion handling failed: {type(error).__name__}: {error}"
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
self._write_job(job)
self._start_next_job_locked()
def _handle_process_outcome(self, outcome: ProcessOutcome) -> None:
with self._lock:
try:
job = self._load_job(outcome.job_id)
except QueryError:
return
if job.get("state") != "running":
return
final_text = extract_final_answer(job["log_path"])
protocol_ok, answer, protocol_error = parse_marked_result(
final_text,
success_marker=QUERY_SUCCESS_MARKER,
failure_marker=QUERY_FAILURE_MARKER,
)
job["exit_code"] = outcome.returncode
job["finished_at"] = utc_now()
job["updated_at"] = utc_now()
job["result_text"] = answer
if outcome.cancelled:
job["state"] = "cancelled"
job["error"] = outcome.error
elif outcome.timed_out:
job["state"] = "failed"
job["error"] = f"OpenCode query exceeded {self.job_timeout_seconds} seconds"
elif outcome.returncode != 0:
job["state"] = "failed"
job["error"] = outcome.error or protocol_error or f"OpenCode exited with code {outcome.returncode}"
elif not protocol_ok:
job["state"] = "failed"
job["error"] = protocol_error
else:
job["state"] = "succeeded"
job["error"] = ""
self._write_job(job)
self._start_next_job_locked()
def _build_prompt(self, job: dict[str, Any]) -> str:
mode = "index-only" if job.get("mode") == "index_only" else "normal"
@@ -313,6 +387,8 @@ class QueryManager:
"Business SQL must follow the schema guardrails: only SELECT/WITH SELECT against documented structured tables, never raw_json, raw_snapshots, writes, DDL, PRAGMA, or sync/config operations.",
"If Visibility is public only, do not use business facts; continue with filtered wiki retrieval.",
"Return the final synthesized answer with wiki citations when available.",
f"If the query succeeds, the final response must start with {QUERY_SUCCESS_MARKER} on its own line, followed by the answer.",
f"If it fails, the final response must start with {QUERY_FAILURE_MARKER} on its own line, followed by the reason.",
"",
f"Optional context note: {note}",
]
@@ -348,6 +424,15 @@ class QueryManager:
self._write_job(job)
def _public_job(self, job: dict[str, Any]) -> dict[str, Any]:
output = str(job.get("result_text") or extract_final_answer(job["log_path"]))
if output.startswith((QUERY_SUCCESS_MARKER, QUERY_FAILURE_MARKER)):
_ok, public_text, error = parse_marked_result(
output,
success_marker=QUERY_SUCCESS_MARKER,
failure_marker=QUERY_FAILURE_MARKER,
)
output = public_text or error
output = self._redact_paths(output)
return {
"job_id": job["job_id"],
"state": job["state"],
@@ -359,12 +444,40 @@ class QueryManager:
"started_at": job.get("started_at"),
"finished_at": job.get("finished_at"),
"exit_code": job.get("exit_code"),
"pid": job.get("pid"),
"error": job.get("error", ""),
"preflight": job.get("preflight"),
"log_tail": self._read_tail(Path(job["log_path"])),
"error": self._redact_paths(str(job.get("error", ""))),
"preflight": self._public_preflight(job.get("preflight")),
"log_tail": output,
}
def _public_preflight(self, preflight: dict[str, Any] | None) -> dict[str, Any] | None:
if preflight is None:
return None
return {
key: self._redact_public_value(value)
for key, value in preflight.items()
if key != "checked_paths"
}
def _redact_public_value(self, value: Any) -> Any:
if isinstance(value, dict):
return {key: self._redact_public_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [self._redact_public_value(item) for item in value]
if isinstance(value, str):
return self._redact_paths(value)
return value
def _redact_paths(self, value: str) -> str:
text = value
for path, label in (
(self.runtime_root, "<runtime>"),
(self.vault, "<vault>"),
(self.project_root, "<project>"),
(Path.home(), "<home>"),
):
text = text.replace(str(path), label)
return text
def _query_result_from_status(self, status: dict[str, Any]) -> dict[str, Any]:
output_tail = status.get("log_tail", "")
answer = output_tail.strip()
@@ -542,7 +655,7 @@ class QueryManager:
self.opencode_path,
"run",
"--format",
"default",
"json",
"--auto",
*self._opencode_model_args(),
"--dir",
+452 -193
View File
@@ -3,14 +3,13 @@
from __future__ import annotations
import argparse
import asyncio
import json
import hmac
import os
import sqlite3
import sys
from email import policy
from email.parser import BytesParser
from fnmatch import fnmatch
import threading
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit
@@ -19,9 +18,11 @@ from obsidian_wiki import __version__
from obsidian_wiki.config import config_value, resolve_vault_path
from obsidian_wiki.file_assets import clean_config_value, db_path_for_vault, registry_path_for_vault
from obsidian_wiki.mcp_ingest import (
DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
DEFAULT_INGEST_MAX_CHUNK_BYTES,
DEFAULT_INGEST_MAX_FILE_BYTES,
DEFAULT_INGEST_MAX_FILES_PER_JOB,
DEFAULT_INGEST_MAX_REQUEST_BYTES,
DEFAULT_INGEST_OPENCODE_MODEL,
DEFAULT_INGEST_OPENCODE_VARIANT,
DEFAULT_UPLOAD_TICKET_SECONDS,
@@ -32,6 +33,7 @@ from obsidian_wiki.mcp_ingest import (
parse_bool,
)
from obsidian_wiki.mcp_query import (
DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
DEFAULT_QUERY_MAX_CHARS,
DEFAULT_QUERY_OPENCODE_MODEL,
DEFAULT_QUERY_OPENCODE_VARIANT,
@@ -56,9 +58,13 @@ DEFAULT_HTTP_PORT = 8765
DEFAULT_HTTP_PATH = "/mcp"
DEFAULT_INGEST_UPLOAD_PATH = "/mcp-ingest/uploads"
DEFAULT_TOKEN_ENV = "OBSIDIAN_MCP_TOKEN"
PUBLIC_BASE_URL_ENV = "OBSIDIAN_MCP_PUBLIC_BASE_URL"
PUBLIC_UPLOAD_URL_ENV = "OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL"
ALLOWED_HOSTS_ENV = "OBSIDIAN_MCP_ALLOWED_HOSTS"
ALLOWED_ORIGINS_ENV = "OBSIDIAN_MCP_ALLOWED_ORIGINS"
MCP_SERVER_NAME = "wiki-file-assets"
CAPABILITIES_SCHEMA_VERSION = 1
TOOLS_VERSION = 1
CAPABILITIES_SCHEMA_VERSION = 2
TOOLS_VERSION = 2
TOOL_FILE_ASSETS = "file-assets"
TOOL_INGEST = "ingest"
TOOL_QUERY = "query"
@@ -178,7 +184,7 @@ class BearerTokenAuthMiddleware:
await self.app(scope, receive, send)
class RequestBodyTooLarge(IngestError):
class RequestBodyTooLarge(IngestError, OSError):
"""Raised when an HTTP ingest upload body exceeds the configured request limit."""
@@ -189,6 +195,7 @@ def create_server(
profile_name: str = DEFAULT_PROFILE_NAME,
max_file_bytes: int = DEFAULT_MAX_FILE_BYTES,
expose_paths: bool = False,
allow_source_files: bool = False,
host: str = DEFAULT_HTTP_HOST,
port: int = DEFAULT_HTTP_PORT,
streamable_http_path: str = DEFAULT_HTTP_PATH,
@@ -209,7 +216,10 @@ def create_server(
ingest_max_file_bytes: int = DEFAULT_INGEST_MAX_FILE_BYTES,
ingest_max_chunk_bytes: int = DEFAULT_INGEST_MAX_CHUNK_BYTES,
ingest_max_files_per_job: int = DEFAULT_INGEST_MAX_FILES_PER_JOB,
ingest_max_request_bytes: int = DEFAULT_INGEST_MAX_REQUEST_BYTES,
ingest_job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
query_max_chars: int = DEFAULT_QUERY_MAX_CHARS,
query_job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
ingest_manager: IngestManager | None = None,
query_manager: QueryManager | None = None,
):
@@ -227,8 +237,10 @@ def create_server(
vault = Path(clean_config_value(str(vault_path))).expanduser()
db = _resolve_db_path(vault, db_path)
project = Path(project_root).expanduser().resolve() if project_root is not None else Path.cwd().resolve()
public_base = _normalize_public_base_url(public_base_url or runtime_config_value("OBSIDIAN_MCP_PUBLIC_BASE_URL", ""))
public_upload_url = _upload_url_from_public_base(public_base, DEFAULT_INGEST_UPLOAD_PATH)
public_base = _normalize_public_base_url(public_base_url or runtime_config_value(PUBLIC_BASE_URL_ENV, ""))
public_upload_url = _normalize_public_upload_url(runtime_config_value(PUBLIC_UPLOAD_URL_ENV, ""))
if not public_upload_url:
public_upload_url = _upload_url_from_public_base(public_base, DEFAULT_INGEST_UPLOAD_PATH)
ingest_runtime = (
Path(ingest_runtime_root).expanduser().resolve()
if ingest_runtime_root is not None
@@ -264,6 +276,7 @@ def create_server(
"profile_name": profile_name,
"max_file_bytes": max_file_bytes,
"expose_paths": expose_paths,
"allow_source_files": allow_source_files,
"enabled_tools": sorted(
([TOOL_FILE_ASSETS] if enable_file_assets else [])
+ ([TOOL_INGEST] if enable_ingest else [])
@@ -297,15 +310,15 @@ def create_server(
return {
"server_name": MCP_SERVER_NAME,
"server_version": __version__,
"schema_version": CAPABILITIES_SCHEMA_VERSION,
"capabilities_schema_version": CAPABILITIES_SCHEMA_VERSION,
"tools_version": TOOLS_VERSION,
"enabled_tools": enabled,
"recommended_first_steps": [
"Use ask_wiki for ordinary wiki, business facts, or hybrid questions when query is enabled.",
"Use search_file_assets before reading source file resources.",
"Use ingest_text_note for short text notes when ingest is enabled.",
"Use create_ingest_upload_ticket before remote HTTP file upload.",
],
"recommended_first_steps": _recommended_first_steps(
enable_file_assets=enable_file_assets,
enable_ingest=enable_ingest,
enable_query=enable_query,
),
"file_index": _file_index_capabilities(vault, db),
"file_assets": {
"enabled": enable_file_assets,
@@ -322,6 +335,7 @@ def create_server(
"best_for": "safe engineering source files such as drawings, CAD/3D models, specifications, and test reports",
"blocked_content": "raw orders, prices, email correspondence, complaint evidence, claims, and personal data",
"max_file_bytes": max_file_bytes,
"source_file_download_enabled": bool(enable_file_assets and allow_source_files),
},
"query": {
"enabled": enable_query,
@@ -353,6 +367,7 @@ def create_server(
"public_only_disables_business_facts": True,
"supports_structured_result": enable_query,
"max_query_chars": query_max_chars,
"job_timeout_seconds": query_job_timeout_seconds,
"known_limitations": (
[
"answer_markdown is produced by the local wiki-query agent; citations and source flags are best-effort structured metadata.",
@@ -370,7 +385,7 @@ def create_server(
"http_upload_url_warning": (
None
if public_upload_url or not enable_ingest
else "Set OBSIDIAN_MCP_PUBLIC_BASE_URL so remote agents can receive a complete upload URL."
else "Set OBSIDIAN_MCP_PUBLIC_BASE_URL or OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL so remote agents can receive a complete upload URL."
),
"preferred_upload_auth": "upload_ticket" if enable_ingest else None,
"tools": (
@@ -391,6 +406,10 @@ def create_server(
),
"max_file_bytes": ingest_max_file_bytes,
"max_files_per_job": ingest_max_files_per_job,
"max_request_bytes": _max_upload_body_bytes(ingest_manager, None)
if enable_ingest and ingest_manager is not None
else ingest_max_request_bytes or None,
"job_timeout_seconds": ingest_job_timeout_seconds,
"last_ingest_at": _last_ingest_at(ingest_manager) if enable_ingest and ingest_manager is not None else None,
"queue": _ingest_queue_summary(ingest_manager) if enable_ingest and ingest_manager is not None else None,
"known_limitations": (
@@ -404,13 +423,21 @@ def create_server(
),
},
"safety": {
"arbitrary_shell": False,
"arbitrary_shell": bool(enable_ingest or enable_query),
"direct_arbitrary_shell_tool": False,
"arbitrary_prompt": False,
"fixed_prompt_entrypoints": True,
"prompt_restrictions_enforced": False,
"shell_sandbox_enforced": False,
"opencode_auto_approve": bool(enable_ingest or enable_query),
"trusted_clients_required": True,
"local_paths_hidden_by_default": not expose_paths,
"model_choice_server_side": True,
"source_file_download_enabled": bool(enable_file_assets and allow_source_files),
},
}
owns_ingest_manager = False
if enable_ingest:
if ingest_manager is None:
ingest_manager = IngestManager(
@@ -423,7 +450,10 @@ def create_server(
max_file_bytes=ingest_max_file_bytes,
max_chunk_bytes=ingest_max_chunk_bytes,
max_files_per_job=ingest_max_files_per_job,
max_request_bytes=ingest_max_request_bytes,
job_timeout_seconds=ingest_job_timeout_seconds,
)
owns_ingest_manager = True
@server.tool()
def get_ingest_upload_instructions() -> dict[str, Any]:
@@ -431,7 +461,7 @@ def create_server(
upload_url_warning = (
None
if public_upload_url
else "Set OBSIDIAN_MCP_PUBLIC_BASE_URL so remote agents can receive a complete upload URL."
else "Set OBSIDIAN_MCP_PUBLIC_BASE_URL or OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL so remote agents can receive a complete upload URL."
)
return {
"preferred_method": "use ingest_text_note for short text; for files, call create_ingest_upload_ticket and upload with X-Obsidian-Wiki-Upload-Token",
@@ -472,6 +502,7 @@ def create_server(
"max_file_bytes": ingest_manager.max_file_bytes,
"max_chunk_bytes": ingest_manager.max_chunk_bytes,
"max_files_per_job": ingest_manager.max_files_per_job,
"max_request_bytes": _max_upload_body_bytes(ingest_manager, None),
},
"staging": {
"note": "uploads are temporary staging files outside the vault; wiki-ingest decides final archive/index placement",
@@ -529,7 +560,7 @@ def create_server(
"upload_url_warning": (
None
if public_upload_url
else "Set OBSIDIAN_MCP_PUBLIC_BASE_URL so remote agents can receive a complete upload URL."
else "Set OBSIDIAN_MCP_PUBLIC_BASE_URL or OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL so remote agents can receive a complete upload URL."
),
"token_header": "X-Obsidian-Wiki-Upload-Token",
"expires_seconds_max": MAX_UPLOAD_TICKET_SECONDS,
@@ -577,19 +608,25 @@ def create_server(
@server.tool()
def check_ingest_opencode_readiness(run_smoke_test: bool = False) -> dict[str, Any]:
"""Check whether local OpenCode can load the wiki-ingest skill for MCP ingest jobs."""
return ingest_manager.check_opencode_readiness(run_smoke_test=run_smoke_test)
return _public_readiness(ingest_manager, run_smoke_test=run_smoke_test)
if enable_query:
if query_manager is None:
query_manager = QueryManager(
vault_path=vault,
project_root=project,
runtime_root=query_runtime,
opencode_path=opencode_path,
opencode_model=query_opencode_model,
opencode_variant=query_opencode_variant,
max_query_chars=query_max_chars,
)
try:
query_manager = QueryManager(
vault_path=vault,
project_root=project,
runtime_root=query_runtime,
opencode_path=opencode_path,
opencode_model=query_opencode_model,
opencode_variant=query_opencode_variant,
max_query_chars=query_max_chars,
job_timeout_seconds=query_job_timeout_seconds,
)
except Exception:
if owns_ingest_manager and ingest_manager is not None:
ingest_manager.close()
raise
@server.tool()
def ask_wiki(
@@ -643,7 +680,7 @@ def create_server(
@server.tool()
def check_query_opencode_readiness(run_smoke_test: bool = False) -> dict[str, Any]:
"""Check whether local OpenCode can load the wiki-query skill for MCP query jobs."""
return query_manager.check_opencode_readiness(run_smoke_test=run_smoke_test)
return _public_readiness(query_manager, run_smoke_test=run_smoke_test)
if enable_file_assets:
@@ -722,6 +759,7 @@ def create_server(
db_path=db,
profile_name=profile_name,
max_file_bytes=max_file_bytes,
allow_source_files=allow_source_files,
)
return resource["bytes"]
@@ -730,20 +768,40 @@ def create_server(
description="Explain how to search and retrieve original source files from the wiki-file-assets MCP server.",
)
def how_to_use_wiki_file_assets() -> str:
download_status = (
"Source file downloads are enabled by the server operator. "
if allow_source_files
else "Source file downloads are disabled; metadata search remains available. "
)
return (
"wiki-file-assets serves original source files archived from an Obsidian wiki vault. "
"Use search_file_assets for keyword searches such as product codes, drawing names, sizes, "
"or document types. Use list_file_assets to browse allowed engineering files with optional "
"filters. Use get_file_asset_metadata with a file_id from search/list results to get the "
"asset://source/sha256/... URI, then read that resource to fetch the original file bytes. "
"This MCP is intentionally limited to safe engineering files such as drawings, CAD/3D models, "
f"{download_status}This MCP is intentionally limited to safe engineering files such as drawings, CAD/3D models, "
"specifications, product parameters, and test reports; it blocks raw orders, prices, email "
"correspondence, complaint evidence, claims, and personal data."
)
setattr(
server,
"_obsidian_job_managers",
tuple(manager for manager in (ingest_manager, query_manager) if manager is not None),
)
return server
def close_server_managers(server: Any) -> list[Exception]:
errors: list[Exception] = []
for manager in getattr(server, "_obsidian_job_managers", ()):
try:
manager.close()
except Exception as exc:
errors.append(exc)
return errors
def read_token_from_env(env_name: str = DEFAULT_TOKEN_ENV) -> str:
token = os.environ.get(env_name, "").strip()
if not token:
@@ -767,6 +825,30 @@ def runtime_config_int(key: str, default: int) -> int:
return default
def runtime_config_list(key: str) -> list[str] | None:
raw = os.environ.get(key)
if raw is None:
configured = config_value(key, "")
if not configured:
return None
raw = configured
return [item.strip() for item in clean_config_value(raw).split(",") if item.strip()]
def _public_readiness(manager: Any, *, run_smoke_test: bool) -> dict[str, Any]:
result = manager.check_opencode_readiness(run_smoke_test=run_smoke_test)
sanitizer = getattr(manager, "_public_preflight", None)
if callable(sanitizer):
public = sanitizer(result)
if isinstance(public, dict):
return public
return {
key: value
for key, value in result.items()
if key not in {"checked_paths", "errors"}
} | ({"errors": ["OpenCode readiness failed"]} if result.get("errors") else {"errors": []})
class IngestUploadRouter:
def __init__(
self,
@@ -782,8 +864,10 @@ class IngestUploadRouter:
self.ingest_manager = ingest_manager
self.upload_path = upload_path
self.bearer_auth_token = bearer_auth_token
self.allowed_hosts = allowed_hosts or []
self.allowed_origins = allowed_origins or []
self.enforce_allowed_hosts = allowed_hosts is not None
self.enforce_allowed_origins = allowed_origins is not None
self.allowed_hosts = list(allowed_hosts or [])
self.allowed_origins = list(allowed_origins or [])
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
if scope.get("type") != "http" or scope.get("path") != self.upload_path:
@@ -809,19 +893,19 @@ class IngestUploadRouter:
try:
data = await self._handle_upload(scope, receive, upload_ticket=upload_ticket)
except RequestBodyTooLarge as exc:
await _send_json(send, 413, {"error": str(exc)})
await _send_json(send, 413, _upload_error_payload(str(exc), upload_ticket=upload_ticket))
return
except IngestError as exc:
await _send_json(send, 400, {"error": str(exc)})
await _send_json(send, 400, _upload_error_payload(str(exc), upload_ticket=upload_ticket))
return
await _send_json(send, 200, data)
def _transport_headers_allowed(self, scope: dict[str, Any]) -> bool:
host = _header_value(scope, b"host")
if self.allowed_hosts and (not host or not _matches_host(host, self.allowed_hosts)):
if self.enforce_allowed_hosts and (not host or not _matches_host(host, self.allowed_hosts)):
return False
origin = _header_value(scope, b"origin")
if origin and self.allowed_origins and not _matches_any(origin, self.allowed_origins):
if origin and self.enforce_allowed_origins and not _matches_any(origin, self.allowed_origins):
return False
return True
@@ -832,7 +916,7 @@ class IngestUploadRouter:
return True
if _header_value(scope, b"x-obsidian-wiki-upload-token"):
return True
return bool(_query_param(scope, "upload_token"))
return False
def _authorize_upload(self, scope: dict[str, Any]) -> dict[str, Any] | None:
authorization = _header_value(scope, b"authorization")
@@ -842,8 +926,6 @@ class IngestUploadRouter:
if self.ingest_manager is None:
raise IngestError("unauthorized")
upload_token = _header_value(scope, b"x-obsidian-wiki-upload-token")
if not upload_token:
upload_token = _query_param(scope, "upload_token")
if not upload_token:
raise IngestError("unauthorized")
return self.ingest_manager.consume_upload_ticket(upload_token)
@@ -858,30 +940,135 @@ class IngestUploadRouter:
manager = self.ingest_manager
if manager is None:
raise IngestError("ingest is not enabled")
body = await _read_http_body(receive, max_bytes=_max_upload_body_bytes(manager, upload_ticket))
content_type = _header_value(scope, b"content-type")
fields, files = _parse_multipart_form(content_type, body)
if not files:
raise IngestError("multipart field 'files' must include at least one file")
if not content_type.lower().startswith("multipart/form-data"):
raise IngestError("content-type must be multipart/form-data")
max_body_bytes = _max_upload_body_bytes(manager, upload_ticket)
content_length = _header_value(scope, b"content-length")
if content_length:
try:
declared_body_bytes = int(content_length)
except ValueError as exc:
raise IngestError("content-length must be an integer") from exc
if declared_body_bytes < 0:
raise IngestError("content-length must be non-negative")
if max_body_bytes is not None and declared_body_bytes > max_body_bytes:
raise RequestBodyTooLarge(
f"request body exceeds max upload size: {declared_body_bytes} > {max_body_bytes}"
)
max_files = int(upload_ticket.get("max_files", manager.max_files_per_job)) if upload_ticket else manager.max_files_per_job
if len(files) > max_files:
raise IngestError(f"too many files for one ingest job: {len(files)} > {max_files}")
project = _first_field(fields, "project") or (upload_ticket or {}).get("project")
doc_type_hint = _first_field(fields, "doc_type_hint") or (upload_ticket or {}).get("doc_type_hint")
note = _first_field(fields, "note") or (upload_ticket or {}).get("note")
start_default = bool((upload_ticket or {}).get("start_job", True))
start_job = parse_bool(_first_field(fields, "start_job"), default=start_default)
uploads = [
manager.create_completed_upload(
file_name=file["file_name"],
payload=file["payload"],
project=project,
doc_type_hint=doc_type_hint,
)
for file in files
]
job = manager.start_ingest_job([upload["upload_id"] for upload in uploads], note=note) if start_job else None
return {"uploads": uploads, "job": job}
received_body_bytes = 0
async def limited_receive() -> dict[str, Any]:
nonlocal received_body_bytes
message = await receive()
if message.get("type") == "http.request":
received_body_bytes += len(message.get("body", b""))
if max_body_bytes is not None and received_body_bytes > max_body_bytes:
raise RequestBodyTooLarge(
f"request body exceeds max upload size: {received_body_bytes} > {max_body_bytes}"
)
return message
try:
from starlette.datastructures import UploadFile
from starlette.formparsers import MultiPartException, MultiPartParser
from starlette.requests import Request
except ImportError as exc:
raise IngestError("streaming multipart support is not installed") from exc
class FilePartTooLarge(MultiPartException):
pass
class LimitedFileMultiPartParser(MultiPartParser):
def __init__(self, *args: Any, max_file_part_size: int, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.max_file_part_size = max_file_part_size
self.current_file_size = 0
def on_part_begin(self) -> None:
super().on_part_begin()
self.current_file_size = 0
def on_part_data(self, data: bytes, start: int, end: int) -> None:
if self._current_part.file is not None:
self.current_file_size += end - start
if self.max_file_part_size > 0 and self.current_file_size > self.max_file_part_size:
raise FilePartTooLarge(
f"file part exceeds max upload size: "
f"{self.current_file_size} > {self.max_file_part_size}"
)
super().on_part_data(data, start, end)
form = None
try:
request = Request(scope, limited_receive)
try:
parser = LimitedFileMultiPartParser(
request.headers,
request.stream(),
max_files=max_files,
max_fields=8,
max_part_size=64 * 1024,
max_file_part_size=manager.max_file_bytes,
)
form = await parser.parse()
except FilePartTooLarge as exc:
raise RequestBodyTooLarge(str(exc)) from exc
except MultiPartException as exc:
raise IngestError(f"multipart request body is invalid: {exc}") from exc
files = [
value
for field_name, value in form.multi_items()
if field_name == "files" and isinstance(value, UploadFile)
]
if not files:
raise IngestError("multipart field 'files' must include at least one file")
if len(files) > max_files:
raise IngestError(f"too many files for one ingest job: {len(files)} > {max_files}")
def first_text_field(name: str) -> str | None:
for value in form.getlist(name):
if isinstance(value, str):
return value
return None
def ticket_bound_field(name: str) -> str | None:
ticket_value = (upload_ticket or {}).get(name)
return ticket_value if ticket_value not in (None, "") else first_text_field(name)
project = ticket_bound_field("project")
doc_type_hint = ticket_bound_field("doc_type_hint")
note = ticket_bound_field("note")
start_default = bool((upload_ticket or {}).get("start_job", True))
requested_start = parse_bool(first_text_field("start_job"), default=start_default)
start_job = requested_start if upload_ticket is None else start_default and requested_start
uploads = []
try:
for file in files:
await file.seek(0)
upload = await _run_blocking(
lambda file=file: manager.create_completed_upload_from_stream(
file_name=file.filename or "",
stream=file.file,
size_bytes=file.size,
project=project,
doc_type_hint=doc_type_hint,
)
)
uploads.append(upload)
job = manager.start_ingest_job([upload["upload_id"] for upload in uploads], note=note) if start_job else None
return {"uploads": uploads, "job": job}
except BaseException:
for upload in reversed(uploads):
try:
manager.discard_upload(upload["upload_id"])
except (IngestError, OSError):
pass
raise
finally:
if form is not None:
await form.close()
def create_http_app(
@@ -905,31 +1092,16 @@ def create_http_app(
return BearerTokenAuthMiddleware(app, token, bypass_paths={upload_path})
async def _read_http_body(receive: Any, *, max_bytes: int | None = None) -> bytes:
chunks: list[bytes] = []
received = 0
while True:
message = await receive()
if message.get("type") == "http.disconnect":
break
if message.get("type") != "http.request":
continue
chunk = message.get("body", b"")
received += len(chunk)
if max_bytes is not None and received > max_bytes:
raise RequestBodyTooLarge(f"request body exceeds max upload size: {received} > {max_bytes}")
chunks.append(chunk)
if not message.get("more_body", False):
break
return b"".join(chunks)
def _max_upload_body_bytes(manager: IngestManager, upload_ticket: dict[str, Any] | None) -> int | None:
if manager.max_file_bytes <= 0:
return None
max_files = int(upload_ticket.get("max_files", manager.max_files_per_job)) if upload_ticket else manager.max_files_per_job
overhead = max(16 * 1024, max_files * 4096)
return manager.max_file_bytes * max_files + overhead
derived_limit = None
if manager.max_file_bytes > 0:
overhead = max(16 * 1024, max_files * 4096)
derived_limit = manager.max_file_bytes * max_files + overhead
configured_limit = int(getattr(manager, "max_request_bytes", 0) or 0)
if configured_limit <= 0:
return derived_limit
return min(configured_limit, derived_limit) if derived_limit is not None else configured_limit
def _header_value(scope: dict[str, Any], name: bytes) -> str:
@@ -937,68 +1109,29 @@ def _header_value(scope: dict[str, Any], name: bytes) -> str:
return headers.get(name.lower(), b"").decode("latin1")
def _query_param(scope: dict[str, Any], name: str) -> str:
from urllib.parse import parse_qs
raw = scope.get("query_string", b"")
query = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
values = parse_qs(query).get(name, [])
return values[0] if values else ""
def _matches_any(value: str, patterns: list[str]) -> bool:
return any(fnmatch(value, pattern) for pattern in patterns)
if value in patterns:
return True
return any(
pattern.endswith(":*") and value.startswith(pattern[:-2] + ":")
for pattern in patterns
)
def _matches_host(value: str, patterns: list[str]) -> bool:
if _matches_any(value, patterns):
return True
host_only = value
if value.startswith("[") and "]" in value:
host_only = value[1 : value.index("]")]
elif ":" in value:
host_only = value.split(":", 1)[0]
return _matches_any(host_only, patterns)
return _matches_any(value, patterns)
def _parse_multipart_form(content_type: str, body: bytes) -> tuple[dict[str, list[str]], list[dict[str, Any]]]:
if not content_type.lower().startswith("multipart/form-data"):
raise IngestError("content-type must be multipart/form-data")
message = BytesParser(policy=policy.default).parsebytes(
b"Content-Type: "
+ content_type.encode("latin1")
+ b"\r\nMIME-Version: 1.0\r\n\r\n"
+ body
)
if not message.is_multipart():
raise IngestError("multipart request body is invalid")
fields: dict[str, list[str]] = {}
files: list[dict[str, Any]] = []
for part in message.iter_parts():
if part.get_content_disposition() != "form-data":
continue
field_name = part.get_param("name", header="content-disposition")
if not field_name:
continue
payload = part.get_payload(decode=True)
if payload is None:
content = part.get_content()
payload = content.encode(part.get_content_charset() or "utf-8") if isinstance(content, str) else bytes(content)
file_name = part.get_filename()
if file_name is None:
fields.setdefault(field_name, []).append(payload.decode(part.get_content_charset() or "utf-8", errors="replace"))
continue
if field_name != "files":
continue
files.append({"file_name": file_name, "payload": payload})
return fields, files
def _first_field(fields: dict[str, list[str]], name: str) -> str | None:
values = fields.get(name) or []
if not values:
return None
return values[0]
def _upload_error_payload(error: str, *, upload_ticket: dict[str, Any] | None) -> dict[str, Any]:
payload: dict[str, Any] = {"error": error}
if upload_ticket is not None:
payload.update(
{
"ticket_consumed": True,
"retry_action": "create_new_upload_ticket",
}
)
return payload
async def _send_json(send: Any, status: int, payload: dict[str, Any]) -> None:
@@ -1016,6 +1149,33 @@ async def _send_json(send: Any, status: int, payload: dict[str, Any]) -> None:
await send({"type": "http.response.body", "body": body})
async def _run_blocking(operation: Any) -> Any:
done = threading.Event()
result: list[Any] = []
errors: list[BaseException] = []
def run() -> None:
try:
result.append(operation())
except BaseException as exc:
errors.append(exc)
finally:
done.set()
worker = threading.Thread(target=run, name="mcp-upload-copy", daemon=True)
worker.start()
try:
while not done.is_set():
await asyncio.sleep(0.005)
except asyncio.CancelledError:
done.wait()
raise
worker.join()
if errors:
raise errors[0]
return result[0]
def run_http_server(
*,
vault_path: str | Path,
@@ -1023,6 +1183,7 @@ def run_http_server(
profile_name: str,
max_file_bytes: int,
expose_paths: bool,
allow_source_files: bool = False,
host: str,
port: int,
token_env: str,
@@ -1043,7 +1204,10 @@ def run_http_server(
ingest_max_file_bytes: int = DEFAULT_INGEST_MAX_FILE_BYTES,
ingest_max_chunk_bytes: int = DEFAULT_INGEST_MAX_CHUNK_BYTES,
ingest_max_files_per_job: int = DEFAULT_INGEST_MAX_FILES_PER_JOB,
ingest_max_request_bytes: int = DEFAULT_INGEST_MAX_REQUEST_BYTES,
ingest_job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
query_max_chars: int = DEFAULT_QUERY_MAX_CHARS,
query_job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
) -> None:
token = read_token_from_env(token_env)
try:
@@ -1069,6 +1233,8 @@ def run_http_server(
max_file_bytes=ingest_max_file_bytes,
max_chunk_bytes=ingest_max_chunk_bytes,
max_files_per_job=ingest_max_files_per_job,
max_request_bytes=ingest_max_request_bytes,
job_timeout_seconds=ingest_job_timeout_seconds,
)
query_manager = None
@@ -1079,53 +1245,72 @@ def run_http_server(
if query_runtime_root is not None
else project / ".runtime" / "mcp-query"
)
query_manager = QueryManager(
vault_path=vault_path,
project_root=project,
runtime_root=query_runtime,
opencode_path=opencode_path,
opencode_model=query_opencode_model,
opencode_variant=query_opencode_variant,
max_query_chars=query_max_chars,
)
try:
query_manager = QueryManager(
vault_path=vault_path,
project_root=project,
runtime_root=query_runtime,
opencode_path=opencode_path,
opencode_model=query_opencode_model,
opencode_variant=query_opencode_variant,
max_query_chars=query_max_chars,
job_timeout_seconds=query_job_timeout_seconds,
)
except Exception:
if ingest_manager is not None:
ingest_manager.close()
raise
server = create_server(
vault_path=vault_path,
db_path=db_path,
profile_name=profile_name,
max_file_bytes=max_file_bytes,
expose_paths=expose_paths,
host=host,
port=port,
streamable_http_path=streamable_http_path,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
enable_file_assets=enable_file_assets,
enable_ingest=enable_ingest,
enable_query=enable_query,
project_root=project_root,
ingest_runtime_root=ingest_runtime_root,
query_runtime_root=query_runtime_root,
opencode_path=opencode_path,
ingest_opencode_model=ingest_opencode_model,
ingest_opencode_variant=ingest_opencode_variant,
query_opencode_model=query_opencode_model,
query_opencode_variant=query_opencode_variant,
ingest_max_file_bytes=ingest_max_file_bytes,
ingest_max_chunk_bytes=ingest_max_chunk_bytes,
ingest_max_files_per_job=ingest_max_files_per_job,
query_max_chars=query_max_chars,
ingest_manager=ingest_manager,
query_manager=query_manager,
)
app = create_http_app(
server,
token,
ingest_manager=ingest_manager,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
)
uvicorn.run(app, host=host, port=port, log_level="info")
server = None
try:
server = create_server(
vault_path=vault_path,
db_path=db_path,
profile_name=profile_name,
max_file_bytes=max_file_bytes,
expose_paths=expose_paths,
allow_source_files=allow_source_files,
host=host,
port=port,
streamable_http_path=streamable_http_path,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
enable_file_assets=enable_file_assets,
enable_ingest=enable_ingest,
enable_query=enable_query,
project_root=project_root,
ingest_runtime_root=ingest_runtime_root,
query_runtime_root=query_runtime_root,
opencode_path=opencode_path,
ingest_opencode_model=ingest_opencode_model,
ingest_opencode_variant=ingest_opencode_variant,
query_opencode_model=query_opencode_model,
query_opencode_variant=query_opencode_variant,
ingest_max_file_bytes=ingest_max_file_bytes,
ingest_max_chunk_bytes=ingest_max_chunk_bytes,
ingest_max_files_per_job=ingest_max_files_per_job,
ingest_max_request_bytes=ingest_max_request_bytes,
ingest_job_timeout_seconds=ingest_job_timeout_seconds,
query_max_chars=query_max_chars,
query_job_timeout_seconds=query_job_timeout_seconds,
ingest_manager=ingest_manager,
query_manager=query_manager,
)
app = create_http_app(
server,
token,
ingest_manager=ingest_manager,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
)
uvicorn.run(app, host=host, port=port, log_level="info")
finally:
if server is not None:
close_server_managers(server)
else:
for manager in (ingest_manager, query_manager):
if manager is not None:
manager.close()
def main(argv: list[str] | None = None) -> int:
@@ -1134,7 +1319,26 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--db", metavar="PATH", help="SQLite file asset index path")
parser.add_argument("--profile", default=None, help=f"bot profile name, default: {DEFAULT_PROFILE_NAME}")
parser.add_argument("--max-file-bytes", type=int, default=None, help="maximum resource size, 0 disables the limit")
parser.add_argument("--expose-paths", action="store_true", help="include archive/absolute paths in tool output")
parser.add_argument(
"--expose-paths",
action="store_true",
default=None,
help="include archive/absolute paths in tool output",
)
source_file_group = parser.add_mutually_exclusive_group()
source_file_group.add_argument(
"--allow-source-files",
dest="allow_source_files",
action="store_true",
default=None,
help="allow file resource bytes to be downloaded",
)
source_file_group.add_argument(
"--deny-source-files",
dest="allow_source_files",
action="store_false",
help="deny file resource bytes while retaining metadata tools",
)
parser.add_argument(
"--transport",
choices=["stdio", "streamable-http"],
@@ -1167,6 +1371,7 @@ def main(argv: list[str] | None = None) -> int:
profile_name=args.profile,
max_file_bytes=args.max_file_bytes,
expose_paths=args.expose_paths,
allow_source_files=args.allow_source_files,
)
transport = args.transport or runtime_config_value("OBSIDIAN_MCP_TRANSPORT", "stdio")
host = args.host or runtime_config_value("OBSIDIAN_MCP_HOST", DEFAULT_HTTP_HOST)
@@ -1183,6 +1388,10 @@ def main(argv: list[str] | None = None) -> int:
"OBSIDIAN_MCP_INGEST_MAX_FILES_PER_JOB",
DEFAULT_INGEST_MAX_FILES_PER_JOB,
)
ingest_max_request_bytes = runtime_config_int(
"OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES",
DEFAULT_INGEST_MAX_REQUEST_BYTES,
)
opencode_path = runtime_config_value("OBSIDIAN_MCP_OPENCODE_PATH", "")
if not opencode_path:
opencode_path = runtime_config_value("OBSIDIAN_MCP_INGEST_OPENCODE_PATH", DEFAULT_OPENCODE_PATH)
@@ -1199,6 +1408,18 @@ def main(argv: list[str] | None = None) -> int:
query_opencode_model = runtime_config_value("OBSIDIAN_MCP_QUERY_OPENCODE_MODEL", DEFAULT_QUERY_OPENCODE_MODEL)
query_opencode_variant = runtime_config_value("OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT", DEFAULT_QUERY_OPENCODE_VARIANT)
query_max_chars = runtime_config_int("OBSIDIAN_MCP_QUERY_MAX_CHARS", DEFAULT_QUERY_MAX_CHARS)
ingest_job_timeout_seconds = runtime_config_int(
"OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS",
DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
)
query_job_timeout_seconds = runtime_config_int(
"OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS",
DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
)
allowed_hosts = args.allowed_hosts if args.allowed_hosts is not None else runtime_config_list(ALLOWED_HOSTS_ENV)
allowed_origins = (
args.allowed_origins if args.allowed_origins is not None else runtime_config_list(ALLOWED_ORIGINS_ENV)
)
try:
if transport == "stdio":
server = create_server(
@@ -1207,6 +1428,7 @@ def main(argv: list[str] | None = None) -> int:
profile_name=options["profile_name"],
max_file_bytes=options["max_file_bytes"],
expose_paths=options["expose_paths"],
allow_source_files=options["allow_source_files"],
enable_file_assets=enable_file_assets,
enable_ingest=enable_ingest,
enable_query=enable_query,
@@ -1221,12 +1443,18 @@ def main(argv: list[str] | None = None) -> int:
ingest_max_file_bytes=ingest_max_file_bytes,
ingest_max_chunk_bytes=ingest_max_chunk_bytes,
ingest_max_files_per_job=ingest_max_files_per_job,
ingest_max_request_bytes=ingest_max_request_bytes,
ingest_job_timeout_seconds=ingest_job_timeout_seconds,
query_max_chars=query_max_chars,
query_job_timeout_seconds=query_job_timeout_seconds,
)
try:
server.run(transport="stdio")
except TypeError:
server.run()
try:
server.run(transport="stdio")
except TypeError:
server.run()
finally:
close_server_managers(server)
return 0
if transport != "streamable-http":
raise RuntimeError(f"unsupported MCP transport: {transport}")
@@ -1236,12 +1464,13 @@ def main(argv: list[str] | None = None) -> int:
profile_name=options["profile_name"],
max_file_bytes=options["max_file_bytes"],
expose_paths=options["expose_paths"],
allow_source_files=options["allow_source_files"],
host=host,
port=port,
token_env=token_env,
streamable_http_path=http_path,
allowed_hosts=args.allowed_hosts,
allowed_origins=args.allowed_origins,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
enable_file_assets=enable_file_assets,
enable_ingest=enable_ingest,
enable_query=enable_query,
@@ -1256,7 +1485,10 @@ def main(argv: list[str] | None = None) -> int:
ingest_max_file_bytes=ingest_max_file_bytes,
ingest_max_chunk_bytes=ingest_max_chunk_bytes,
ingest_max_files_per_job=ingest_max_files_per_job,
ingest_max_request_bytes=ingest_max_request_bytes,
ingest_job_timeout_seconds=ingest_job_timeout_seconds,
query_max_chars=query_max_chars,
query_job_timeout_seconds=query_job_timeout_seconds,
)
except (RuntimeError, FileNotFoundError, AssetAccessError, AssetNotFoundError, AssetTooLargeError, IngestError, QueryError) as exc:
print(f"error: {exc}", file=sys.stderr)
@@ -1283,6 +1515,17 @@ def _normalize_public_base_url(raw_url: str | None) -> str:
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", ""))
def _normalize_public_upload_url(raw_url: str | None) -> str:
value = clean_config_value(str(raw_url or "")).strip().rstrip("/")
if not value:
return ""
parsed = urlsplit(value)
if not parsed.scheme or not parsed.netloc:
return ""
path = parsed.path.rstrip("/") or DEFAULT_INGEST_UPLOAD_PATH
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
def _upload_url_from_public_base(public_base_url: str, upload_path: str) -> str | None:
if not public_base_url:
return None
@@ -1290,9 +1533,24 @@ def _upload_url_from_public_base(public_base_url: str, upload_path: str) -> str
return urlunsplit((parsed.scheme, parsed.netloc, upload_path, "", ""))
def _recommended_first_steps(*, enable_file_assets: bool, enable_ingest: bool, enable_query: bool) -> list[str]:
steps: list[str] = []
if enable_query:
steps.append("Use ask_wiki for ordinary wiki, business facts, or hybrid questions.")
if enable_file_assets:
steps.append("Use search_file_assets before reading source file resources.")
if enable_ingest:
steps.append("Use ingest_text_note for short text notes.")
steps.append("Use create_ingest_upload_ticket before remote HTTP file upload.")
if not steps:
steps.append("No optional MCP tool groups are enabled; ask the operator to enable file-assets, query, or ingest.")
return steps
def _file_index_capabilities(vault: Path, db: Path) -> dict[str, Any]:
registry = registry_path_for_vault(vault)
record_count: int | None = None
last_rebuild_at = _path_mtime_iso(db) if db.is_file() else None
if db.is_file():
try:
conn = sqlite3.connect(str(db))
@@ -1308,7 +1566,8 @@ def _file_index_capabilities(vault: Path, db: Path) -> dict[str, Any]:
"exists": db.is_file(),
"registry_exists": registry.is_file(),
"record_count": record_count,
"last_rebuild_at": _path_mtime_iso(db) if db.is_file() else None,
"last_rebuild_at": last_rebuild_at,
"last_index_rebuild_at": last_rebuild_at,
}
+4 -1
View File
@@ -34,7 +34,10 @@ classifiers = [
dependencies = []
[project.optional-dependencies]
mcp = ["mcp[cli]>=1.28,<2; python_version >= '3.10'"]
mcp = [
"mcp[cli]>=1.28,<2; python_version >= '3.10'",
"python-multipart>=0.0.20,<1; python_version >= '3.10'",
]
[project.urls]
Homepage = "https://github.com/Ar9av/obsidian-wiki"
+436 -118
View File
@@ -13,7 +13,7 @@ import socket
import subprocess
import sys
import time
from dataclasses import dataclass
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Sequence
from urllib.parse import urlsplit, urlunsplit
@@ -33,6 +33,10 @@ DEFAULT_PORT = 8765
DEFAULT_PATH = "/mcp"
DEFAULT_INGEST_UPLOAD_PATH = "/mcp-ingest/uploads"
PUBLIC_BASE_URL_ENV = "OBSIDIAN_MCP_PUBLIC_BASE_URL"
PUBLIC_UPLOAD_URL_ENV = "OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL"
ALLOW_SOURCE_FILES_ENV = "OBSIDIAN_MCP_ALLOW_SOURCE_FILES"
ALLOWED_HOSTS_ENV = "OBSIDIAN_MCP_ALLOWED_HOSTS"
ALLOWED_ORIGINS_ENV = "OBSIDIAN_MCP_ALLOWED_ORIGINS"
DEFAULT_TOKEN_BYTES = 32
DEFAULT_UV_CACHE_DIR = "/tmp/uv-cache"
DEFAULT_OPENCODE_PATH = "opencode"
@@ -40,6 +44,9 @@ DEFAULT_INGEST_OPENCODE_MODEL = "openai/gpt-5.5"
DEFAULT_INGEST_OPENCODE_VARIANT = "high"
DEFAULT_QUERY_OPENCODE_MODEL = "openai/gpt-5.5"
DEFAULT_QUERY_OPENCODE_VARIANT = "minimal"
DEFAULT_INGEST_JOB_TIMEOUT_SECONDS = 30 * 60
DEFAULT_QUERY_JOB_TIMEOUT_SECONDS = 5 * 60
DEFAULT_INGEST_MAX_REQUEST_BYTES = 0
DEFAULT_SERVICE_MEMORY_MAX = "2G"
DEFAULT_RUNTIME_CLEANUP_DAYS = 30
TOOL_FILE_ASSETS = "file-assets"
@@ -79,6 +86,10 @@ def parse_bool(value: str | None) -> bool:
return str(value or "").strip().lower() in {"1", "true", "yes", "on", "enabled"}
def parse_config_list(value: str | None) -> tuple[str, ...]:
return tuple(item.strip() for item in str(value or "").split(",") if item.strip())
def normalize_enabled_tools(tools: Sequence[str]) -> tuple[str, ...]:
enabled = {normalize_tool_name(tool) for tool in tools if str(tool).strip()}
return tuple(tool for tool in AVAILABLE_TOOL_GROUPS if tool in enabled)
@@ -134,6 +145,25 @@ class OpenCodeModelSettings:
query_variant: str = DEFAULT_QUERY_OPENCODE_VARIANT
@dataclass(frozen=True)
class ServiceConfig:
token: str
host: str = DEFAULT_HOST
port: int = DEFAULT_PORT
path: str = DEFAULT_PATH
public_base_url: str = ""
public_upload_url: str = ""
enabled_tools: tuple[str, ...] = DEFAULT_ENABLED_TOOLS
allow_source_files: bool = False
allowed_hosts: tuple[str, ...] = ()
allowed_origins: tuple[str, ...] = ()
ingest_job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS
query_job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS
ingest_max_request_bytes: int = DEFAULT_INGEST_MAX_REQUEST_BYTES
opencode_path: str = ""
model_settings: OpenCodeModelSettings = OpenCodeModelSettings()
def generate_token(num_bytes: int = DEFAULT_TOKEN_BYTES) -> str:
return secrets.token_urlsafe(num_bytes)
@@ -203,17 +233,11 @@ def render_service_unit(
systemd_quote(vault_path),
"--transport",
"streamable-http",
"--host",
host,
"--port",
str(port),
"--path",
path,
]
)
return (
"[Unit]\n"
"Description=obsidian-wiki readonly HTTP MCP server\n"
"Description=obsidian-wiki HTTP MCP server\n"
"After=network-online.target\n"
"Wants=network-online.target\n"
"\n"
@@ -266,16 +290,40 @@ def write_token_env_file(
port: int = DEFAULT_PORT,
path: str = DEFAULT_PATH,
public_base_url: str | None = None,
public_upload_url: str | None = None,
enabled_tools: Sequence[str] | None = None,
enable_ingest: bool | None = None,
enable_query: bool | None = None,
allow_source_files: bool | None = None,
allowed_hosts: Sequence[str] | None = None,
allowed_origins: Sequence[str] | None = None,
ingest_job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
query_job_timeout_seconds: int = DEFAULT_QUERY_JOB_TIMEOUT_SECONDS,
ingest_max_request_bytes: int = DEFAULT_INGEST_MAX_REQUEST_BYTES,
opencode_path: str | Path | None = None,
model_settings: OpenCodeModelSettings | None = None,
) -> None:
tools = normalize_enabled_tools(DEFAULT_ENABLED_TOOLS if enabled_tools is None else enabled_tools)
tools = update_ingest_tool(tools, enable_ingest)
tools = update_query_tool(tools, enable_query)
source_files_allowed = bool(allow_source_files)
models = model_settings or OpenCodeModelSettings()
normalized_public_base_url = normalize_public_base_url(public_base_url)
normalized_public_upload_url = normalize_public_upload_url(public_upload_url)
effective_allowed_hosts = tuple(
derive_allowed_hosts(
[host],
public_base_url=normalized_public_base_url,
public_upload_url=normalized_public_upload_url,
)
if allowed_hosts is None
else allowed_hosts
)
effective_allowed_origins = tuple(
derive_allowed_origins(normalized_public_base_url, normalized_public_upload_url)
if allowed_origins is None
else allowed_origins
)
env_path.parent.mkdir(parents=True, exist_ok=True)
env_path.parent.chmod(0o700)
fd = os.open(env_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
@@ -284,8 +332,16 @@ def write_token_env_file(
env_file.write(f"OBSIDIAN_MCP_HOST={host}\n")
env_file.write(f"OBSIDIAN_MCP_PORT={port}\n")
env_file.write(f"OBSIDIAN_MCP_PATH={path}\n")
if public_base_url:
env_file.write(f"{PUBLIC_BASE_URL_ENV}={public_base_url.rstrip('/')}\n")
if normalized_public_base_url:
env_file.write(f"{PUBLIC_BASE_URL_ENV}={normalized_public_base_url}\n")
if normalized_public_upload_url:
env_file.write(f"{PUBLIC_UPLOAD_URL_ENV}={normalized_public_upload_url}\n")
env_file.write(f"{ALLOW_SOURCE_FILES_ENV}={1 if source_files_allowed else 0}\n")
env_file.write(f"{ALLOWED_HOSTS_ENV}={','.join(effective_allowed_hosts)}\n")
env_file.write(f"{ALLOWED_ORIGINS_ENV}={','.join(effective_allowed_origins)}\n")
env_file.write(f"OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS={max(1, int(ingest_job_timeout_seconds))}\n")
env_file.write(f"OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS={max(1, int(query_job_timeout_seconds))}\n")
env_file.write(f"OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES={max(0, int(ingest_max_request_bytes))}\n")
env_file.write(f"OBSIDIAN_MCP_ENABLED_TOOLS={format_enabled_tools(tools)}\n")
env_file.write(f"OBSIDIAN_MCP_ENABLE_INGEST={1 if TOOL_INGEST in tools else 0}\n")
env_file.write(f"OBSIDIAN_MCP_INGEST_OPENCODE_MODEL={models.ingest_model}\n")
@@ -330,7 +386,7 @@ def model_settings_from_config(values: dict[str, str]) -> OpenCodeModelSettings:
)
def saved_service_config(project_root: Path) -> tuple[str, str, int, str, str, tuple[str, ...], str, OpenCodeModelSettings]:
def saved_service_config(project_root: Path) -> ServiceConfig:
paths = resolve_service_paths(project_root)
values = read_service_env_file(paths.env_path)
token = values.get("OBSIDIAN_MCP_TOKEN", "").strip()
@@ -339,13 +395,53 @@ def saved_service_config(project_root: Path) -> tuple[str, str, int, str, str, t
host = values.get("OBSIDIAN_MCP_HOST") or DEFAULT_HOST
port = int(values.get("OBSIDIAN_MCP_PORT") or DEFAULT_PORT)
path = values.get("OBSIDIAN_MCP_PATH") or DEFAULT_PATH
public_base_url = values.get(PUBLIC_BASE_URL_ENV, "").strip()
public_base_url = normalize_public_base_url(values.get(PUBLIC_BASE_URL_ENV))
public_upload_url = normalize_public_upload_url(values.get(PUBLIC_UPLOAD_URL_ENV))
enabled_tools = enabled_tools_from_config(values)
if ALLOW_SOURCE_FILES_ENV in values:
allow_source_files = parse_bool(values[ALLOW_SOURCE_FILES_ENV])
else:
allow_source_files = TOOL_FILE_ASSETS in enabled_tools
allowed_hosts = parse_config_list(values.get(ALLOWED_HOSTS_ENV))
if not allowed_hosts:
allowed_hosts = derive_allowed_hosts(
[host, *detect_remote_hosts()],
public_base_url=public_base_url,
public_upload_url=public_upload_url,
)
allowed_origins = parse_config_list(values.get(ALLOWED_ORIGINS_ENV))
if ALLOWED_ORIGINS_ENV not in values:
allowed_origins = derive_allowed_origins(public_base_url, public_upload_url)
opencode_path = (
values.get("OBSIDIAN_MCP_OPENCODE_PATH", "").strip()
or values.get("OBSIDIAN_MCP_INGEST_OPENCODE_PATH", "").strip()
)
return token, host, port, path, public_base_url, enabled_tools, opencode_path, model_settings_from_config(values)
return ServiceConfig(
token=token,
host=host,
port=port,
path=path,
public_base_url=public_base_url,
public_upload_url=public_upload_url,
enabled_tools=enabled_tools,
allow_source_files=allow_source_files,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
ingest_job_timeout_seconds=max(
1,
int(values.get("OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS") or DEFAULT_INGEST_JOB_TIMEOUT_SECONDS),
),
query_job_timeout_seconds=max(
1,
int(values.get("OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS") or DEFAULT_QUERY_JOB_TIMEOUT_SECONDS),
),
ingest_max_request_bytes=max(
0,
int(values.get("OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES") or DEFAULT_INGEST_MAX_REQUEST_BYTES),
),
opencode_path=opencode_path,
model_settings=model_settings_from_config(values),
)
def write_service_unit(
@@ -397,14 +493,94 @@ def detect_remote_hosts() -> list[str]:
def public_base_url_from_hosts(hosts: Sequence[str], *, port: int = DEFAULT_PORT, path: str = DEFAULT_PATH) -> str:
host = hosts[0] if hosts else ""
return f"http://{host}:{port}{path}" if host else ""
return f"http://{_url_host(host)}:{port}{path}" if host else ""
def upload_url_from_public_base(public_base_url: str, *, fallback_host: str, port: int = DEFAULT_PORT) -> str:
def public_hosts_for_bind(bind_host: str, detected_hosts: Sequence[str]) -> tuple[str, ...]:
normalized = str(bind_host).strip()
if normalized.strip("[]") in {"0.0.0.0", "::", ""}:
return tuple(detected_hosts)
return (normalized,)
def _url_host(host: str) -> str:
value = str(host).strip()
if ":" in value and not value.startswith("["):
return f"[{value}]"
return value
def normalize_public_base_url(public_base_url: str | None) -> str:
value = str(public_base_url or "").strip().rstrip("/")
if not value:
return ""
parsed = urlsplit(value)
if not parsed.scheme or not parsed.netloc:
return ""
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", ""))
def upload_url_from_public_base(
public_base_url: str,
*,
fallback_host: str,
port: int = DEFAULT_PORT,
public_upload_url: str | None = None,
) -> str:
explicit_upload_url = normalize_public_upload_url(public_upload_url)
if explicit_upload_url:
return explicit_upload_url
parsed = urlsplit(public_base_url)
if parsed.scheme and parsed.netloc:
return urlunsplit((parsed.scheme, parsed.netloc, DEFAULT_INGEST_UPLOAD_PATH, "", ""))
return f"http://{fallback_host}:{port}{DEFAULT_INGEST_UPLOAD_PATH}"
return f"http://{_url_host(fallback_host)}:{port}{DEFAULT_INGEST_UPLOAD_PATH}"
def normalize_public_upload_url(public_upload_url: str | None) -> str:
value = str(public_upload_url or "").strip().rstrip("/")
if not value:
return ""
parsed = urlsplit(value)
if not parsed.scheme or not parsed.netloc:
return ""
path = parsed.path.rstrip("/") or DEFAULT_INGEST_UPLOAD_PATH
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
def derive_allowed_hosts(
hosts: Sequence[str],
*,
public_base_url: str | None = None,
public_upload_url: str | None = None,
) -> tuple[str, ...]:
patterns = {"localhost", "localhost:*", "127.0.0.1", "127.0.0.1:*", "[::1]", "[::1]:*"}
candidates = [str(host).strip() for host in hosts]
for url in (public_base_url, public_upload_url):
parsed = urlsplit(str(url or ""))
if parsed.hostname:
candidates.append(parsed.hostname)
for candidate in candidates:
value = candidate.strip().strip("[]")
if not value or value in {"0.0.0.0", "::"}:
continue
if ":" in value:
patterns.update({f"[{value}]", f"[{value}]:*"})
else:
patterns.update({value, f"{value}:*"})
return tuple(sorted(patterns))
def derive_allowed_origins(*urls: str | None) -> tuple[str, ...]:
origins = set()
for url in urls:
parsed = urlsplit(str(url or ""))
if parsed.scheme and parsed.netloc:
origins.add(urlunsplit((parsed.scheme, parsed.netloc, "", "", "")))
return tuple(sorted(origins))
def merge_allowlist(existing: Sequence[str], required: Sequence[str]) -> tuple[str, ...]:
return tuple(sorted(set(existing) | set(required)))
def render_remote_config(
@@ -414,14 +590,22 @@ def render_remote_config(
port: int = DEFAULT_PORT,
path: str = DEFAULT_PATH,
public_base_url: str | None = None,
public_upload_url: str | None = None,
config_path: Path | None = None,
enabled_tools: Sequence[str] | None = None,
) -> str:
host = hosts[0] if hosts else "<server-ip>"
url = f"http://{host}:{port}{path}"
public_url = (public_base_url or url).rstrip("/")
upload_url = upload_url_from_public_base(public_url, fallback_host=host, port=port)
alternatives = "\n".join(f"- http://{candidate}:{port}{path}" for candidate in hosts[1:])
detected_urls = [f"http://{_url_host(candidate)}:{port}{path}" for candidate in hosts]
fallback_url = detected_urls[0] if detected_urls else f"http://{_url_host(host)}:{port}{path}"
public_url = normalize_public_base_url(public_base_url)
url = public_url or fallback_url
upload_url = upload_url_from_public_base(
url,
fallback_host=host,
port=port,
public_upload_url=public_upload_url,
)
alternatives = "\n".join(f"- {candidate}" for candidate in detected_urls if candidate != url)
alternatives_section = f"\nOther detected addresses:\n{alternatives}\n" if alternatives else ""
config_section = f"\nConfig file:\n{config_path}\n" if config_path is not None else ""
enabled_section = ""
@@ -429,8 +613,11 @@ def render_remote_config(
enabled_section = f"\nEnabled tool groups:\n{format_enabled_tools(enabled_tools) or 'none'}\n"
ingest_upload_section = ""
if enabled_tools is not None and TOOL_INGEST in normalize_enabled_tools(enabled_tools):
detected_upload_urls = [
f"http://{_url_host(candidate)}:{port}{DEFAULT_INGEST_UPLOAD_PATH}" for candidate in hosts[1:]
]
upload_alternatives = "\n".join(
f"- http://{candidate}:{port}{DEFAULT_INGEST_UPLOAD_PATH}" for candidate in hosts[1:]
f"- {candidate}" for candidate in detected_upload_urls if candidate != upload_url
)
upload_alternatives_section = (
f"\nOther ingest upload addresses:\n{upload_alternatives}\n" if upload_alternatives else ""
@@ -483,15 +670,16 @@ def render_remote_config(
def show_config(project_root: Path) -> str:
paths = resolve_service_paths(project_root)
token, _host, port, path, public_base_url, enabled_tools, _opencode_path, _models = saved_service_config(project_root)
config = saved_service_config(project_root)
return render_remote_config(
hosts=detect_remote_hosts(),
token=token,
port=port,
path=path,
public_base_url=public_base_url,
token=config.token,
port=config.port,
path=config.path,
public_base_url=config.public_base_url,
public_upload_url=config.public_upload_url,
config_path=paths.env_path,
enabled_tools=enabled_tools,
enabled_tools=config.enabled_tools,
)
@@ -512,46 +700,106 @@ def restart_service_if_installed(paths: ServicePaths) -> bool:
return True
def write_service_config(env_path: Path, config: ServiceConfig) -> None:
write_token_env_file(
env_path,
config.token,
host=config.host,
port=config.port,
path=config.path,
public_base_url=config.public_base_url,
public_upload_url=config.public_upload_url,
enabled_tools=config.enabled_tools,
allow_source_files=config.allow_source_files,
allowed_hosts=config.allowed_hosts,
allowed_origins=config.allowed_origins,
ingest_job_timeout_seconds=config.ingest_job_timeout_seconds,
query_job_timeout_seconds=config.query_job_timeout_seconds,
ingest_max_request_bytes=config.ingest_max_request_bytes,
opencode_path=config.opencode_path,
model_settings=config.model_settings,
)
def new_token(
*,
project_root: Path,
host: str = DEFAULT_HOST,
port: int = DEFAULT_PORT,
path: str = DEFAULT_PATH,
host: str | None = None,
port: int | None = None,
path: str | None = None,
public_base_url: str | None = None,
public_upload_url: str | None = None,
token_bytes: int = DEFAULT_TOKEN_BYTES,
) -> str:
paths = resolve_service_paths(project_root)
enabled_tools = DEFAULT_ENABLED_TOOLS
opencode_path = ""
model_settings = OpenCodeModelSettings()
public_base_url = ""
try:
_old_token, _old_host, _old_port, _old_path, public_base_url, enabled_tools, opencode_path, model_settings = saved_service_config(project_root)
saved = saved_service_config(project_root)
has_saved_config = True
except RuntimeError:
pass
saved = ServiceConfig(token="")
has_saved_config = False
hosts = detect_remote_hosts()
public_base_url = public_base_url or public_base_url_from_hosts(hosts, port=port, path=path)
token = generate_token(token_bytes)
write_token_env_file(
paths.env_path,
token,
host=host,
port=port,
path=path,
public_base_url=public_base_url,
enabled_tools=enabled_tools,
opencode_path=opencode_path,
model_settings=model_settings,
effective_host = saved.host if host is None else host
effective_port = saved.port if port is None else port
effective_path = saved.path if path is None else path
address_overridden = any(value is not None for value in (host, port, path))
if public_base_url is not None:
effective_public_base_url = normalize_public_base_url(public_base_url)
elif has_saved_config and not address_overridden:
effective_public_base_url = saved.public_base_url
else:
effective_public_base_url = public_base_url_from_hosts(
public_hosts_for_bind(effective_host, hosts),
port=effective_port,
path=effective_path,
)
effective_public_upload_url = (
saved.public_upload_url
if public_upload_url is None
else normalize_public_upload_url(public_upload_url)
)
endpoint_overridden = any(
value is not None
for value in (host, port, path, public_base_url, public_upload_url)
)
if has_saved_config and not endpoint_overridden:
allowed_hosts = saved.allowed_hosts
allowed_origins = saved.allowed_origins
else:
allowed_hosts = merge_allowlist(
saved.allowed_hosts if has_saved_config else (),
derive_allowed_hosts(
[effective_host, *hosts],
public_base_url=effective_public_base_url,
public_upload_url=effective_public_upload_url,
),
)
allowed_origins = merge_allowlist(
saved.allowed_origins if has_saved_config else (),
derive_allowed_origins(effective_public_base_url, effective_public_upload_url),
)
config = replace(
saved,
token=generate_token(token_bytes),
host=effective_host,
port=effective_port,
path=effective_path,
public_base_url=effective_public_base_url,
public_upload_url=effective_public_upload_url,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
)
write_service_config(paths.env_path, config)
restart_service_if_installed(paths)
return render_remote_config(
hosts=hosts,
token=token,
port=port,
path=path,
public_base_url=public_base_url,
token=config.token,
port=config.port,
path=config.path,
public_base_url=config.public_base_url,
public_upload_url=config.public_upload_url,
config_path=paths.env_path,
enabled_tools=enabled_tools,
enabled_tools=config.enabled_tools,
)
@@ -559,59 +807,112 @@ def start_service(
*,
project_root: Path,
vault_path: Path | str | None = None,
host: str = DEFAULT_HOST,
port: int = DEFAULT_PORT,
path: str = DEFAULT_PATH,
host: str | None = None,
port: int | None = None,
path: str | None = None,
public_base_url: str | None = None,
public_upload_url: str | None = None,
token_bytes: int = DEFAULT_TOKEN_BYTES,
enable_ingest: bool | None = None,
enable_query: bool | None = None,
allow_source_files: bool | None = None,
) -> str:
paths = resolve_service_paths(project_root)
try:
token, host, port, path, public_base_url, enabled_tools, opencode_path, model_settings = saved_service_config(project_root)
saved = saved_service_config(project_root)
has_saved_config = True
except RuntimeError:
if paths.env_path.exists():
raise
token = generate_token(token_bytes)
enabled_tools = DEFAULT_ENABLED_TOOLS
opencode_path = ""
model_settings = OpenCodeModelSettings()
public_base_url = ""
enabled_tools = update_ingest_tool(enabled_tools, enable_ingest)
saved = ServiceConfig(token=generate_token(token_bytes))
has_saved_config = False
effective_host = saved.host if host is None else host
effective_port = saved.port if port is None else port
effective_path = saved.path if path is None else path
address_overridden = any(value is not None for value in (host, port, path))
enabled_tools = update_ingest_tool(saved.enabled_tools, enable_ingest)
enabled_tools = update_query_tool(enabled_tools, enable_query)
opencode_path = saved.opencode_path
if ({TOOL_INGEST, TOOL_QUERY} & set(enabled_tools)) and not opencode_path:
opencode_path = resolve_opencode_path()
hosts = detect_remote_hosts()
public_base_url = public_base_url or public_base_url_from_hosts(hosts, port=port, path=path)
write_token_env_file(
paths.env_path,
token,
host=host,
port=port,
path=path,
public_base_url=public_base_url,
if public_base_url is not None:
effective_public_base_url = normalize_public_base_url(public_base_url)
elif has_saved_config and not address_overridden:
effective_public_base_url = saved.public_base_url
else:
effective_public_base_url = public_base_url_from_hosts(
public_hosts_for_bind(effective_host, hosts),
port=effective_port,
path=effective_path,
)
effective_public_upload_url = (
saved.public_upload_url
if public_upload_url is None
else normalize_public_upload_url(public_upload_url)
)
endpoint_overridden = any(
value is not None
for value in (host, port, path, public_base_url, public_upload_url)
)
if has_saved_config and not endpoint_overridden:
allowed_hosts = saved.allowed_hosts
allowed_origins = saved.allowed_origins
else:
allowed_hosts = merge_allowlist(
saved.allowed_hosts if has_saved_config else (),
derive_allowed_hosts(
[effective_host, *hosts],
public_base_url=effective_public_base_url,
public_upload_url=effective_public_upload_url,
),
)
allowed_origins = merge_allowlist(
saved.allowed_origins if has_saved_config else (),
derive_allowed_origins(effective_public_base_url, effective_public_upload_url),
)
config = replace(
saved,
host=effective_host,
port=effective_port,
path=effective_path,
public_base_url=effective_public_base_url,
public_upload_url=effective_public_upload_url,
enabled_tools=enabled_tools,
opencode_path=opencode_path,
model_settings=model_settings,
allow_source_files=(
saved.allow_source_files if allow_source_files is None else allow_source_files
),
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
)
write_service_config(paths.env_path, config)
uv_path = find_uv_path()
prepare_runtime(paths.project_root, uv_path=uv_path)
resolved_vault = resolve_vault_for_service(paths.project_root, vault_path=vault_path)
write_service_unit(paths, vault_path=resolved_vault, host=host, port=port, path=path, uv_path=uv_path)
write_service_unit(
paths,
vault_path=resolved_vault,
host=config.host,
port=config.port,
path=config.path,
uv_path=uv_path,
)
if {TOOL_INGEST, TOOL_QUERY} & set(enabled_tools):
if {TOOL_INGEST, TOOL_QUERY} & set(config.enabled_tools):
import_proxy_environment()
run_systemctl(["daemon-reload"])
run_systemctl(["enable", SERVICE_NAME])
run_systemctl(["restart", SERVICE_NAME])
return render_remote_config(
hosts=hosts,
token=token,
port=port,
path=path,
public_base_url=public_base_url,
token=config.token,
port=config.port,
path=config.path,
public_base_url=config.public_base_url,
public_upload_url=config.public_upload_url,
config_path=paths.env_path,
enabled_tools=enabled_tools,
enabled_tools=config.enabled_tools,
)
@@ -638,59 +939,53 @@ def render_tool_status(
return "\n".join(lines) + "\n"
def ensure_service_config(project_root: Path) -> tuple[ServicePaths, str, str, int, str, str, tuple[str, ...], str, OpenCodeModelSettings]:
def ensure_service_config(project_root: Path) -> tuple[ServicePaths, ServiceConfig]:
paths = resolve_service_paths(project_root)
try:
token, host, port, path, public_base_url, enabled_tools, opencode_path, model_settings = saved_service_config(project_root)
config = saved_service_config(project_root)
except RuntimeError:
token = generate_token()
host = DEFAULT_HOST
port = DEFAULT_PORT
path = DEFAULT_PATH
public_base_url = public_base_url_from_hosts(detect_remote_hosts(), port=port, path=path)
enabled_tools = DEFAULT_ENABLED_TOOLS
opencode_path = ""
model_settings = OpenCodeModelSettings()
write_token_env_file(
paths.env_path,
token,
host=host,
port=port,
path=path,
public_base_url=public_base_url,
enabled_tools=enabled_tools,
model_settings=model_settings,
hosts = detect_remote_hosts()
public_base_url = public_base_url_from_hosts(
hosts,
port=DEFAULT_PORT,
path=DEFAULT_PATH,
)
return paths, token, host, port, path, public_base_url, enabled_tools, opencode_path, model_settings
config = ServiceConfig(
token=generate_token(),
public_base_url=public_base_url,
allowed_hosts=derive_allowed_hosts(
[DEFAULT_HOST, *hosts],
public_base_url=public_base_url,
),
allowed_origins=derive_allowed_origins(public_base_url),
)
write_service_config(paths.env_path, config)
return paths, config
def list_tools(project_root: Path) -> str:
paths, _token, _host, _port, _path, _public_base_url, enabled_tools, _opencode_path, _models = ensure_service_config(project_root)
return render_tool_status(config_path=paths.env_path, enabled_tools=enabled_tools)
paths, config = ensure_service_config(project_root)
return render_tool_status(config_path=paths.env_path, enabled_tools=config.enabled_tools)
def set_tool_enabled(project_root: Path, tool: str, enabled: bool) -> str:
tool_name = normalize_tool_name(tool)
paths, token, host, port, path, public_base_url, enabled_tools, opencode_path, model_settings = ensure_service_config(project_root)
current = set(enabled_tools)
paths, config = ensure_service_config(project_root)
current = set(config.enabled_tools)
if enabled:
current.add(tool_name)
else:
current.discard(tool_name)
updated_tools = tuple(item for item in AVAILABLE_TOOL_GROUPS if item in current)
opencode_path = config.opencode_path
if ({TOOL_INGEST, TOOL_QUERY} & set(updated_tools)) and not opencode_path:
opencode_path = resolve_opencode_path()
write_token_env_file(
paths.env_path,
token,
host=host,
port=port,
path=path,
public_base_url=public_base_url,
config = replace(
config,
enabled_tools=updated_tools,
opencode_path=opencode_path,
model_settings=model_settings,
)
write_service_config(paths.env_path, config)
if paths.unit_path.exists() and {TOOL_INGEST, TOOL_QUERY} & set(updated_tools):
import_proxy_environment()
restarted = restart_service_if_installed(paths)
@@ -847,22 +1142,26 @@ def _tree_mtime(path: Path) -> float:
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Manage the readonly HTTP obsidian-wiki MCP user service.")
parser = argparse.ArgumentParser(description="Manage the HTTP obsidian-wiki MCP user service.")
subparsers = parser.add_subparsers(dest="command", required=True)
new_token_parser = subparsers.add_parser("new-token", help="Generate a token and print remote MCP client configuration.")
new_token_parser.add_argument("--project-root", default=Path(__file__).resolve().parent.parent, type=Path)
new_token_parser.add_argument("--host", default=DEFAULT_HOST)
new_token_parser.add_argument("--port", default=DEFAULT_PORT, type=int)
new_token_parser.add_argument("--path", default=DEFAULT_PATH)
new_token_parser.add_argument("--host")
new_token_parser.add_argument("--port", type=int)
new_token_parser.add_argument("--path")
new_token_parser.add_argument("--public-base-url")
new_token_parser.add_argument("--public-upload-url")
new_token_parser.add_argument("--token-bytes", default=DEFAULT_TOKEN_BYTES, type=int)
start_parser = subparsers.add_parser("start", help="Install or update the user service and print remote MCP config.")
start_parser.add_argument("--project-root", default=Path(__file__).resolve().parent.parent, type=Path)
start_parser.add_argument("--vault", dest="vault_path", type=Path, help="Vault path to serve. Defaults to OBSIDIAN_VAULT_PATH config.")
start_parser.add_argument("--host", default=DEFAULT_HOST)
start_parser.add_argument("--port", default=DEFAULT_PORT, type=int)
start_parser.add_argument("--path", default=DEFAULT_PATH)
start_parser.add_argument("--host")
start_parser.add_argument("--port", type=int)
start_parser.add_argument("--path")
start_parser.add_argument("--public-base-url")
start_parser.add_argument("--public-upload-url")
start_parser.add_argument("--token-bytes", default=DEFAULT_TOKEN_BYTES, type=int)
ingest_group = start_parser.add_mutually_exclusive_group()
ingest_group.add_argument(
@@ -892,6 +1191,20 @@ def main(argv: Sequence[str] | None = None) -> int:
action="store_false",
help="Disable local OpenCode wiki-query MCP tools.",
)
source_files_group = start_parser.add_mutually_exclusive_group()
source_files_group.add_argument(
"--allow-source-files",
dest="allow_source_files",
action="store_true",
default=None,
help="Allow file-assets tools to return source file content.",
)
source_files_group.add_argument(
"--deny-source-files",
dest="allow_source_files",
action="store_false",
help="Deny source file content while retaining file-assets metadata tools.",
)
subparsers.add_parser("stop", help="Stop the user service without deleting service files.")
@@ -930,6 +1243,8 @@ def main(argv: Sequence[str] | None = None) -> int:
host=args.host,
port=args.port,
path=args.path,
public_base_url=args.public_base_url,
public_upload_url=args.public_upload_url,
token_bytes=args.token_bytes,
),
end="",
@@ -943,9 +1258,12 @@ def main(argv: Sequence[str] | None = None) -> int:
host=args.host,
port=args.port,
path=args.path,
public_base_url=args.public_base_url,
public_upload_url=args.public_upload_url,
token_bytes=args.token_bytes,
enable_ingest=args.enable_ingest,
enable_query=args.enable_query,
allow_source_files=args.allow_source_files,
),
end="",
)
+211 -12
View File
@@ -1,8 +1,11 @@
import hashlib
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from obsidian_wiki.access_control import source_decision as actual_source_decision
from obsidian_wiki.file_assets import connect, init_db, upsert_asset
from obsidian_wiki.mcp_assets import (
AssetAccessError,
@@ -10,7 +13,9 @@ from obsidian_wiki.mcp_assets import (
asset_uri_for_file_id,
file_id_from_asset_uri,
get_file_asset_metadata,
mcp_runtime_options,
read_file_asset_resource,
record_allowed_for_mcp,
search_file_assets,
)
@@ -23,13 +28,11 @@ class McpAssetTests(unittest.TestCase):
self.db_path = self.vault / "_meta" / "file-assets.sqlite"
self.registry_path = self.vault / "_meta" / "source-registry.jsonl"
init_db(self.db_path)
self._write_profile_file()
self.drawing_id = "sha256:" + "a" * 64
self.order_id = "sha256:" + "b" * 64
self.complaint_id = "sha256:" + "c" * 64
self._write_asset_file("drawings/a.pdf", b"%PDF allowed drawing")
self._write_asset_file("orders/order.xlsx", b"customer price data")
self._write_asset_file("quality/photo.png", b"complaint evidence")
self.drawing_id = self._write_asset_file("drawings/a.pdf", b"%PDF allowed drawing")
self.order_id = self._write_asset_file("orders/order.xlsx", b"customer price data")
self.complaint_id = self._write_asset_file("quality/photo.png", b"complaint evidence")
self._insert_asset(
self.drawing_id,
@@ -46,6 +49,7 @@ class McpAssetTests(unittest.TestCase):
series=["0.5-18"],
pitch=["0.5"],
pin_count=["18"],
allowed_bots=["bjd-general-bot", "missing-bot"],
)
self._insert_asset(
self.order_id,
@@ -77,6 +81,7 @@ class McpAssetTests(unittest.TestCase):
path = self.vault / "_sources" / "archive" / rel_path
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
return "sha256:" + hashlib.sha256(data).hexdigest()
def _insert_asset(self, file_id, **fields):
record = {
@@ -142,7 +147,17 @@ class McpAssetTests(unittest.TestCase):
"allowed_business_domains": ["products", "engineering"],
"blocked_doc_types": ["order_data", "evidence_media"],
"include_source_files": False,
}
},
"public-product-bot": {
"allowed_visibility": ["public", "internal"],
"allowed_data_classification": [
"public",
"internal",
"technical-confidential",
],
"allowed_business_domains": ["products", "engineering"],
"include_source_files": False,
},
},
}
(self.vault / "_meta" / "bot-profiles.json").write_text(json.dumps(data), encoding="utf-8")
@@ -164,9 +179,10 @@ class McpAssetTests(unittest.TestCase):
self.assertEqual(rows[0]["pin_count"], ["18"])
def test_customer_shareable_engineering_files_work_with_source_file_profile(self):
self._write_profile_file()
shareable_id = "sha256:" + "d" * 64
self._write_asset_file("drawings/customer-shareable.pdf", b"%PDF customer drawing")
shareable_id = self._write_asset_file(
"drawings/customer-shareable.pdf",
b"%PDF customer drawing",
)
self._insert_asset(
shareable_id,
archive_path="_sources/archive/drawings/customer-shareable.pdf",
@@ -193,7 +209,10 @@ class McpAssetTests(unittest.TestCase):
def test_asset_uri_round_trips_file_id(self):
uri = asset_uri_for_file_id(self.drawing_id)
self.assertEqual(uri, "asset://source/sha256/" + "a" * 64)
self.assertEqual(
uri,
"asset://source/sha256/" + self.drawing_id.removeprefix("sha256:"),
)
self.assertEqual(file_id_from_asset_uri(uri), self.drawing_id)
def test_resource_read_rechecks_acl_and_returns_file_bytes(self):
@@ -201,6 +220,7 @@ class McpAssetTests(unittest.TestCase):
self.vault,
asset_uri_for_file_id(self.drawing_id),
db_path=self.db_path,
allow_source_files=True,
)
self.assertEqual(data["name"], "B3501_11101c_X2.pdf")
@@ -213,6 +233,7 @@ class McpAssetTests(unittest.TestCase):
self.vault,
asset_uri_for_file_id(self.order_id),
db_path=self.db_path,
allow_source_files=True,
)
def test_resource_read_enforces_max_size(self):
@@ -222,6 +243,67 @@ class McpAssetTests(unittest.TestCase):
asset_uri_for_file_id(self.drawing_id),
db_path=self.db_path,
max_file_bytes=5,
allow_source_files=True,
)
def test_resource_read_requires_explicit_server_opt_in(self):
with self.assertRaisesRegex(AssetAccessError, "source file reads are disabled"):
read_file_asset_resource(
self.vault,
asset_uri_for_file_id(self.drawing_id),
db_path=self.db_path,
)
def test_runtime_options_source_file_gate_defaults_closed_and_reads_env(self):
with patch.dict("os.environ", {}, clear=True):
self.assertFalse(mcp_runtime_options()["allow_source_files"])
with patch.dict("os.environ", {"OBSIDIAN_MCP_ALLOW_SOURCE_FILES": "1"}, clear=True):
self.assertTrue(mcp_runtime_options()["allow_source_files"])
def test_resource_read_rejects_db_registry_archive_path_mismatch(self):
other_path = "_sources/archive/drawings/other.pdf"
self._write_asset_file("drawings/other.pdf", b"%PDF other drawing")
with connect(self.db_path) as conn:
conn.execute(
"UPDATE file_assets SET archive_path = ? WHERE file_id = ?",
(other_path, self.drawing_id),
)
with self.assertRaisesRegex(AssetAccessError, "archive_path mismatch"):
read_file_asset_resource(
self.vault,
asset_uri_for_file_id(self.drawing_id),
db_path=self.db_path,
allow_source_files=True,
)
def test_resource_read_accepts_normalization_equivalent_archive_paths(self):
with connect(self.db_path) as conn:
conn.execute(
"UPDATE file_assets SET archive_path = ? WHERE file_id = ?",
("_sources/archive/drawings/../drawings/a.pdf", self.drawing_id),
)
data = read_file_asset_resource(
self.vault,
asset_uri_for_file_id(self.drawing_id),
db_path=self.db_path,
allow_source_files=True,
)
self.assertEqual(data["bytes"], b"%PDF allowed drawing")
def test_resource_read_rejects_content_that_no_longer_matches_file_id(self):
(self.vault / "_sources" / "archive" / "drawings" / "a.pdf").write_bytes(
b"%PDF tampered drawing"
)
with self.assertRaisesRegex(AssetAccessError, "content hash mismatch"):
read_file_asset_resource(
self.vault,
asset_uri_for_file_id(self.drawing_id),
db_path=self.db_path,
allow_source_files=True,
)
def test_metadata_uses_safe_fields(self):
@@ -232,7 +314,7 @@ class McpAssetTests(unittest.TestCase):
self.assertEqual(metadata["resource_uri"], asset_uri_for_file_id(self.drawing_id))
self.assertNotIn("absolute_path", metadata)
def test_registry_allowed_bots_are_enforced_without_profile_file(self):
def test_registry_allowed_bots_are_enforced_with_valid_profile(self):
with self.assertRaises(AssetAccessError):
get_file_asset_metadata(
self.vault,
@@ -241,6 +323,123 @@ class McpAssetTests(unittest.TestCase):
profile_name="public-product-bot",
)
def test_missing_requested_profile_fails_closed(self):
with self.assertRaisesRegex(AssetAccessError, "bot profile not found"):
search_file_assets(
self.vault,
"B3501",
db_path=self.db_path,
profile_name="missing-bot",
)
with self.assertRaisesRegex(AssetAccessError, "bot profile not found"):
get_file_asset_metadata(
self.vault,
self.drawing_id,
db_path=self.db_path,
profile_name="missing-bot",
)
def test_metadata_acl_does_not_rewrite_profile_source_file_setting(self):
profile = {
"allowed_visibility": ["internal"],
"allowed_data_classification": ["technical-confidential"],
"allowed_business_domains": ["products", "engineering"],
"include_source_files": False,
}
record = {
"archive_path": "_sources/archive/drawings/a.pdf",
"doc_type": "drawing",
"visibility": "internal",
"data_classification": "customer-shareable",
"business_categories": ["products/btb", "drawings/customer"],
"allowed_bots": ["bjd-general-bot"],
}
with patch(
"obsidian_wiki.mcp_assets.source_decision",
wraps=actual_source_decision,
) as mocked_decision:
allowed, reason = record_allowed_for_mcp(
record,
profile_name="bjd-general-bot",
profile=profile,
)
self.assertTrue(allowed, reason)
runtime_profile = mocked_decision.call_args.args[1]
self.assertIsNot(runtime_profile, profile)
self.assertTrue(runtime_profile["include_source_files"])
self.assertIn("customer-shareable", runtime_profile["allowed_data_classification"])
self.assertFalse(profile["include_source_files"])
self.assertNotIn("customer-shareable", profile["allowed_data_classification"])
def test_search_filters_acl_before_applying_limit(self):
allowed_id = self._write_asset_file(
"drawings/allowed-limit.pdf",
b"%PDF allowed lower-ranked limit result",
)
self._insert_asset(
allowed_id,
archive_path="_sources/archive/drawings/allowed-limit.pdf",
original_name="Allowed limit drawing.pdf",
doc_type="drawing",
source_type="pdf",
product_lines=["BTB"],
business_categories=["products/btb"],
data_classification="technical-confidential",
visibility="internal",
summary="BLOCKME appears only as unstructured text.",
)
for index in range(4):
blocked_id = self._write_asset_file(
f"drawings/blocked-limit-{index}.pdf",
f"%PDF blocked limit result {index}".encode(),
)
self._insert_asset(
blocked_id,
archive_path=f"_sources/archive/drawings/blocked-limit-{index}.pdf",
original_name=f"Blocked limit drawing {index}.pdf",
doc_type="drawing",
source_type="pdf",
product_lines=["BTB"],
business_categories=["products/btb"],
data_classification="technical-confidential",
visibility="internal",
summary="BLOCKME structured identifier result.",
bjd_part_numbers=["BLOCKME"],
allowed_bots=["other-bot"],
)
rows = search_file_assets(self.vault, "BLOCKME", db_path=self.db_path, limit=1)
self.assertEqual([row["file_id"] for row in rows], [allowed_id])
def test_search_clamps_limit_to_public_range(self):
for index in range(101):
file_id = self._write_asset_file(
f"drawings/bulk-{index}.pdf",
f"%PDF BULK-LIMIT {index}".encode(),
)
self._insert_asset(
file_id,
archive_path=f"_sources/archive/drawings/bulk-{index}.pdf",
original_name=f"Bulk limit drawing {index}.pdf",
doc_type="drawing",
source_type="pdf",
product_lines=["BTB"],
business_categories=["products/btb"],
data_classification="technical-confidential",
visibility="internal",
summary="BULK-LIMIT clamp candidate.",
)
lower = search_file_assets(self.vault, "B3501", db_path=self.db_path, limit=0)
upper = search_file_assets(self.vault, "BULK-LIMIT", db_path=self.db_path, limit=1000)
self.assertEqual(len(lower), 1)
self.assertEqual(len(upper), 100)
if __name__ == "__main__":
unittest.main()
+154
View File
@@ -1,4 +1,5 @@
import base64
import io
import json
import os
import subprocess
@@ -35,6 +36,7 @@ class McpIngestTests(unittest.TestCase):
)
def tearDown(self):
self.manager.close()
self.tempdir.cleanup()
def test_parse_bool_accepts_enabled_values_only(self):
@@ -45,6 +47,18 @@ class McpIngestTests(unittest.TestCase):
self.assertFalse(parse_bool("0"))
self.assertFalse(parse_bool("false"))
def test_close_still_closes_supervisor_when_queued_job_write_fails(self):
queued = {"state": "queued"}
with (
patch.object(self.manager, "_iter_jobs", return_value=[queued]),
patch.object(self.manager, "_write_job", side_effect=OSError("disk full")),
patch.object(self.manager._supervisor, "close") as supervisor_close,
):
with self.assertRaises(OSError):
self.manager.close()
supervisor_close.assert_called_once_with()
def test_uploads_are_staged_in_runtime_not_vault(self):
upload = self.manager.create_upload(file_name="drawing.pdf", size_bytes=0, sha256=self._sha256(b""))
upload_path = self._upload_payload_path(upload["upload_id"]).resolve()
@@ -193,6 +207,56 @@ class McpIngestTests(unittest.TestCase):
self.assertNotIn("path", upload)
self.assertEqual(self._upload_payload_path(upload["upload_id"]).read_bytes(), payload)
def test_create_completed_upload_from_stream_reads_bounded_chunks(self):
payload = b"streamed upload payload" * 100_000
class BoundedReadStream(io.BytesIO):
def __init__(self, data):
super().__init__(data)
self.read_sizes = []
def read(self, size=-1):
if size < 0:
raise AssertionError("stream upload must not use an unbounded read")
self.read_sizes.append(size)
return super().read(size)
manager = IngestManager(
vault_path=self.vault,
project_root=self.project,
runtime_root=self.root / "stream-runtime",
max_file_bytes=len(payload) + 1,
)
stream = BoundedReadStream(payload)
upload = manager.create_completed_upload_from_stream(
file_name="stream.bin",
stream=stream,
size_bytes=len(payload),
)
self.assertEqual(upload["state"], "complete")
self.assertEqual(upload["sha256"], "sha256:" + self._sha256(payload))
self.assertGreater(len(stream.read_sizes), 2)
self.assertTrue(all(size == 1024 * 1024 for size in stream.read_sizes))
def test_create_completed_upload_from_stream_cleans_partial_file_when_oversized(self):
manager = IngestManager(
vault_path=self.vault,
project_root=self.project,
runtime_root=self.root / "small-stream-runtime",
max_file_bytes=5,
)
with self.assertRaises(IngestError):
manager.create_completed_upload_from_stream(
file_name="too-large.bin",
stream=io.BytesIO(b"123456"),
)
self.assertEqual(list(manager.upload_root.rglob("upload.json")), [])
self.assertEqual([path for path in manager.upload_root.rglob("*") if path.is_file()], [])
def test_upload_ticket_is_single_use_and_preserves_defaults(self):
ticket = self.manager.create_upload_ticket(
expires_seconds=60,
@@ -341,6 +405,8 @@ class McpIngestTests(unittest.TestCase):
self.assertNotIn("../secret", prompt)
self.assertNotIn("path", job["files"][0])
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text(self._json_text_event("MCP_INGEST_OK\nIngested source.pdf"), encoding="utf-8")
popen.returncode = 0
self.manager.refresh_jobs()
status = self.manager.get_job_status(job["job_id"])
@@ -348,6 +414,61 @@ class McpIngestTests(unittest.TestCase):
self.assertEqual(status["exit_code"], 0)
self.assertNotIn("path", status["files"][0])
def test_ingest_result_rejects_skill_failure_with_zero_exit_code(self):
upload = self._complete_upload("source.pdf", b"source data")
popen = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_ingest.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_ingest.subprocess.Popen", return_value=popen),
):
job = self.manager.start_ingest_job([upload["upload_id"]])
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text(self._json_text_event("MCP_INGEST_FAILED\nSKILL_LOAD_FAILED: missing"), encoding="utf-8")
popen.returncode = 0
result = self.manager.get_ingest_job_result(job["job_id"])
self.assertEqual(result["state"], "failed")
self.assertIn("SKILL_LOAD_FAILED", result["error"])
def test_ingest_public_status_hides_raw_tool_logs_paths_and_pid(self):
upload = self._complete_upload("source.pdf", b"source data")
popen = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_ingest.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_ingest.subprocess.Popen", return_value=popen),
):
job = self.manager.start_ingest_job([upload["upload_id"]])
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text(
'{"type":"tool_use","part":{"state":{"output":"/home/private/secret"}}}\n'
+ self._json_text_event("MCP_INGEST_OK\nIngest complete"),
encoding="utf-8",
)
popen.returncode = 0
result = self.manager.get_ingest_job_result(job["job_id"])
status = self.manager.get_ingest_job_status(job["job_id"])
self.assertEqual(result["state"], "succeeded")
self.assertEqual(result["opencode_output_tail"], "Ingest complete")
self.assertNotIn("/home/private", result["opencode_output_tail"])
self.assertNotIn("pid", status)
self.assertNotIn("checked_paths", status["preflight"])
internal = self.manager._load_job(job["job_id"])
internal["error"] = f"failed while reading {self.project / 'secret.txt'}"
internal["preflight"]["warnings"] = [f"check {self.runtime / 'private.log'}"]
self.manager._write_job(internal)
redacted = self.manager.get_ingest_job_status(job["job_id"])
serialized = json.dumps(redacted)
self.assertNotIn(str(self.project), serialized)
self.assertNotIn(str(self.runtime), serialized)
self.assertIn("<project>", serialized)
self.assertIn("<runtime>", serialized)
def test_jobs_queue_one_at_a_time_and_cancel_queued_job(self):
first = self._complete_upload("first.pdf", b"first")
second = self._complete_upload("second.pdf", b"second")
@@ -367,6 +488,27 @@ class McpIngestTests(unittest.TestCase):
self.assertEqual(cancelled["state"], "cancelled")
def test_start_ingest_job_is_idempotent_for_claimed_uploads(self):
upload = self._complete_upload("source.pdf", b"source")
popen = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_ingest.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_ingest.subprocess.Popen", return_value=popen) as popen_mock,
):
first = self.manager.start_ingest_job([upload["upload_id"]])
repeated = self.manager.start_ingest_job([upload["upload_id"]])
self.assertEqual(repeated["job_id"], first["job_id"])
self.assertEqual(repeated["state"], "running")
popen_mock.assert_called_once()
def test_start_ingest_job_rejects_duplicate_upload_ids(self):
upload = self._complete_upload("source.pdf", b"source")
with self.assertRaisesRegex(IngestError, "upload_ids must be unique"):
self.manager.start_ingest_job([upload["upload_id"], upload["upload_id"]])
def test_cleanup_runtime_prunes_expired_tickets_and_old_completed_artifacts(self):
upload = self._complete_upload("old.pdf", b"old payload")
ticket = self.manager.create_upload_ticket(expires_seconds=60, max_files=1)
@@ -476,6 +618,18 @@ class McpIngestTests(unittest.TestCase):
raise subprocess.TimeoutExpired(args, kwargs.get("timeout"))
raise AssertionError(f"unexpected subprocess.run args: {args}")
def _json_text_event(self, text):
return json.dumps(
{
"type": "text",
"part": {
"type": "text",
"text": text,
"metadata": {"openai": {"phase": "final_answer"}},
},
}
) + "\n"
class FakePopen:
def __init__(self, returncode=None):
+207
View File
@@ -0,0 +1,207 @@
import json
import subprocess
import tempfile
import threading
import time
import unittest
from pathlib import Path
from obsidian_wiki.mcp_jobs import (
ProcessSupervisor,
RuntimeLeaseError,
extract_final_answer,
parse_marked_result,
)
class ProcessSupervisorTests(unittest.TestCase):
def test_long_final_answer_keeps_protocol_marker_when_truncated(self):
with tempfile.TemporaryDirectory() as tmp:
log_path = Path(tmp) / "opencode.log"
answer = "MCP_QUERY_OK\n" + ("x" * 20_000)
log_path.write_text(
'{"type":"text","part":{"text":' + json.dumps(answer) + "}}\n",
encoding="utf-8",
)
extracted = extract_final_answer(log_path, limit=12_000)
ok, public_text, error = parse_marked_result(
extracted,
success_marker="MCP_QUERY_OK",
failure_marker="MCP_QUERY_FAILED",
)
self.assertTrue(ok, error)
self.assertTrue(extracted.startswith("MCP_QUERY_OK\n"))
self.assertLessEqual(len(extracted), 12_000)
self.assertEqual(len(public_text), 12_000 - len("MCP_QUERY_OK\n"))
def test_completion_callback_runs_without_client_polling(self):
process = FakeProcess(returncode=0)
completed = threading.Event()
outcomes = []
with tempfile.TemporaryDirectory() as tmp:
supervisor = ProcessSupervisor(
Path(tmp),
popen_factory=lambda *_args, **_kwargs: process,
poll_interval_seconds=0.01,
)
supervisor.start(
"job-1",
["opencode", "run"],
cwd=Path(tmp),
log_path=Path(tmp) / "job.log",
timeout_seconds=30,
on_complete=lambda outcome: (outcomes.append(outcome), completed.set()),
)
self.assertTrue(completed.wait(1))
self.assertEqual(outcomes[0].returncode, 0)
self.assertFalse(outcomes[0].cancelled)
self.assertFalse(outcomes[0].timed_out)
supervisor.close()
def test_completion_callback_failure_invokes_error_fallback(self):
process = FakeProcess(returncode=0)
recovered = threading.Event()
failures = []
with tempfile.TemporaryDirectory() as tmp:
supervisor = ProcessSupervisor(
Path(tmp),
popen_factory=lambda *_args, **_kwargs: process,
poll_interval_seconds=0.01,
)
def fail_callback(_outcome):
raise RuntimeError("callback failed")
supervisor.start(
"job-1",
["opencode", "run"],
cwd=Path(tmp),
log_path=Path(tmp) / "job.log",
timeout_seconds=30,
on_complete=fail_callback,
on_callback_error=lambda outcome, error: (
failures.append((outcome, error)),
recovered.set(),
),
)
self.assertTrue(recovered.wait(1))
self.assertEqual(failures[0][0].job_id, "job-1")
self.assertEqual(str(failures[0][1]), "callback failed")
supervisor.close()
def test_cancel_waits_for_exit_before_reporting_completion(self):
process = FakeProcess(returncode=None, terminate_returncode=-15)
outcomes = []
with tempfile.TemporaryDirectory() as tmp:
supervisor = ProcessSupervisor(
Path(tmp),
popen_factory=lambda *_args, **_kwargs: process,
poll_interval_seconds=0.01,
)
supervisor.start(
"job-1",
["opencode", "run"],
cwd=Path(tmp),
log_path=Path(tmp) / "job.log",
timeout_seconds=30,
on_complete=outcomes.append,
)
self.assertTrue(supervisor.cancel("job-1", grace_seconds=0.1))
self.assertTrue(process.terminated)
self.assertEqual(process.wait_calls, 1)
self.assertEqual(len(outcomes), 1)
self.assertTrue(outcomes[0].cancelled)
self.assertEqual(outcomes[0].returncode, -15)
supervisor.close()
def test_timeout_terminates_process_and_marks_outcome(self):
process = FakeProcess(returncode=None, terminate_returncode=-15)
completed = threading.Event()
outcomes = []
with tempfile.TemporaryDirectory() as tmp:
supervisor = ProcessSupervisor(
Path(tmp),
popen_factory=lambda *_args, **_kwargs: process,
poll_interval_seconds=0.01,
)
supervisor.start(
"job-1",
["opencode", "run"],
cwd=Path(tmp),
log_path=Path(tmp) / "job.log",
timeout_seconds=0.01,
on_complete=lambda outcome: (outcomes.append(outcome), completed.set()),
)
self.assertTrue(completed.wait(1))
self.assertTrue(process.terminated)
self.assertTrue(outcomes[0].timed_out)
supervisor.close()
def test_runtime_root_allows_only_one_supervisor(self):
with tempfile.TemporaryDirectory() as tmp:
first = ProcessSupervisor(Path(tmp), poll_interval_seconds=0.01)
with self.assertRaises(RuntimeLeaseError):
ProcessSupervisor(Path(tmp), poll_interval_seconds=0.01)
first.close()
def test_closed_supervisor_rejects_start_without_spawning(self):
calls = []
with tempfile.TemporaryDirectory() as tmp:
supervisor = ProcessSupervisor(
Path(tmp),
popen_factory=lambda *_args, **_kwargs: calls.append(True),
poll_interval_seconds=0.01,
)
supervisor.close()
with self.assertRaises(RuntimeError):
supervisor.start(
"job-1",
["opencode", "run"],
cwd=Path(tmp),
log_path=Path(tmp) / "job.log",
timeout_seconds=30,
on_complete=lambda _outcome: None,
)
self.assertEqual(calls, [])
class FakeProcess:
def __init__(self, *, returncode, terminate_returncode=-15):
self.returncode = returncode
self.terminate_returncode = terminate_returncode
self.pid = 4242
self.terminated = False
self.killed = False
self.wait_calls = 0
def poll(self):
return self.returncode
def wait(self, timeout=None):
self.wait_calls += 1
if self.returncode is None:
raise subprocess.TimeoutExpired("fake", timeout)
return self.returncode
def terminate(self):
self.terminated = True
self.returncode = self.terminate_returncode
def kill(self):
self.killed = True
self.returncode = -9
if __name__ == "__main__":
unittest.main()
+171 -8
View File
@@ -1,4 +1,5 @@
import subprocess
import json
import os
import tempfile
import time
@@ -28,6 +29,7 @@ class McpQueryTests(unittest.TestCase):
)
def tearDown(self):
self.manager.close()
self.tempdir.cleanup()
def test_check_opencode_readiness_static_success(self):
@@ -71,7 +73,7 @@ class McpQueryTests(unittest.TestCase):
self.assertEqual(result["runtime_smoke_test"], "passed")
self.assertEqual(run_mock.call_count, 3)
smoke_args = run_mock.call_args_list[-1].args[0]
self.assertEqual(smoke_args[:4], ["/usr/bin/opencode", "run", "--format", "default"])
self.assertEqual(smoke_args[:4], ["/usr/bin/opencode", "run", "--format", "json"])
self.assertEqual(smoke_args[smoke_args.index("--model") + 1], "openai/gpt-5.5")
self.assertEqual(smoke_args[smoke_args.index("--variant") + 1], "minimal")
self.assertIn("--auto", smoke_args)
@@ -86,6 +88,18 @@ class McpQueryTests(unittest.TestCase):
with self.assertRaises(QueryError):
self.manager.start_query_job("what do I know?", mode="full")
def test_close_still_closes_supervisor_when_queued_job_write_fails(self):
queued = {"state": "queued"}
with (
patch.object(self.manager, "_iter_jobs", return_value=[queued]),
patch.object(self.manager, "_write_job", side_effect=OSError("disk full")),
patch.object(self.manager._supervisor, "close") as supervisor_close,
):
with self.assertRaises(OSError):
self.manager.close()
supervisor_close.assert_called_once_with()
def test_start_query_job_constructs_restricted_opencode_command(self):
self.manager.opencode_model = "openai/gpt-5.5"
self.manager.opencode_variant = "minimal"
@@ -103,7 +117,7 @@ class McpQueryTests(unittest.TestCase):
)
args = popen_mock.call_args.args[0]
self.assertEqual(args[:5], ["/usr/bin/opencode", "run", "--format", "default", "--auto"])
self.assertEqual(args[:5], ["/usr/bin/opencode", "run", "--format", "json", "--auto"])
self.assertEqual(args[args.index("--model") + 1], "openai/gpt-5.5")
self.assertEqual(args[args.index("--variant") + 1], "minimal")
self.assertIn("--dir", args)
@@ -144,7 +158,10 @@ class McpQueryTests(unittest.TestCase):
status = self.manager.get_query_job_status(job["job_id"])
self.assertNotIn("log_path", status)
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text("Based on the wiki:\nAnswer body\n", encoding="utf-8")
log_path.write_text(
self._json_text_event("MCP_QUERY_OK\nBased on the wiki:\nAnswer body"),
encoding="utf-8",
)
popen.returncode = 0
result = self.manager.get_query_job_result(job["job_id"])
@@ -170,9 +187,12 @@ class McpQueryTests(unittest.TestCase):
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text(
"Based on [[Mini FAKRA]] and [[BJD]].\n"
"Business facts used. Data through 2026-06-02.\n"
"Answer body\n",
self._json_text_event(
"MCP_QUERY_OK\n"
"Based on [[Mini FAKRA]] and [[BJD]].\n"
"Business facts used. Data through 2026-06-02.\n"
"Answer body"
),
encoding="utf-8",
)
popen.returncode = 0
@@ -195,13 +215,67 @@ class McpQueryTests(unittest.TestCase):
job = self.manager.start_query_job("what failed?")
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text("Traceback: failed\n", encoding="utf-8")
log_path.write_text(self._json_text_event("MCP_QUERY_FAILED\nTraceback: failed"), encoding="utf-8")
result = self.manager.get_query_job_result(job["job_id"])
self.assertEqual(result["state"], "failed")
self.assertEqual(result["confidence"], "low")
self.assertTrue(any("failed" in warning.lower() for warning in result["warnings"]))
def test_query_result_rejects_skill_failure_with_zero_exit_code(self):
popen = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_query.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_query.subprocess.Popen", return_value=popen),
):
job = self.manager.start_query_job("what failed?")
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text(self._json_text_event("MCP_QUERY_FAILED\nSKILL_LOAD_FAILED: missing"), encoding="utf-8")
popen.returncode = 0
result = self.manager.get_query_job_result(job["job_id"])
self.assertEqual(result["state"], "failed")
self.assertEqual(result["confidence"], "low")
self.assertIn("SKILL_LOAD_FAILED", result["error"])
def test_query_result_accepts_only_final_answer_event_and_hides_paths(self):
popen = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_query.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_query.subprocess.Popen", return_value=popen),
):
job = self.manager.start_query_job("safe answer?")
log_path = Path(self.manager._load_job(job["job_id"])["log_path"])
log_path.write_text(
'{"type":"tool_use","part":{"state":{"output":"/home/private/secret"}}}\n'
+ self._json_text_event("MCP_QUERY_OK\nBased on [[BJD]]."),
encoding="utf-8",
)
popen.returncode = 0
result = self.manager.get_query_job_result(job["job_id"])
self.assertEqual(result["state"], "succeeded")
self.assertEqual(result["answer_markdown"], "Based on [[BJD]].")
self.assertNotIn("/home/private", result["opencode_output_tail"])
status = self.manager.get_query_job_status(job["job_id"])
self.assertNotIn("pid", status)
self.assertNotIn("checked_paths", status["preflight"])
internal = self.manager._load_job(job["job_id"])
internal["error"] = f"failed while reading {self.project / 'secret.txt'}"
internal["preflight"]["warnings"] = [f"check {self.runtime / 'private.log'}"]
self.manager._write_job(internal)
redacted = self.manager.get_query_job_status(job["job_id"])
serialized = json.dumps(redacted)
self.assertNotIn(str(self.project), serialized)
self.assertNotIn(str(self.runtime), serialized)
self.assertIn("<project>", serialized)
self.assertIn("<runtime>", serialized)
def test_ask_wiki_returns_timeout_payload_with_job_id(self):
popen = FakePopen(returncode=None)
@@ -237,6 +311,76 @@ class McpQueryTests(unittest.TestCase):
self.assertEqual(cancelled["state"], "cancelled")
def test_close_cancels_running_and_queued_jobs_and_releases_runtime(self):
popen = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_query.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_query.subprocess.Popen", return_value=popen),
):
first = self.manager.start_query_job("first query")
second = self.manager.start_query_job("second query")
self.manager.close()
self.assertEqual(self.manager._load_job(first["job_id"])["state"], "cancelled")
self.assertEqual(self.manager._load_job(second["job_id"])["state"], "cancelled")
replacement = QueryManager(
vault_path=self.vault,
project_root=self.project,
runtime_root=self.runtime,
opencode_path=Path("/usr/bin/opencode"),
)
replacement.close()
with self.assertRaises(QueryError):
self.manager.start_query_job("after close")
def test_completed_job_advances_queue_without_status_polling(self):
first_process = FakePopen(returncode=None)
second_process = FakePopen(returncode=None)
with (
patch("obsidian_wiki.mcp_query.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_query.subprocess.Popen", side_effect=[first_process, second_process]),
):
first = self.manager.start_query_job("first query")
second = self.manager.start_query_job("second query")
first_log = Path(self.manager._load_job(first["job_id"])["log_path"])
first_log.write_text(self._json_text_event("MCP_QUERY_OK\nfirst answer"), encoding="utf-8")
first_process.returncode = 0
deadline = time.monotonic() + 1
while time.monotonic() < deadline and self.manager._load_job(second["job_id"])["state"] != "running":
time.sleep(0.01)
self.assertEqual(self.manager._load_job(first["job_id"])["state"], "succeeded")
self.assertEqual(self.manager._load_job(second["job_id"])["state"], "running")
def test_failed_preflight_does_not_block_later_queued_job(self):
first_process = FakePopen(returncode=None)
third_process = FakePopen(returncode=None)
failed_readiness = {"ok": False, "errors": ["skill missing"]}
successful_readiness = {"ok": True, "errors": []}
with (
patch("obsidian_wiki.mcp_query.subprocess.run", side_effect=self._fake_static_readiness_run),
patch("obsidian_wiki.mcp_query.subprocess.Popen", side_effect=[first_process, third_process]),
):
first = self.manager.start_query_job("first query")
second = self.manager.start_query_job("second query")
third = self.manager.start_query_job("third query")
first_log = Path(self.manager._load_job(first["job_id"])["log_path"])
first_log.write_text(self._json_text_event("MCP_QUERY_OK\nfirst answer"), encoding="utf-8")
with patch.object(
self.manager,
"check_opencode_readiness",
side_effect=[failed_readiness, successful_readiness],
):
completed = self.manager._supervisor._complete_naturally(first["job_id"], 0)
self.assertTrue(completed)
self.assertEqual(self.manager._load_job(second["job_id"])["state"], "failed")
self.assertEqual(self.manager._load_job(third["job_id"])["state"], "running")
def test_cleanup_runtime_prunes_old_terminal_query_jobs(self):
popen = FakePopen(returncode=0)
with (
@@ -290,9 +434,28 @@ class McpQueryTests(unittest.TestCase):
if args in (["/usr/bin/opencode", "--version"], ["/usr/bin/opencode", "run", "--help"]):
return self._fake_static_readiness_run(args, **kwargs)
if args[:3] == ["/usr/bin/opencode", "run", "--format"]:
return subprocess.CompletedProcess(args, 0, stdout="WIKI_QUERY_SKILL_LOADED\n", stderr="")
return subprocess.CompletedProcess(
args,
0,
stdout=self._json_text_event("WIKI_QUERY_SKILL_LOADED"),
stderr="",
)
raise AssertionError(f"unexpected subprocess.run args: {args}")
def _json_text_event(self, text):
import json
return json.dumps(
{
"type": "text",
"part": {
"type": "text",
"text": text,
"metadata": {"openai": {"phase": "final_answer"}},
},
}
) + "\n"
class FakePopen:
def __init__(self, returncode=None):
+661 -3
View File
@@ -3,6 +3,7 @@ import hashlib
import json
import os
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import patch
@@ -129,6 +130,14 @@ class RemoteMcpServerTests(unittest.TestCase):
with patch.dict(os.environ, {"OBSIDIAN_MCP_PORT": "not-an-int"}, clear=True):
self.assertEqual(mcp_server.runtime_config_int("OBSIDIAN_MCP_PORT", 8765), 8765)
with patch.dict(os.environ, {"OBSIDIAN_MCP_ALLOWED_HOSTS": "wiki.example:*, 10.0.0.2:*"}, clear=True):
self.assertEqual(
mcp_server.runtime_config_list("OBSIDIAN_MCP_ALLOWED_HOSTS"),
["wiki.example:*", "10.0.0.2:*"],
)
with patch.dict(os.environ, {}, clear=True):
self.assertIsNone(mcp_server.runtime_config_list("OBSIDIAN_MCP_ALLOWED_HOSTS"))
def test_enabled_tools_config_prefers_tool_list_over_legacy_ingest_flag(self):
with patch.dict(
os.environ,
@@ -200,6 +209,130 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(data["uploads"][0]["sha256"], "sha256:" + hashlib.sha256(payload).hexdigest())
self.assertNotIn("path", data["uploads"][0])
def test_http_upload_endpoint_streams_multipart_without_buffering_whole_body(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
payload = b"stream this upload"
body, boundary = multipart_body(
{"start_job": "false"},
[("files", "stream.txt", "text/plain", payload)],
)
with (
patch(
"obsidian_wiki.mcp_server._read_http_body",
side_effect=AssertionError("buffered body"),
create=True,
),
patch.object(manager, "create_completed_upload", side_effect=AssertionError("bytes upload")),
):
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii")),
],
body=body,
)
self.assertEqual(messages[0]["status"], 200)
response = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body")
data = json.loads(response.decode("utf-8"))
self.assertEqual(data["uploads"][0]["sha256"], "sha256:" + hashlib.sha256(payload).hexdigest())
def test_http_upload_file_copy_does_not_block_event_loop(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
body, boundary = multipart_body(
{"start_job": "false"},
[("files", "stream.txt", "text/plain", b"stream this upload")],
)
original = manager.create_completed_upload_from_stream
def slow_copy(*args, **kwargs):
time.sleep(0.1)
return original(*args, **kwargs)
async def exercise():
messages = []
sent = False
scope = {
"type": "http",
"method": "POST",
"path": "/mcp-ingest/uploads",
"raw_path": b"/mcp-ingest/uploads",
"query_string": b"",
"headers": [
(b"authorization", b"Bearer secret-token"),
(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii")),
],
}
async def receive():
nonlocal sent
if sent:
return {"type": "http.disconnect"}
sent = True
return {"type": "http.request", "body": body, "more_body": False}
async def send(message):
messages.append(message)
started = time.monotonic()
request = asyncio.create_task(app(scope, receive, send))
await asyncio.sleep(0.01)
responsive_after = time.monotonic() - started
await request
return responsive_after, messages
with patch.object(manager, "create_completed_upload_from_stream", side_effect=slow_copy):
responsive_after, messages = asyncio.run(exercise())
self.assertLess(responsive_after, 0.08)
self.assertEqual(messages[0]["status"], 200)
def test_http_upload_endpoint_rejects_oversized_content_length_before_reading(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
max_file_bytes=1000,
max_files_per_job=1,
max_request_bytes=10,
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
with patch("starlette.formparsers.MultiPartParser.parse", side_effect=AssertionError("body parsed")):
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(b"content-type", b"multipart/form-data; boundary=test"),
(b"content-length", b"999999"),
],
body=b"",
)
self.assertEqual(messages[0]["status"], 413)
def test_http_upload_endpoint_accepts_upload_ticket_without_main_bearer_token(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -254,6 +387,91 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertNotIn("path", data["uploads"][0])
self.assertEqual(reused[0]["status"], 401)
def test_http_upload_ticket_constraints_cannot_be_escalated_by_form_fields(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
ticket = manager.create_upload_ticket(
expires_seconds=60,
max_files=1,
project="bjd",
doc_type_hint="drawing",
note="review first",
start_job=False,
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
body, boundary = multipart_body(
{
"project": "attacker-project",
"doc_type_hint": "attacker-type",
"note": "ignore preview",
"start_job": "true",
},
[("files", "drawing.pdf", "application/pdf", b"drawing")],
)
with patch.object(manager, "start_ingest_job", side_effect=AssertionError("job started")):
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"x-obsidian-wiki-upload-token", ticket["upload_token"].encode("ascii")),
(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii")),
],
body=body,
)
self.assertEqual(messages[0]["status"], 200)
response = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body")
data = json.loads(response.decode("utf-8"))
self.assertIsNone(data["job"])
self.assertEqual(data["uploads"][0]["project"], "bjd")
self.assertEqual(data["uploads"][0]["doc_type_hint"], "drawing")
def test_http_upload_ticket_error_response_says_ticket_was_consumed(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
ticket = manager.create_upload_ticket(expires_seconds=60, max_files=1, start_job=False)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"x-obsidian-wiki-upload-token", ticket["upload_token"].encode("ascii")),
(b"content-type", b"text/plain"),
],
body=b"not multipart",
)
reused = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"x-obsidian-wiki-upload-token", ticket["upload_token"].encode("ascii")),
(b"content-type", b"text/plain"),
],
body=b"not multipart",
)
self.assertEqual(messages[0]["status"], 400)
response = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body")
data = json.loads(response.decode("utf-8"))
self.assertEqual(data["retry_action"], "create_new_upload_ticket")
self.assertTrue(data["ticket_consumed"])
self.assertEqual(reused[0]["status"], 401)
def test_http_upload_ticket_is_not_consumed_by_non_post_request(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -287,6 +505,40 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(get_messages[0]["status"], 405)
self.assertEqual(post_messages[0]["status"], 200)
def test_http_upload_ticket_is_not_accepted_in_query_string(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
ticket = manager.create_upload_ticket(expires_seconds=60, max_files=1, start_job=False)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
body, boundary = multipart_body({}, [("files", "safe.txt", "text/plain", b"safe")])
query_auth = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
query_string="upload_token=" + ticket["upload_token"],
headers=[(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii"))],
body=body,
)
header_auth = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"x-obsidian-wiki-upload-token", ticket["upload_token"].encode("ascii")),
(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii")),
],
body=body,
)
self.assertEqual(query_auth[0]["status"], 401)
self.assertEqual(header_auth[0]["status"], 200)
def test_http_upload_endpoint_rejects_oversized_body_with_413(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -313,6 +565,74 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(messages[0]["status"], 413)
def test_http_upload_endpoint_rejects_oversized_file_part_during_parsing(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
max_file_bytes=5,
max_files_per_job=2,
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
body, boundary = multipart_body(
{"start_job": "false"},
[("files", "too-large.txt", "text/plain", b"123456")],
)
with patch.object(
manager,
"create_completed_upload_from_stream",
side_effect=AssertionError("file was fully parsed"),
):
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii")),
],
body=body,
)
self.assertEqual(messages[0]["status"], 413)
def test_http_upload_endpoint_rolls_back_files_when_one_file_is_invalid(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
max_file_bytes=5,
max_files_per_job=2,
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
body, boundary = multipart_body(
{"start_job": "false"},
[
("files", "valid.txt", "text/plain", b"12345"),
("files", "../invalid.txt", "text/plain", b"123"),
],
)
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(b"content-type", f"multipart/form-data; boundary={boundary}".encode("ascii")),
],
body=body,
)
remaining_uploads = list(manager.upload_root.rglob("upload.json"))
self.assertEqual(messages[0]["status"], 400)
self.assertEqual(remaining_uploads, [])
def test_http_upload_endpoint_enforces_allowed_host_and_origin(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -325,7 +645,7 @@ class RemoteMcpServerTests(unittest.TestCase):
FakeApp(),
"secret-token",
ingest_manager=manager,
allowed_hosts=["good.example"],
allowed_hosts=["good.example:*"],
allowed_origins=["https://app.example"],
)
body, boundary = multipart_body(
@@ -363,6 +683,35 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(bad_origin[0]["status"], 403)
self.assertEqual(allowed[0]["status"], 200)
def test_http_upload_endpoint_rejects_present_origin_when_origin_allowlist_is_empty(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
app = mcp_server.create_http_app(
FakeApp(),
"secret-token",
ingest_manager=manager,
allowed_hosts=["good.example"],
allowed_origins=[],
)
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(b"host", b"good.example"),
(b"origin", b"https://unexpected.example"),
],
)
self.assertEqual(messages[0]["status"], 403)
def test_http_upload_endpoint_requires_ingest_and_bearer_token(self):
no_ingest_app = mcp_server.create_http_app(FakeApp(), "secret-token")
missing_token = run_asgi_http_request(no_ingest_app, method="POST", path="/mcp-ingest/uploads")
@@ -598,6 +947,7 @@ class RemoteMcpServerTests(unittest.TestCase):
ingest_runtime_root=root / "ingest-runtime",
query_runtime_root=root / "query-runtime",
public_base_url="https://wiki.example.test/mcp",
allow_source_files=True,
ingest_manager=IngestManager(
vault_path=root / "vault",
project_root=root / "project",
@@ -620,12 +970,14 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(capabilities["server_name"], "wiki-file-assets")
self.assertIn("server_version", capabilities)
self.assertEqual(capabilities["capabilities_schema_version"], 1)
self.assertEqual(capabilities["tools_version"], 1)
self.assertEqual(capabilities["schema_version"], 2)
self.assertEqual(capabilities["capabilities_schema_version"], 2)
self.assertEqual(capabilities["tools_version"], 2)
self.assertEqual(capabilities["enabled_tools"], ["file-assets", "ingest", "query"])
self.assertIn("file_index", capabilities)
self.assertIn("record_count", capabilities["file_index"])
self.assertIn("last_rebuild_at", capabilities["file_index"])
self.assertIn("last_index_rebuild_at", capabilities["file_index"])
self.assertTrue(capabilities["query"]["enabled"])
self.assertEqual(capabilities["query"]["preferred_tool"], "ask_wiki")
self.assertIn("ask_wiki", capabilities["query"]["tools"])
@@ -642,11 +994,204 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertIn("last_ingest_at", capabilities["ingest"])
self.assertTrue(capabilities["ingest"]["known_limitations"])
self.assertIn("search_file_assets", capabilities["file_assets"]["tools"])
self.assertTrue(capabilities["file_assets"]["source_file_download_enabled"])
self.assertTrue(capabilities["safety"]["arbitrary_shell"])
self.assertFalse(capabilities["safety"]["shell_sandbox_enforced"])
self.assertTrue(capabilities["safety"]["opencode_auto_approve"])
self.assertTrue(capabilities["safety"]["trusted_clients_required"])
self.assertNotIn("vault_path", json.dumps(capabilities))
self.assertNotIn("runtime", json.dumps(capabilities).lower())
self.assertNotIn("Bearer ", json.dumps(capabilities))
self.assertNotIn("upload_token", json.dumps(capabilities))
def test_server_resource_read_honors_source_file_gate(self):
class FakeMCP:
resources = {}
def __init__(self, _name, **_kwargs):
pass
def tool(self):
return lambda func: func
def resource(self, template, **_kwargs):
def decorator(func):
self.resources[template] = func
return func
return decorator
def prompt(self, *_args, **_kwargs):
return lambda func: func
with (
tempfile.TemporaryDirectory() as tmp,
patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}),
patch.object(
mcp_server,
"read_file_asset_resource",
return_value={"bytes": b"content"},
) as read_resource,
):
root = Path(tmp)
mcp_server.create_server(vault_path=root, allow_source_files=False)
resource = FakeMCP.resources["asset://source/sha256/{digest}"]
self.assertEqual(resource("a" * 64), b"content")
self.assertFalse(read_resource.call_args.kwargs["allow_source_files"])
mcp_server.create_server(vault_path=root, allow_source_files=True)
resource = FakeMCP.resources["asset://source/sha256/{digest}"]
self.assertEqual(resource("b" * 64), b"content")
self.assertTrue(read_resource.call_args.kwargs["allow_source_files"])
def test_readiness_tools_hide_local_paths(self):
class FakeMCP:
tools = {}
def __init__(self, _name, **_kwargs):
pass
def tool(self):
def decorator(func):
self.tools[func.__name__] = func
return func
return decorator
def resource(self, *_args, **_kwargs):
return lambda func: func
def prompt(self, *_args, **_kwargs):
return lambda func: func
class FakeManager:
def check_opencode_readiness(self, *, run_smoke_test=False):
return {
"ok": False,
"checked_paths": ["/secret/project/.agents/skills/wiki-ingest/SKILL.md"],
"errors": ["missing /secret/project/.agents/skills/wiki-ingest/SKILL.md"],
}
def _public_preflight(self, result):
return {"ok": result["ok"], "errors": ["missing <project> skill"]}
with (
tempfile.TemporaryDirectory() as tmp,
patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}),
):
mcp_server.create_server(
vault_path=Path(tmp) / "vault",
enable_ingest=True,
ingest_manager=FakeManager(),
)
readiness = FakeMCP.tools["check_ingest_opencode_readiness"]()
self.assertNotIn("checked_paths", readiness)
self.assertNotIn("/secret/project", json.dumps(readiness))
def test_create_server_passes_job_timeouts_to_managers(self):
class FakeMCP:
def __init__(self, _name, **_kwargs):
pass
def tool(self):
return lambda func: func
def resource(self, *_args, **_kwargs):
return lambda func: func
def prompt(self, *_args, **_kwargs):
return lambda func: func
with (
tempfile.TemporaryDirectory() as tmp,
patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}),
patch.object(mcp_server, "IngestManager") as ingest_manager,
patch.object(mcp_server, "QueryManager") as query_manager,
):
root = Path(tmp)
mcp_server.create_server(
vault_path=root / "vault",
enable_ingest=True,
enable_query=True,
project_root=root / "project",
ingest_runtime_root=root / "ingest",
query_runtime_root=root / "query",
ingest_job_timeout_seconds=17,
query_job_timeout_seconds=23,
)
self.assertEqual(ingest_manager.call_args.kwargs["job_timeout_seconds"], 17)
self.assertEqual(query_manager.call_args.kwargs["job_timeout_seconds"], 23)
def test_get_mcp_capabilities_recommends_only_enabled_tool_groups(self):
class FakeMCP:
tools = {}
def __init__(self, _name, **_kwargs):
pass
def tool(self):
def decorator(func):
self.tools[func.__name__] = func
return func
return decorator
def resource(self, *_args, **_kwargs):
def decorator(func):
return func
return decorator
def prompt(self, *_args, **_kwargs):
def decorator(func):
return func
return decorator
class FakeQueryManager:
def ask_wiki(self, query, mode="normal", public_only=False, note=None, timeout_seconds=90, poll_interval_seconds=2):
return {}
def start_query_job(self, query, mode="normal", public_only=False, note=None):
return {}
def get_query_job_status(self, job_id):
return {}
def get_query_job_result(self, job_id):
return {}
def cancel_query_job(self, job_id):
return {}
def check_opencode_readiness(self, *, run_smoke_test=False):
return {}
with (
tempfile.TemporaryDirectory() as tmp,
patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}),
):
root = Path(tmp)
mcp_server.create_server(
vault_path=root / "vault",
enable_file_assets=False,
enable_ingest=False,
enable_query=True,
project_root=root / "project",
query_runtime_root=root / "query-runtime",
query_manager=FakeQueryManager(),
)
capabilities = FakeMCP.tools["get_mcp_capabilities"]()
first_steps = "\n".join(capabilities["recommended_first_steps"])
self.assertEqual(capabilities["enabled_tools"], ["query"])
self.assertIn("ask_wiki", first_steps)
self.assertNotIn("search_file_assets", first_steps)
self.assertNotIn("ingest_text_note", first_steps)
self.assertNotIn("create_ingest_upload_ticket", first_steps)
def test_file_index_status_reports_opencode_model_settings(self):
class FakeMCP:
tools = {}
@@ -795,6 +1340,61 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertNotIn("Authorization: Bearer", instructions["http_upload"]["curl_example"])
self.assertEqual(instructions["direct_text_tool"]["name"], "ingest_text_note")
def test_create_ingest_upload_ticket_prefers_explicit_upload_url(self):
class FakeMCP:
tools = {}
def __init__(self, _name, **_kwargs):
pass
def tool(self):
def decorator(func):
self.tools[func.__name__] = func
return func
return decorator
def resource(self, *_args, **_kwargs):
def decorator(func):
return func
return decorator
def prompt(self, *_args, **_kwargs):
def decorator(func):
return func
return decorator
with (
tempfile.TemporaryDirectory() as tmp,
patch.dict(
os.environ,
{"OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL": "https://uploads.example.test/custom/upload"},
clear=False,
),
patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}),
):
root = Path(tmp)
mcp_server.create_server(
vault_path=root / "vault",
enable_ingest=True,
project_root=root / "project",
ingest_runtime_root=root / "runtime",
public_base_url="https://wiki.example.test/proxy/mcp",
)
capabilities = FakeMCP.tools["get_mcp_capabilities"]()
ticket = FakeMCP.tools["create_ingest_upload_ticket"](
expires_seconds=60,
max_files=1,
start_job=False,
)
instructions = FakeMCP.tools["get_ingest_upload_instructions"]()
self.assertEqual(capabilities["ingest"]["http_upload_url"], "https://uploads.example.test/custom/upload")
self.assertEqual(ticket["upload_url"], "https://uploads.example.test/custom/upload")
self.assertEqual(instructions["http_upload"]["url"], "https://uploads.example.test/custom/upload")
def test_ingest_tools_validate_upload_and_preview_metadata(self):
class FakeMCP:
tools = {}
@@ -1034,6 +1634,64 @@ class RemoteMcpServerTests(unittest.TestCase):
)
create_server.assert_not_called()
def test_main_forwards_managed_security_and_timeout_environment(self):
with (
patch.dict(
os.environ,
{
"OBSIDIAN_MCP_TRANSPORT": "streamable-http",
"OBSIDIAN_MCP_ALLOW_SOURCE_FILES": "1",
"OBSIDIAN_MCP_ALLOWED_HOSTS": "wiki.example:*,10.0.0.2:*",
"OBSIDIAN_MCP_ALLOWED_ORIGINS": "https://app.example",
"OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS": "123",
"OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS": "45",
"OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES": "999",
},
clear=True,
),
patch.object(mcp_server, "resolve_vault_path", return_value="/tmp/test-vault"),
patch.object(mcp_server, "run_http_server") as run_http_server,
):
result = mcp_server.main([])
self.assertEqual(result, 0)
kwargs = run_http_server.call_args.kwargs
self.assertTrue(kwargs["allow_source_files"])
self.assertEqual(kwargs["allowed_hosts"], ["wiki.example:*", "10.0.0.2:*"])
self.assertEqual(kwargs["allowed_origins"], ["https://app.example"])
self.assertEqual(kwargs["ingest_job_timeout_seconds"], 123)
self.assertEqual(kwargs["query_job_timeout_seconds"], 45)
self.assertEqual(kwargs["ingest_max_request_bytes"], 999)
def test_main_closes_job_managers_after_stdio_server_stops(self):
manager = type("Manager", (), {"close": unittest.mock.Mock()})()
class FakeServer:
_obsidian_job_managers = (manager,)
def run(self, **_kwargs):
return None
with (
patch.dict(os.environ, {"OBSIDIAN_MCP_TRANSPORT": "stdio"}, clear=True),
patch.object(mcp_server, "resolve_vault_path", return_value="/tmp/test-vault"),
patch.object(mcp_server, "create_server", return_value=FakeServer()),
):
result = mcp_server.main([])
self.assertEqual(result, 0)
manager.close.assert_called_once_with()
def test_close_server_managers_attempts_every_manager(self):
first = type("Manager", (), {"close": lambda self: (_ for _ in ()).throw(OSError("disk full"))})()
second = type("Manager", (), {"close": unittest.mock.Mock()})()
server = type("Server", (), {"_obsidian_job_managers": (first, second)})()
errors = mcp_server.close_server_managers(server)
self.assertEqual(len(errors), 1)
second.close.assert_called_once_with()
if __name__ == "__main__":
unittest.main()
+546 -2
View File
@@ -45,12 +45,14 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("/home/nac/.local/bin/uv run --no-sync python -m obsidian_wiki.mcp_server", unit)
self.assertIn("--vault /data/bjd-vault", unit)
self.assertIn("--transport streamable-http", unit)
self.assertIn("--host 0.0.0.0", unit)
self.assertIn("--port 8765", unit)
self.assertNotIn("--host", unit)
self.assertNotIn("--port", unit)
self.assertNotIn("--path", unit)
self.assertIn("PassEnvironment=http_proxy https_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY", unit)
self.assertIn("MemoryMax=2G", unit)
self.assertIn("Restart=on-failure", unit)
self.assertNotIn("OBSIDIAN_MCP_TOKEN=", unit)
self.assertNotIn("readonly", unit.lower())
def test_write_token_env_file_is_private(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -63,7 +65,12 @@ class McpServiceConfigTests(unittest.TestCase):
port=8765,
path="/mcp",
public_base_url="http://192.168.1.10:8765/mcp",
public_upload_url="https://uploads.example.test/custom/upload",
enable_ingest=True,
allow_source_files=False,
ingest_job_timeout_seconds=111,
query_job_timeout_seconds=22,
ingest_max_request_bytes=333,
)
content = env_path.read_text(encoding="utf-8")
@@ -72,15 +79,112 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("OBSIDIAN_MCP_PORT=8765\n", content)
self.assertIn("OBSIDIAN_MCP_PATH=/mcp\n", content)
self.assertIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=http://192.168.1.10:8765/mcp\n", content)
self.assertIn("OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=https://uploads.example.test/custom/upload\n", content)
self.assertIn("OBSIDIAN_MCP_ALLOW_SOURCE_FILES=0\n", content)
self.assertIn("OBSIDIAN_MCP_ALLOWED_HOSTS=", content)
self.assertIn("192.168.1.10:*", content)
self.assertIn("OBSIDIAN_MCP_ALLOWED_ORIGINS=", content)
self.assertIn("http://192.168.1.10:8765", content)
self.assertIn("https://uploads.example.test", content)
self.assertIn("OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,ingest\n", content)
self.assertIn("OBSIDIAN_MCP_ENABLE_INGEST=1\n", content)
self.assertIn("OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5\n", content)
self.assertIn("OBSIDIAN_MCP_INGEST_OPENCODE_VARIANT=high\n", content)
self.assertIn("OBSIDIAN_MCP_QUERY_OPENCODE_MODEL=openai/gpt-5.5\n", content)
self.assertIn("OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT=minimal\n", content)
self.assertIn("OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS=111\n", content)
self.assertIn("OBSIDIAN_MCP_QUERY_JOB_TIMEOUT_SECONDS=22\n", content)
self.assertIn("OBSIDIAN_MCP_INGEST_MAX_REQUEST_BYTES=333\n", content)
self.assertEqual(stat.S_IMODE(os.stat(env_path).st_mode), 0o600)
self.assertEqual(stat.S_IMODE(os.stat(env_path.parent).st_mode), 0o700)
def test_saved_service_config_returns_dataclass_and_preserves_explicit_source_file_setting(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
env_path = project_root / ".runtime" / "obsidian-wiki-mcp.env"
manage_mcp_service.write_token_env_file(
env_path,
"saved-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
public_upload_url="https://uploads.example.test/saved",
enabled_tools=("file-assets", "query"),
allow_source_files=False,
ingest_job_timeout_seconds=444,
query_job_timeout_seconds=55,
ingest_max_request_bytes=666,
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertIsInstance(config, manage_mcp_service.ServiceConfig)
self.assertEqual(config.token, "saved-token")
self.assertEqual(config.host, "127.0.0.1")
self.assertEqual(config.port, 9988)
self.assertEqual(config.path, "/saved-mcp")
self.assertEqual(config.public_base_url, "https://wiki.example.test/saved-mcp")
self.assertEqual(config.public_upload_url, "https://uploads.example.test/saved")
self.assertEqual(config.enabled_tools, ("file-assets", "query"))
self.assertFalse(config.allow_source_files)
self.assertEqual(config.ingest_job_timeout_seconds, 444)
self.assertEqual(config.query_job_timeout_seconds, 55)
self.assertEqual(config.ingest_max_request_bytes, 666)
self.assertIn("wiki.example.test:*", config.allowed_hosts)
self.assertIn("uploads.example.test:*", config.allowed_hosts)
self.assertIn("https://wiki.example.test", config.allowed_origins)
self.assertIn("https://uploads.example.test", config.allowed_origins)
def test_saved_service_config_migrates_legacy_source_file_setting(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
env_path = project_root / ".runtime" / "obsidian-wiki-mcp.env"
env_path.parent.mkdir(parents=True)
env_path.write_text(
"OBSIDIAN_MCP_TOKEN=saved-token\n"
"OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,query\n",
encoding="utf-8",
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertTrue(config.allow_source_files)
env_path.write_text(
"OBSIDIAN_MCP_TOKEN=saved-token\n"
"OBSIDIAN_MCP_ENABLED_TOOLS=query\n",
encoding="utf-8",
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertFalse(config.allow_source_files)
def test_ensure_service_config_returns_paths_and_default_service_config(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "generate_token", return_value="created-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
):
paths, config = manage_mcp_service.ensure_service_config(project_root)
self.assertIsInstance(paths, manage_mcp_service.ServicePaths)
self.assertIsInstance(config, manage_mcp_service.ServiceConfig)
self.assertEqual(config.host, "0.0.0.0")
self.assertEqual(config.port, 8765)
self.assertEqual(config.path, "/mcp")
self.assertEqual(config.public_base_url, "http://10.0.0.2:8765/mcp")
self.assertFalse(config.allow_source_files)
self.assertIn("10.0.0.2:*", config.allowed_hosts)
self.assertIn("http://10.0.0.2:8765", config.allowed_origins)
self.assertIn(
"OBSIDIAN_MCP_ALLOW_SOURCE_FILES=0\n",
paths.env_path.read_text(encoding="utf-8"),
)
def test_resolve_vault_for_service_uses_explicit_or_configured_vault(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
@@ -115,6 +219,17 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn('"wiki-file-assets"', output)
self.assertNotIn('"obsidian-wiki"', output)
def test_remote_urls_bracket_ipv6_hosts(self):
self.assertEqual(
manage_mcp_service.public_base_url_from_hosts(["2001:db8::1"]),
"http://[2001:db8::1]:8765/mcp",
)
output = manage_mcp_service.render_remote_config(
hosts=["2001:db8::1"],
token="secret-token",
)
self.assertIn('"url": "http://[2001:db8::1]:8765/mcp"', output)
def test_render_remote_config_prints_ingest_upload_endpoint_when_enabled(self):
output = manage_mcp_service.render_remote_config(
hosts=["192.168.1.10", "10.0.0.2"],
@@ -143,6 +258,41 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("https://wiki.example.test/mcp-ingest/uploads", output)
self.assertNotIn("http://192.168.1.10:8765/mcp-ingest/uploads", output)
def test_render_remote_config_uses_normalized_public_url_as_primary_client_endpoint(self):
output = manage_mcp_service.render_remote_config(
hosts=["192.168.1.10", "10.0.0.2"],
token="test-token",
port=8765,
path="/mcp",
public_base_url=" https://wiki.example.test/proxy/mcp/?ignored=yes#fragment ",
)
self.assertIn(
"Use this URL from another machine:\nhttps://wiki.example.test/proxy/mcp\n",
output,
)
self.assertIn('"url": "https://wiki.example.test/proxy/mcp"', output)
self.assertIn(
"Other detected addresses:\n"
"- http://192.168.1.10:8765/mcp\n"
"- http://10.0.0.2:8765/mcp\n",
output,
)
def test_render_remote_config_prefers_explicit_upload_url(self):
output = manage_mcp_service.render_remote_config(
hosts=["192.168.1.10"],
token="test-token",
port=8765,
path="/mcp",
public_base_url="https://wiki.example.test/proxy/mcp",
public_upload_url="https://uploads.example.test/custom/upload",
enabled_tools=("file-assets", "ingest"),
)
self.assertIn("https://uploads.example.test/custom/upload", output)
self.assertNotIn("https://wiki.example.test/mcp-ingest/uploads", output)
def test_new_token_writes_token_without_touching_systemd_when_unit_missing(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
@@ -160,6 +310,7 @@ class McpServiceConfigTests(unittest.TestCase):
env_content = env_path.read_text(encoding="utf-8")
self.assertIn("OBSIDIAN_MCP_TOKEN=new-token\n", env_content)
self.assertIn("OBSIDIAN_MCP_PORT=8765\n", env_content)
self.assertIn("OBSIDIAN_MCP_ALLOW_SOURCE_FILES=0\n", env_content)
self.assertIn("http://10.0.0.2:8765/mcp", output)
run_systemctl.assert_not_called()
@@ -184,6 +335,158 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("http://10.0.0.2:8765/mcp", output)
run_systemctl.assert_called_once_with(["restart", manage_mcp_service.SERVICE_NAME])
def test_new_token_preserves_saved_service_config_when_endpoint_arguments_are_omitted(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
env_path = project_root / ".runtime" / "obsidian-wiki-mcp.env"
manage_mcp_service.write_token_env_file(
env_path,
"old-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
public_upload_url="https://uploads.example.test/saved",
enabled_tools=("file-assets", "query"),
allow_source_files=False,
)
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "generate_token", return_value="new-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "run_systemctl") as run_systemctl,
):
output = manage_mcp_service.new_token(project_root=project_root)
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.token, "new-token")
self.assertEqual(config.host, "127.0.0.1")
self.assertEqual(config.port, 9988)
self.assertEqual(config.path, "/saved-mcp")
self.assertEqual(config.public_base_url, "https://wiki.example.test/saved-mcp")
self.assertEqual(config.public_upload_url, "https://uploads.example.test/saved")
self.assertEqual(config.enabled_tools, ("file-assets", "query"))
self.assertFalse(config.allow_source_files)
self.assertIn('"url": "https://wiki.example.test/saved-mcp"', output)
run_systemctl.assert_not_called()
def test_new_token_only_changes_token_when_saved_public_urls_are_empty(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
env_path = project_root / ".runtime" / "obsidian-wiki-mcp.env"
manage_mcp_service.write_token_env_file(
env_path,
"old-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
allow_source_files=False,
)
original_lines = env_path.read_text(encoding="utf-8").splitlines()
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "generate_token", return_value="new-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "run_systemctl"),
):
output = manage_mcp_service.new_token(project_root=project_root)
updated_lines = env_path.read_text(encoding="utf-8").splitlines()
self.assertEqual(updated_lines[1:], original_lines[1:])
self.assertEqual(updated_lines[0], "OBSIDIAN_MCP_TOKEN=new-token")
self.assertNotIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=", "\n".join(updated_lines))
self.assertIn('"url": "http://10.0.0.2:9988/saved-mcp"', output)
def test_new_token_explicit_endpoint_and_public_urls_override_saved_config(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
manage_mcp_service.write_token_env_file(
project_root / ".runtime" / "obsidian-wiki-mcp.env",
"old-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
public_upload_url="https://uploads.example.test/saved",
)
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "generate_token", return_value="new-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "run_systemctl"),
):
output = manage_mcp_service.new_token(
project_root=project_root,
host="0.0.0.0",
port=8877,
path="/new-mcp",
public_base_url="https://new.example.test/api/mcp/",
public_upload_url="https://uploads-new.example.test/api/upload/",
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.host, "0.0.0.0")
self.assertEqual(config.port, 8877)
self.assertEqual(config.path, "/new-mcp")
self.assertEqual(config.public_base_url, "https://new.example.test/api/mcp")
self.assertEqual(config.public_upload_url, "https://uploads-new.example.test/api/upload")
self.assertIn("new.example.test", config.allowed_hosts)
self.assertIn("uploads-new.example.test", config.allowed_hosts)
self.assertIn("https://new.example.test", config.allowed_origins)
self.assertIn("https://uploads-new.example.test", config.allowed_origins)
self.assertIn('"url": "https://new.example.test/api/mcp"', output)
def test_new_token_recomputes_public_url_when_only_endpoint_changes(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
manage_mcp_service.write_token_env_file(
project_root / ".runtime" / "obsidian-wiki-mcp.env",
"old-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
)
with (
patch.object(manage_mcp_service, "generate_token", return_value="new-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "run_systemctl"),
):
manage_mcp_service.new_token(
project_root=project_root,
port=8877,
path="/new-mcp",
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.public_base_url, "http://127.0.0.1:8877/new-mcp")
def test_new_token_first_install_uses_concrete_bind_host_as_public_url(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
with (
patch.object(manage_mcp_service, "generate_token", return_value="new-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "run_systemctl"),
):
manage_mcp_service.new_token(project_root=project_root, host="127.0.0.1")
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.public_base_url, "http://127.0.0.1:8765/mcp")
def test_start_service_uses_saved_token_and_restarts_service(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
@@ -216,6 +519,8 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("Bearer saved-token", output)
env_content = (project_root / ".runtime" / "obsidian-wiki-mcp.env").read_text(encoding="utf-8")
self.assertIn("OBSIDIAN_MCP_ENABLED_TOOLS=file-assets\n", env_content)
self.assertNotIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=", env_content)
self.assertNotIn("OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=", env_content)
generate_token.assert_not_called()
prepare_runtime.assert_called_once_with(project_root, uv_path=Path("/home/nac/.local/bin/uv"))
run_systemctl.assert_has_calls(
@@ -226,6 +531,156 @@ class McpServiceConfigTests(unittest.TestCase):
]
)
def test_start_service_preserves_saved_endpoint_and_public_urls_when_arguments_are_omitted(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
env_path = project_root / ".runtime" / "obsidian-wiki-mcp.env"
manage_mcp_service.write_token_env_file(
env_path,
"saved-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
public_upload_url="https://uploads.example.test/saved",
allow_source_files=False,
)
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "find_uv_path", return_value=Path("/home/nac/.local/bin/uv")),
patch.object(manage_mcp_service, "resolve_vault_for_service", return_value=Path("/data/bjd-vault")),
patch.object(manage_mcp_service, "prepare_runtime"),
patch.object(manage_mcp_service, "run_systemctl"),
):
output = manage_mcp_service.start_service(project_root=project_root)
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.host, "127.0.0.1")
self.assertEqual(config.port, 9988)
self.assertEqual(config.path, "/saved-mcp")
self.assertEqual(config.public_base_url, "https://wiki.example.test/saved-mcp")
self.assertEqual(config.public_upload_url, "https://uploads.example.test/saved")
self.assertFalse(config.allow_source_files)
unit_content = (
config_home / "systemd" / "user" / manage_mcp_service.SERVICE_NAME
).read_text(encoding="utf-8")
self.assertNotIn("--host", unit_content)
env_content = env_path.read_text(encoding="utf-8")
self.assertIn("OBSIDIAN_MCP_HOST=127.0.0.1", env_content)
self.assertIn("OBSIDIAN_MCP_PORT=9988", env_content)
self.assertIn("OBSIDIAN_MCP_PATH=/saved-mcp", env_content)
self.assertIn('"url": "https://wiki.example.test/saved-mcp"', output)
def test_start_service_explicit_arguments_override_saved_config(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
manage_mcp_service.write_token_env_file(
project_root / ".runtime" / "obsidian-wiki-mcp.env",
"saved-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
public_upload_url="https://uploads.example.test/saved",
allow_source_files=False,
)
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "find_uv_path", return_value=Path("/home/nac/.local/bin/uv")),
patch.object(manage_mcp_service, "resolve_vault_for_service", return_value=Path("/data/bjd-vault")),
patch.object(manage_mcp_service, "prepare_runtime"),
patch.object(manage_mcp_service, "run_systemctl"),
):
output = manage_mcp_service.start_service(
project_root=project_root,
host="0.0.0.0",
port=8877,
path="/new-mcp",
public_base_url="https://new.example.test/api/mcp/",
public_upload_url="https://uploads-new.example.test/api/upload/",
allow_source_files=True,
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.host, "0.0.0.0")
self.assertEqual(config.port, 8877)
self.assertEqual(config.path, "/new-mcp")
self.assertEqual(config.public_base_url, "https://new.example.test/api/mcp")
self.assertEqual(config.public_upload_url, "https://uploads-new.example.test/api/upload")
self.assertTrue(config.allow_source_files)
self.assertIn("new.example.test", config.allowed_hosts)
self.assertIn("uploads-new.example.test", config.allowed_hosts)
self.assertIn("https://new.example.test", config.allowed_origins)
self.assertIn("https://uploads-new.example.test", config.allowed_origins)
unit_content = (
config_home / "systemd" / "user" / manage_mcp_service.SERVICE_NAME
).read_text(encoding="utf-8")
self.assertNotIn("--host", unit_content)
env_content = (project_root / ".runtime" / "obsidian-wiki-mcp.env").read_text(encoding="utf-8")
self.assertIn("OBSIDIAN_MCP_HOST=0.0.0.0", env_content)
self.assertIn("OBSIDIAN_MCP_PORT=8877", env_content)
self.assertIn("OBSIDIAN_MCP_PATH=/new-mcp", env_content)
self.assertIn('"url": "https://new.example.test/api/mcp"', output)
def test_start_service_recomputes_public_url_when_only_endpoint_changes(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
manage_mcp_service.write_token_env_file(
project_root / ".runtime" / "obsidian-wiki-mcp.env",
"saved-token",
host="127.0.0.1",
port=9988,
path="/saved-mcp",
public_base_url="https://wiki.example.test/saved-mcp",
)
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "find_uv_path", return_value=Path("/home/nac/.local/bin/uv")),
patch.object(manage_mcp_service, "resolve_vault_for_service", return_value=Path("/data/bjd-vault")),
patch.object(manage_mcp_service, "prepare_runtime"),
patch.object(manage_mcp_service, "run_systemctl"),
):
manage_mcp_service.start_service(
project_root=project_root,
port=8877,
path="/new-mcp",
)
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.public_base_url, "http://127.0.0.1:8877/new-mcp")
def test_start_service_first_install_uses_concrete_bind_host_as_public_url(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
config_home = Path(tmp) / "config"
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
patch.object(manage_mcp_service, "generate_token", return_value="created-token"),
patch.object(manage_mcp_service, "detect_remote_hosts", return_value=["10.0.0.2"]),
patch.object(manage_mcp_service, "find_uv_path", return_value=Path("/home/nac/.local/bin/uv")),
patch.object(manage_mcp_service, "resolve_vault_for_service", return_value=Path("/data/bjd-vault")),
patch.object(manage_mcp_service, "prepare_runtime"),
patch.object(manage_mcp_service, "run_systemctl"),
):
manage_mcp_service.start_service(project_root=project_root, host="127.0.0.1")
config = manage_mcp_service.saved_service_config(project_root)
self.assertEqual(config.public_base_url, "http://127.0.0.1:8765/mcp")
def test_start_service_generates_token_when_missing_and_prints_config(self):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
@@ -251,9 +706,11 @@ class McpServiceConfigTests(unittest.TestCase):
env_path = project_root / ".runtime" / "obsidian-wiki-mcp.env"
env_content = env_path.read_text(encoding="utf-8")
self.assertIn("OBSIDIAN_MCP_TOKEN=created-token\n", env_content)
self.assertIn("OBSIDIAN_MCP_HOST=0.0.0.0\n", env_content)
self.assertIn("OBSIDIAN_MCP_PORT=9876\n", env_content)
self.assertIn("OBSIDIAN_MCP_PATH=/wiki-mcp\n", env_content)
self.assertIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=http://10.0.0.4:9876/wiki-mcp\n", env_content)
self.assertIn("OBSIDIAN_MCP_ALLOW_SOURCE_FILES=0\n", env_content)
self.assertIn("OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,ingest\n", env_content)
self.assertIn("OBSIDIAN_MCP_ENABLE_INGEST=1\n", env_content)
self.assertIn("OBSIDIAN_MCP_INGEST_OPENCODE_PATH=/home/nac/.opencode/bin/opencode\n", env_content)
@@ -301,6 +758,11 @@ class McpServiceConfigTests(unittest.TestCase):
+ "OBSIDIAN_MCP_PUBLIC_BASE_URL=https://wiki.example.test/mcp\n",
encoding="utf-8",
)
env_path.write_text(
env_path.read_text(encoding="utf-8")
+ "OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=https://uploads.example.test/custom/upload\n",
encoding="utf-8",
)
with (
patch.dict("os.environ", {"XDG_CONFIG_HOME": str(config_home)}),
@@ -321,6 +783,7 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("OBSIDIAN_MCP_QUERY_OPENCODE_MODEL=openai/custom-fast\n", env_content)
self.assertIn("OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT=low\n", env_content)
self.assertIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=https://wiki.example.test/mcp\n", env_content)
self.assertIn("OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=https://uploads.example.test/custom/upload\n", env_content)
def test_start_service_can_enable_query_without_ingest(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -496,6 +959,82 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertTrue(start_service.call_args.kwargs["enable_query"])
print_fn.assert_called_once_with("config\n", end="")
def test_main_start_passes_none_for_omitted_overrides_and_source_file_policy_flags(self):
with (
patch.object(manage_mcp_service, "start_service", return_value="config\n") as start_service,
patch("builtins.print"),
):
status = manage_mcp_service.main(["start", "--deny-source-files"])
self.assertEqual(status, 0)
kwargs = start_service.call_args.kwargs
self.assertIsNone(kwargs["host"])
self.assertIsNone(kwargs["port"])
self.assertIsNone(kwargs["path"])
self.assertIsNone(kwargs["public_base_url"])
self.assertIsNone(kwargs["public_upload_url"])
self.assertFalse(kwargs["allow_source_files"])
with (
patch.object(manage_mcp_service, "start_service", return_value="config\n") as start_service,
patch("builtins.print"),
):
status = manage_mcp_service.main(["start", "--allow-source-files"])
self.assertEqual(status, 0)
self.assertTrue(start_service.call_args.kwargs["allow_source_files"])
def test_main_passes_explicit_endpoint_and_public_url_overrides(self):
endpoint_args = [
"--host",
"127.0.0.1",
"--port",
"9988",
"--path",
"/custom-mcp",
"--public-base-url",
"https://wiki.example.test/custom-mcp",
"--public-upload-url",
"https://uploads.example.test/custom",
]
with (
patch.object(manage_mcp_service, "start_service", return_value="config\n") as start_service,
patch("builtins.print"),
):
start_status = manage_mcp_service.main(["start", *endpoint_args])
self.assertEqual(start_status, 0)
self.assertEqual(start_service.call_args.kwargs["host"], "127.0.0.1")
self.assertEqual(start_service.call_args.kwargs["port"], 9988)
self.assertEqual(start_service.call_args.kwargs["path"], "/custom-mcp")
self.assertEqual(
start_service.call_args.kwargs["public_base_url"],
"https://wiki.example.test/custom-mcp",
)
self.assertEqual(
start_service.call_args.kwargs["public_upload_url"],
"https://uploads.example.test/custom",
)
with (
patch.object(manage_mcp_service, "new_token", return_value="config\n") as new_token,
patch("builtins.print"),
):
token_status = manage_mcp_service.main(["new-token", *endpoint_args])
self.assertEqual(token_status, 0)
self.assertEqual(new_token.call_args.kwargs["host"], "127.0.0.1")
self.assertEqual(new_token.call_args.kwargs["port"], 9988)
self.assertEqual(new_token.call_args.kwargs["path"], "/custom-mcp")
self.assertEqual(
new_token.call_args.kwargs["public_base_url"],
"https://wiki.example.test/custom-mcp",
)
self.assertEqual(
new_token.call_args.kwargs["public_upload_url"],
"https://uploads.example.test/custom",
)
def test_main_new_token_does_not_expect_enable_ingest_argument(self):
with (
patch.object(manage_mcp_service, "new_token", return_value="config\n") as new_token,
@@ -506,6 +1045,11 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertEqual(status, 0)
new_token.assert_called_once()
self.assertNotIn("enable_ingest", new_token.call_args.kwargs)
self.assertIsNone(new_token.call_args.kwargs["host"])
self.assertIsNone(new_token.call_args.kwargs["port"])
self.assertIsNone(new_token.call_args.kwargs["path"])
self.assertIsNone(new_token.call_args.kwargs["public_base_url"])
self.assertIsNone(new_token.call_args.kwargs["public_upload_url"])
print_fn.assert_called_once_with("config\n", end="")
def test_main_cleanup_delegates_to_runtime_cleanup(self):