gitea-admin/obsidian-wiki
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(mcp): stage remote ingest by default

Browse Source 
Default ticket, multipart, and text-note ingest entry points to staging without launching a job. Propagate WIKI_ARCHIVE_SOURCES from project configuration into the managed service environment.\n\nValidated with the full 206-test suite.
This commit is contained in:
Codex
2026-07-16 15:56:54 +00:00
parent 2183404966
commit 6edf02c3eb
7 changed files with 106 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -475,9 +475,9 @@ This adds write-capable MCP tools to the same `wiki-file-assets` server and bear
The managed service config sets the OpenCode model server-side. By default ingest uses `OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5`, `OBSIDIAN_MCP_INGEST_OPENCODE_VARIANT=high`, and an `OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS=1800` deadline. It also writes `OBSIDIAN_MCP_PUBLIC_BASE_URL` from the detected remote address so ticket responses can include a complete upload URL; set it manually when the service is behind a tunnel, reverse proxy, or custom DNS name. If the upload endpoint is exposed through a different public URL than `/mcp`, set `OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL` explicitly and it will override the derived upload URL in capabilities, upload instructions, tickets, and `show-config`. Change these values in `.runtime/obsidian-wiki-mcp.env` on the server if needed; remote MCP clients do not choose the model per call.
For short text thoughts, notes, findings, or pasted context, call `ingest_text_note(text, title?, project?, doc_type_hint?, note?, start_job?)`. It uses `doc_type_hint="thought"` by default and returns a one-item Bundle, the created upload, and the queued job when `start_job=true`.
For short text thoughts, notes, findings, or pasted context, call `ingest_text_note(text, title?, project?, doc_type_hint?, note?, start_job?)`. It uses `doc_type_hint="thought"` and `start_job=false` by default, returning a one-item Bundle and the staged upload; it also returns the queued job when `start_job=true` is explicitly requested.
For binary files and larger source documents, remote agents should use upload tickets by default: call `create_ingest_upload_ticket(start_job=false)`, upload with the returned `upload_url` and `X-Obsidian-Wiki-Upload-Token`, call `preview_upload_metadata(upload_ids)`, then call `start_ingest_job(upload_ids)` when the preview looks correct. The same multipart POST can carry `body_text` plus repeated `files`; the body becomes the Bundle's primary Markdown source and the files become attachments:
For binary files and larger source documents, remote agents should use upload tickets by default: call `create_ingest_upload_ticket()` (also `start_job=false` by default), upload with the returned `upload_url` and `X-Obsidian-Wiki-Upload-Token`, call `preview_upload_metadata(upload_ids)`, then call `start_ingest_job(upload_ids)` when the preview looks correct. The same multipart POST can carry `body_text` plus repeated `files`; the body becomes the Bundle's primary Markdown source and the files become attachments:
```bash
curl -H "X-Obsidian-Wiki-Upload-Token: <upload-token>" \
obsidian_wiki/mcp_ingest.py
+1 -1
View File
@@ -248,7 +248,7 @@ class IngestManager:
bundle_id: str | None = None,
source_type: str | None = None,
label: str | None = None,
start_job: bool = True,
start_job: bool = False,
) -> dict[str, Any]:
if expires_seconds <= 0 or expires_seconds > MAX_UPLOAD_TICKET_SECONDS:
raise IngestError(
obsidian_wiki/mcp_server.py
+6 -6
View File
@@ -510,11 +510,11 @@ def create_server(
"note",
"start_job",
],
"start_job_default": True,
"start_job_default": False,
"curl_example": (
"curl -H 'X-Obsidian-Wiki-Upload-Token: <upload-token>' "
"-F 'files=@drawing.pdf' -F 'project=bjd' "
"-F 'doc_type_hint=drawing' -F 'start_job=true' "
"-F 'doc_type_hint=drawing' -F 'start_job=false' "
f"{public_upload_url or '<upload-url>'}"
),
},
@@ -530,7 +530,7 @@ def create_server(
"direct_text_tool": {
"name": "ingest_text_note",
"best_for": "short remote-agent thoughts, notes, findings, or pasted text",
"start_job_default": True,
"start_job_default": False,
"notes": "wraps text as a temporary markdown upload, then uses the same fixed wiki-ingest job path",
},
"limits": {
@@ -557,7 +557,7 @@ def create_server(
project: str | None = None,
doc_type_hint: str | None = "thought",
note: str | None = None,
start_job: bool = True,
start_job: bool = False,
) -> dict[str, Any]:
"""Ingest a short text note by wrapping it as a temporary markdown upload."""
payload = text_note_payload(text, title=title)
@@ -593,7 +593,7 @@ def create_server(
bundle_id: str | None = None,
source_type: str | None = None,
label: str | None = None,
start_job: bool = True,
start_job: bool = False,
) -> dict[str, Any]:
"""Create a short-lived HTTP upload ticket so remote agents can upload files without the main MCP token."""
ticket = ingest_manager.create_upload_ticket(
@@ -1094,7 +1094,7 @@ class IngestUploadRouter:
bundle_id = ticket_bound_field("bundle_id")
source_type = ticket_bound_field("source_type")
label = ticket_bound_field("label")
start_default = bool((upload_ticket or {}).get("start_job", True))
start_default = bool((upload_ticket or {}).get("start_job", False))
requested_start = parse_bool(first_text_field("start_job"), default=start_default)
start_job = requested_start if upload_ticket is None else start_default and requested_start
has_body = bool(body_text and body_text.strip())
scripts/manage_mcp_service.py
+24 -1
View File
@@ -22,7 +22,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from obsidian_wiki.config import resolve_vault_path
from obsidian_wiki.config import load_config, resolve_vault_path
SERVICE_NAME = "obsidian-wiki-mcp.service"
@@ -37,6 +37,7 @@ PUBLIC_UPLOAD_URL_ENV = "OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL"
ALLOW_SOURCE_FILES_ENV = "OBSIDIAN_MCP_ALLOW_SOURCE_FILES"
ALLOWED_HOSTS_ENV = "OBSIDIAN_MCP_ALLOWED_HOSTS"
ALLOWED_ORIGINS_ENV = "OBSIDIAN_MCP_ALLOWED_ORIGINS"
ARCHIVE_SOURCES_ENV = "WIKI_ARCHIVE_SOURCES"
DEFAULT_TOKEN_BYTES = 32
DEFAULT_UV_CACHE_DIR = "/tmp/uv-cache"
DEFAULT_OPENCODE_PATH = "opencode"
@@ -86,6 +87,11 @@ def parse_bool(value: str | None) -> bool:
return str(value or "").strip().lower() in {"1", "true", "yes", "on", "enabled"}
def project_archive_sources(project_root: Path, *, default: bool = False) -> bool:
value = load_config(project_root).get(ARCHIVE_SOURCES_ENV)
return default if value in (None, "") else parse_bool(value)
def parse_config_list(value: str | None) -> tuple[str, ...]:
return tuple(item.strip() for item in str(value or "").split(",") if item.strip())
@@ -155,6 +161,7 @@ class ServiceConfig:
public_upload_url: str = ""
enabled_tools: tuple[str, ...] = DEFAULT_ENABLED_TOOLS
allow_source_files: bool = False
archive_sources: bool = False
allowed_hosts: tuple[str, ...] = ()
allowed_origins: tuple[str, ...] = ()
ingest_job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS
@@ -295,6 +302,7 @@ def write_token_env_file(
enable_ingest: bool | None = None,
enable_query: bool | None = None,
allow_source_files: bool | None = None,
archive_sources: bool = False,
allowed_hosts: Sequence[str] | None = None,
allowed_origins: Sequence[str] | None = None,
ingest_job_timeout_seconds: int = DEFAULT_INGEST_JOB_TIMEOUT_SECONDS,
@@ -307,6 +315,7 @@ def write_token_env_file(
tools = update_ingest_tool(tools, enable_ingest)
tools = update_query_tool(tools, enable_query)
source_files_allowed = bool(allow_source_files)
archive_sources_enabled = bool(archive_sources)
models = model_settings or OpenCodeModelSettings()
normalized_public_base_url = normalize_public_base_url(public_base_url)
normalized_public_upload_url = normalize_public_upload_url(public_upload_url)
@@ -337,6 +346,7 @@ def write_token_env_file(
if normalized_public_upload_url:
env_file.write(f"{PUBLIC_UPLOAD_URL_ENV}={normalized_public_upload_url}\n")
env_file.write(f"{ALLOW_SOURCE_FILES_ENV}={1 if source_files_allowed else 0}\n")
env_file.write(f"{ARCHIVE_SOURCES_ENV}={1 if archive_sources_enabled else 0}\n")
env_file.write(f"{ALLOWED_HOSTS_ENV}={','.join(effective_allowed_hosts)}\n")
env_file.write(f"{ALLOWED_ORIGINS_ENV}={','.join(effective_allowed_origins)}\n")
env_file.write(f"OBSIDIAN_MCP_INGEST_JOB_TIMEOUT_SECONDS={max(1, int(ingest_job_timeout_seconds))}\n")
@@ -402,6 +412,7 @@ def saved_service_config(project_root: Path) -> ServiceConfig:
allow_source_files = parse_bool(values[ALLOW_SOURCE_FILES_ENV])
else:
allow_source_files = TOOL_FILE_ASSETS in enabled_tools
archive_sources = parse_bool(values.get(ARCHIVE_SOURCES_ENV))
allowed_hosts = parse_config_list(values.get(ALLOWED_HOSTS_ENV))
if not allowed_hosts:
allowed_hosts = derive_allowed_hosts(
@@ -425,6 +436,7 @@ def saved_service_config(project_root: Path) -> ServiceConfig:
public_upload_url=public_upload_url,
enabled_tools=enabled_tools,
allow_source_files=allow_source_files,
archive_sources=archive_sources,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
ingest_job_timeout_seconds=max(
@@ -711,6 +723,7 @@ def write_service_config(env_path: Path, config: ServiceConfig) -> None:
public_upload_url=config.public_upload_url,
enabled_tools=config.enabled_tools,
allow_source_files=config.allow_source_files,
archive_sources=config.archive_sources,
allowed_hosts=config.allowed_hosts,
allowed_origins=config.allowed_origins,
ingest_job_timeout_seconds=config.ingest_job_timeout_seconds,
@@ -778,6 +791,10 @@ def new_token(
saved.allowed_origins if has_saved_config else (),
derive_allowed_origins(effective_public_base_url, effective_public_upload_url),
)
archive_sources = project_archive_sources(
paths.project_root,
default=saved.archive_sources,
)
config = replace(
saved,
token=generate_token(token_bytes),
@@ -786,6 +803,7 @@ def new_token(
path=effective_path,
public_base_url=effective_public_base_url,
public_upload_url=effective_public_upload_url,
archive_sources=archive_sources,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
)
@@ -871,6 +889,10 @@ def start_service(
saved.allowed_origins if has_saved_config else (),
derive_allowed_origins(effective_public_base_url, effective_public_upload_url),
)
archive_sources = project_archive_sources(
paths.project_root,
default=saved.archive_sources,
)
config = replace(
saved,
host=effective_host,
@@ -883,6 +905,7 @@ def start_service(
allow_source_files=(
saved.allow_source_files if allow_source_files is None else allow_source_files
),
archive_sources=archive_sources,
allowed_hosts=allowed_hosts,
allowed_origins=allowed_origins,
)
tests/test_mcp_ingest.py
-1
View File
@@ -268,7 +268,6 @@ class McpIngestTests(unittest.TestCase):
bundle_id="drawing-package-1",
source_type="report_package",
label="Customer drawing package",
start_job=False,
)
self.assertIn("upload_token", ticket)
tests/test_mcp_server_remote.py
+56 -2
View File
@@ -209,6 +209,53 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(data["uploads"][0]["sha256"], "sha256:" + hashlib.sha256(payload).hexdigest())
self.assertNotIn("path", data["uploads"][0])
def test_http_upload_endpoint_stages_without_starting_job_by_default(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
app = mcp_server.create_http_app(
FakeApp(),
"secret-token",
ingest_manager=manager,
)
body, boundary = multipart_body(
{"project": "bjd"},
[("files", "drawing.pdf", "application/pdf", b"drawing")],
)
with patch.object(
manager,
"start_ingest_job",
side_effect=AssertionError("default upload staging must not start a job"),
) as start_job:
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(
b"content-type",
f"multipart/form-data; boundary={boundary}".encode("ascii"),
),
],
body=body,
)
self.assertEqual(messages[0]["status"], 200)
response = b"".join(
message.get("body", b"")
for message in messages
if message["type"] == "http.response.body"
)
data = json.loads(response.decode("utf-8"))
self.assertIsNone(data["job"])
start_job.assert_not_called()
def test_http_upload_endpoint_creates_bundle_from_body_and_attachments(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -411,7 +458,6 @@ class RemoteMcpServerTests(unittest.TestCase):
bundle_id="drawing-package-1",
source_type="report_package",
label="Customer drawing package",
start_job=False,
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
payload = b"ticket uploaded drawing"
@@ -1419,6 +1465,9 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(ticket["label"], "Customer drawing package")
self.assertFalse(ticket["start_job"])
self.assertEqual(instructions["preferred_upload_auth"], "upload_ticket")
self.assertFalse(instructions["http_upload"]["start_job_default"])
self.assertFalse(instructions["direct_text_tool"]["start_job_default"])
self.assertIn("start_job=false", instructions["http_upload"]["curl_example"])
self.assertEqual(instructions["http_upload"]["url"], "https://wiki.example.test/mcp-ingest/uploads")
self.assertIn("body_text", instructions["http_upload"]["optional_fields"])
self.assertIn("bundle_id", instructions["http_upload"]["optional_fields"])
@@ -1588,14 +1637,19 @@ class RemoteMcpServerTests(unittest.TestCase):
ingest_runtime_root=root / "runtime",
ingest_manager=manager,
)
with patch.object(
manager,
"start_ingest_job",
side_effect=AssertionError("default text staging must not start a job"),
) as start_job:
result = FakeMCP.tools["ingest_text_note"](
text="Remote agent thought\n\nNeed to remember this.",
title="Customer thought: ACME / risk",
project="bjd",
doc_type_hint="thought",
note="captured from remote agent",
start_job=False,
)
start_job.assert_not_called()
payload_path = Path(manager._load_upload(result["uploads"][0]["upload_id"])["path"])
payload = payload_path.read_text(encoding="utf-8")
tests/test_mcp_service.py
+10
View File
@@ -68,6 +68,7 @@ class McpServiceConfigTests(unittest.TestCase):
public_upload_url="https://uploads.example.test/custom/upload",
enable_ingest=True,
allow_source_files=False,
archive_sources=True,
ingest_job_timeout_seconds=111,
query_job_timeout_seconds=22,
ingest_max_request_bytes=333,
@@ -81,6 +82,7 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=http://192.168.1.10:8765/mcp\n", content)
self.assertIn("OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=https://uploads.example.test/custom/upload\n", content)
self.assertIn("OBSIDIAN_MCP_ALLOW_SOURCE_FILES=0\n", content)
self.assertIn("WIKI_ARCHIVE_SOURCES=1\n", content)
self.assertIn("OBSIDIAN_MCP_ALLOWED_HOSTS=", content)
self.assertIn("192.168.1.10:*", content)
self.assertIn("OBSIDIAN_MCP_ALLOWED_ORIGINS=", content)
@@ -112,6 +114,7 @@ class McpServiceConfigTests(unittest.TestCase):
public_upload_url="https://uploads.example.test/saved",
enabled_tools=("file-assets", "query"),
allow_source_files=False,
archive_sources=True,
ingest_job_timeout_seconds=444,
query_job_timeout_seconds=55,
ingest_max_request_bytes=666,
@@ -128,6 +131,7 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertEqual(config.public_upload_url, "https://uploads.example.test/saved")
self.assertEqual(config.enabled_tools, ("file-assets", "query"))
self.assertFalse(config.allow_source_files)
self.assertTrue(config.archive_sources)
self.assertEqual(config.ingest_job_timeout_seconds, 444)
self.assertEqual(config.query_job_timeout_seconds, 55)
self.assertEqual(config.ingest_max_request_bytes, 666)
@@ -491,6 +495,11 @@ class McpServiceConfigTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp:
project_root = Path(tmp) / "project"
project_root.mkdir()
(project_root / ".env").write_text(
"OBSIDIAN_VAULT_PATH=/data/bjd-vault\n"
"WIKI_ARCHIVE_SOURCES=true\n",
encoding="utf-8",
)
config_home = Path(tmp) / "config"
manage_mcp_service.write_token_env_file(
project_root / ".runtime" / "obsidian-wiki-mcp.env",
@@ -519,6 +528,7 @@ class McpServiceConfigTests(unittest.TestCase):
self.assertIn("Bearer saved-token", output)
env_content = (project_root / ".runtime" / "obsidian-wiki-mcp.env").read_text(encoding="utf-8")
self.assertIn("OBSIDIAN_MCP_ENABLED_TOOLS=file-assets\n", env_content)
self.assertIn("WIKI_ARCHIVE_SOURCES=1\n", env_content)
self.assertNotIn("OBSIDIAN_MCP_PUBLIC_BASE_URL=", env_content)
self.assertNotIn("OBSIDIAN_MCP_INGEST_PUBLIC_UPLOAD_URL=", env_content)
generate_token.assert_not_called()