From 688bebb64c378da871e0445489ba7442f246045f Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 8 Jul 2026 11:18:32 +0000 Subject: [PATCH] feat(mcp): ingest direct text notes --- AGENTS.md | 2 +- README.md | 7 +- obsidian_wiki/mcp_server.py | 58 ++++++++++++++- tests/test_mcp_server_remote.py | 120 +++++++++++++++++++++++++++++++- 4 files changed, 182 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 11485f0..8249d65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,7 +130,7 @@ See `wiki-query` and `wiki-export` skills for how the filter is applied. - **Keep product lookup fields explicit.** Carry forward `bjd_part_numbers`, `competitor_part_numbers`, `series`, `pitch`, `pin_count`, `product_lines`, `customers`, `competitors`, `standards`, `business_categories`, `related_pages`, and `catalog_paths` whenever they are known. - **Merge, do not overwrite.** Automatic identifier extraction may add terms, but it must not discard curated registry metadata. Registry values are the durable source for future index rebuilds and remote MCP lookup. - **Prefer precise identifiers in search ranking.** Exact or prefix matches in structured part-number and series fields should rank above incidental filename substrings such as timestamps or archive hashes. -- **Keep remote ingest gated.** MCP ingest tools are write-capable and must remain disabled unless the service config explicitly enables the `ingest` tool group via `OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,ingest`, `mcp_service.sh tools enable ingest`, or `mcp_service.sh start --enable-ingest`. Uploaded files must stage outside the vault under the MCP runtime root; `wiki-ingest` is responsible for accepting, archiving, and indexing sources inside the vault. Use the HTTP multipart endpoint `/mcp-ingest/uploads` for remote file transfer; if the remote agent cannot read the main MCP bearer token, it must call `create_ingest_upload_ticket` and send the returned single-use `X-Obsidian-Wiki-Upload-Token` header. Do not expose base64 or chunked upload tools in the MCP tool surface. They may launch local OpenCode only through the fixed `/wiki-ingest` prompt; never expose arbitrary prompt or shell execution through MCP. Remote ingest must keep the OpenCode readiness preflight in front of job launch, and `check_ingest_opencode_readiness(run_smoke_test=true)` should stay available for operators to verify that OpenCode can load the project `wiki-ingest` skill before accepting uploads. +- **Keep remote ingest gated.** MCP ingest tools are write-capable and must remain disabled unless the service config explicitly enables the `ingest` tool group via `OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,ingest`, `mcp_service.sh tools enable ingest`, or `mcp_service.sh start --enable-ingest`. Uploaded files and direct text notes must stage outside the vault under the MCP runtime root; `wiki-ingest` is responsible for accepting, archiving, and indexing sources inside the vault. Use `ingest_text_note` only for short remote-agent thoughts/notes; it must wrap the text as a temporary markdown upload and then use the fixed `/wiki-ingest` job path. Use the HTTP multipart endpoint `/mcp-ingest/uploads` for file transfer; if the remote agent cannot read the main MCP bearer token, it must call `create_ingest_upload_ticket` and send the returned single-use `X-Obsidian-Wiki-Upload-Token` header. Do not expose base64 or chunked upload tools in the MCP tool surface. They may launch local OpenCode only through the fixed `/wiki-ingest` prompt; never expose arbitrary prompt or shell execution through MCP. Remote ingest must keep the OpenCode readiness preflight in front of job launch, and `check_ingest_opencode_readiness(run_smoke_test=true)` should stay available for operators to verify that OpenCode can load the project `wiki-ingest` skill before accepting uploads. - **Keep remote query gated.** MCP query tools must remain disabled unless the service config explicitly enables the `query` tool group via `OBSIDIAN_MCP_ENABLED_TOOLS=file-assets,query`, `mcp_service.sh tools enable query`, or `mcp_service.sh start --enable-query`. They may launch local OpenCode only through the fixed `/wiki-query` prompt, with no arbitrary prompt or shell execution. `public_only` defaults to false, meaning remote answers may use the full local vault visibility; enable the tool group only for trusted clients and set `public_only=true` per call for filtered user-facing answers. - **Keep OpenCode model choice server-side.** MCP clients must not pass arbitrary model names through ingest/query tools. Configure local defaults in `.runtime/obsidian-wiki-mcp.env`: ingest uses `OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5` with `OBSIDIAN_MCP_INGEST_OPENCODE_VARIANT=high`; query uses `OBSIDIAN_MCP_QUERY_OPENCODE_MODEL=openai/gpt-5.5` with `OBSIDIAN_MCP_QUERY_OPENCODE_VARIANT=minimal`. diff --git a/README.md b/README.md index 048ad9c..f5085e4 100644 --- a/README.md +++ b/README.md @@ -462,6 +462,7 @@ After `ingest` is enabled once, later `scripts/obsidian-wiki-ops.py mcp start` p This adds write-capable MCP tools to the same `wiki-file-assets` server and bearer token. The server is tool-first: some MCP clients may show empty `resources` or `prompts` lists until a concrete `asset://source/sha256/...` URI is returned by `get_file_asset_metadata`, so use the tool list as the source of truth. - `get_ingest_upload_instructions` — returns the preferred upload methods, HTTP endpoint path, fields, limits, and curl example for remote agents. +- `ingest_text_note` — accepts a short thought/note as text, wraps it as a temporary markdown upload, and optionally starts the normal fixed `/wiki-ingest` job. - `create_ingest_upload_ticket` — returns a short-lived, single-use upload token for the HTTP endpoint so remote agents can upload with curl without reading the main MCP bearer token. - `check_ingest_opencode_readiness` — verifies the local OpenCode binary, required run flags, project ingest skills, and optionally runs a no-write smoke test that confirms `wiki-ingest` can be loaded. - `start_ingest_job` — queues a local `opencode run --auto --format json` job with a fixed `/wiki-ingest` prompt for the completed uploads. @@ -469,7 +470,9 @@ This adds write-capable MCP tools to the same `wiki-file-assets` server and bear The managed service config sets the OpenCode model server-side. By default ingest uses `OBSIDIAN_MCP_INGEST_OPENCODE_MODEL=openai/gpt-5.5` and `OBSIDIAN_MCP_INGEST_OPENCODE_VARIANT=high` for stronger file processing. Change those in `.runtime/obsidian-wiki-mcp.env` on the server if needed; remote MCP clients do not choose the model per call. -For binary files, use the HTTP upload endpoint. If the remote agent cannot read the main MCP bearer token, first call `create_ingest_upload_ticket`, then upload with the returned token: +For short text thoughts, notes, findings, or pasted context, call `ingest_text_note(text, title?, project?, doc_type_hint?, note?, start_job?)`. It uses `doc_type_hint="thought"` by default and returns the created upload plus the queued job when `start_job=true`. + +For binary files and larger source documents, use the HTTP upload endpoint. If the remote agent cannot read the main MCP bearer token, first call `create_ingest_upload_ticket`, then upload with the returned token: ```bash curl -H "X-Obsidian-Wiki-Upload-Token: " \ @@ -484,7 +487,7 @@ The endpoint accepts repeated `files` fields plus optional `project`, `doc_type_ Run `check_ingest_opencode_readiness` before the first remote upload. Use `run_smoke_test: true` when you want OpenCode itself to prove it can load the project `wiki-ingest` skill; otherwise the static preflight is also run automatically before every `start_ingest_job`. -Uploads land under `.runtime/mcp-ingest/uploads/`, outside the Obsidian vault, and are treated only as temporary source staging. Job state and logs live under `.runtime/mcp-ingest/jobs/`. The MCP server refuses to enable ingest if `OBSIDIAN_MCP_INGEST_RUNTIME_ROOT` points inside the vault; let `wiki-ingest` decide how to archive sources into `_sources/archive/`, update registries, and rebuild the file index. The ingest prompt is generated by the MCP service, requires OpenCode to load `wiki-ingest`, and restricts OpenCode to the uploaded file list; clients cannot pass arbitrary shell commands through this interface. Because enabling ingest gives the remote token write access to the vault through local OpenCode, keep it on a trusted network and rotate the token if it has been shared too broadly. +Uploads and text notes land under `.runtime/mcp-ingest/uploads/`, outside the Obsidian vault, and are treated only as temporary source staging. Job state and logs live under `.runtime/mcp-ingest/jobs/`. The MCP server refuses to enable ingest if `OBSIDIAN_MCP_INGEST_RUNTIME_ROOT` points inside the vault; let `wiki-ingest` decide how to archive sources into `_sources/archive/`, update registries, and rebuild the file index. The ingest prompt is generated by the MCP service, requires OpenCode to load `wiki-ingest`, and restricts OpenCode to the uploaded file list; clients cannot pass arbitrary shell commands through this interface. Because enabling ingest gives the remote token write access to the vault through local OpenCode, keep it on a trusted network and rotate the token if it has been shared too broadly. To let a remote MCP client ask synthesized questions against the compiled wiki, enable the `query` tool group: diff --git a/obsidian_wiki/mcp_server.py b/obsidian_wiki/mcp_server.py index 8a60244..d899ff3 100644 --- a/obsidian_wiki/mcp_server.py +++ b/obsidian_wiki/mcp_server.py @@ -60,6 +60,36 @@ DEFAULT_ENABLED_TOOLS = {TOOL_FILE_ASSETS} AVAILABLE_TOOLS = {TOOL_FILE_ASSETS, TOOL_INGEST, TOOL_QUERY} +def text_note_file_name(title: str | None) -> str: + text = str(title or "").strip().lower() + slug = [] + last_dash = False + for char in text: + if char.isascii() and char.isalnum(): + slug.append(char) + last_dash = False + elif char in {"-", "_"}: + slug.append(char) + last_dash = False + elif not last_dash: + slug.append("-") + last_dash = True + name = "".join(slug).strip("-_")[:80].strip("-_") + return (name or "thought") + ".md" + + +def text_note_payload(text: str, title: str | None = None) -> bytes: + body = str(text).replace("\x00", "").strip() + if not body: + raise IngestError("text must not be empty") + heading = " ".join(str(title or "").replace("\x00", "").split()) + if heading: + content = f"# {heading}\n\n{body}\n" + else: + content = body + "\n" + return content.encode("utf-8") + + def normalize_tool_name(name: str) -> str: normalized = name.strip().lower().replace("_", "-") aliases = { @@ -255,7 +285,7 @@ def create_server( def get_ingest_upload_instructions() -> dict[str, Any]: """Return preferred upload methods and limits for remote MCP ingest clients.""" return { - "preferred_method": "POST multipart/form-data files to the HTTP upload endpoint", + "preferred_method": "use ingest_text_note for short text; POST multipart/form-data files to the HTTP upload endpoint for files", "http_upload": { "method": "POST", "path": DEFAULT_INGEST_UPLOAD_PATH, @@ -282,6 +312,12 @@ def create_server( "-F 'files=@drawing.pdf' http://:/mcp-ingest/uploads" ), }, + "direct_text_tool": { + "name": "ingest_text_note", + "best_for": "short remote-agent thoughts, notes, findings, or pasted text", + "start_job_default": True, + "notes": "wraps text as a temporary markdown upload, then uses the same fixed wiki-ingest job path", + }, "limits": { "max_file_bytes": ingest_manager.max_file_bytes, "max_chunk_bytes": ingest_manager.max_chunk_bytes, @@ -294,6 +330,26 @@ def create_server( }, } + @server.tool() + def ingest_text_note( + text: str, + title: str | None = None, + project: str | None = None, + doc_type_hint: str | None = "thought", + note: str | None = None, + start_job: bool = True, + ) -> dict[str, Any]: + """Ingest a short text note by wrapping it as a temporary markdown upload.""" + payload = text_note_payload(text, title=title) + upload = ingest_manager.create_completed_upload( + file_name=text_note_file_name(title), + payload=payload, + project=project, + doc_type_hint=doc_type_hint or "thought", + ) + job = ingest_manager.start_ingest_job([upload["upload_id"]], note=note) if start_job else None + return {"uploads": [upload], "job": job, "text_bytes": len(payload)} + @server.tool() def create_ingest_upload_ticket( expires_seconds: int = DEFAULT_UPLOAD_TICKET_SECONDS, diff --git a/tests/test_mcp_server_remote.py b/tests/test_mcp_server_remote.py index b50247a..0d72b83 100644 --- a/tests/test_mcp_server_remote.py +++ b/tests/test_mcp_server_remote.py @@ -8,7 +8,7 @@ from pathlib import Path from unittest.mock import patch from obsidian_wiki import mcp_server -from obsidian_wiki.mcp_ingest import IngestManager +from obsidian_wiki.mcp_ingest import IngestError, IngestManager class FakeApp: @@ -356,6 +356,7 @@ class RemoteMcpServerTests(unittest.TestCase): self.assertIn("check_ingest_opencode_readiness", FakeMCP.tool_names) self.assertIn("get_ingest_upload_instructions", FakeMCP.tool_names) self.assertIn("create_ingest_upload_ticket", FakeMCP.tool_names) + self.assertIn("ingest_text_note", FakeMCP.tool_names) self.assertNotIn("ingest_file_base64", FakeMCP.tool_names) def test_create_server_registers_query_tools_only_when_enabled(self): @@ -536,12 +537,129 @@ class RemoteMcpServerTests(unittest.TestCase): doc_type_hint="drawing", start_job=False, ) + instructions = FakeMCP.tools["get_ingest_upload_instructions"]() self.assertIn("upload_token", ticket) self.assertEqual(ticket["token_header"], "X-Obsidian-Wiki-Upload-Token") self.assertEqual(ticket["upload_path"], "/mcp-ingest/uploads") self.assertEqual(ticket["max_files"], 1) self.assertFalse(ticket["start_job"]) + self.assertIn("ingest_text_note", instructions["preferred_method"]) + self.assertEqual(instructions["direct_text_tool"]["name"], "ingest_text_note") + + def test_ingest_text_note_tool_creates_completed_markdown_upload(self): + class FakeMCP: + tools = {} + + def __init__(self, _name, **_kwargs): + pass + + def tool(self): + def decorator(func): + self.tools[func.__name__] = func + return func + + return decorator + + def resource(self, *_args, **_kwargs): + def decorator(func): + return func + + return decorator + + def prompt(self, *_args, **_kwargs): + def decorator(func): + return func + + return decorator + + with ( + tempfile.TemporaryDirectory() as tmp, + patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}), + ): + root = Path(tmp) + manager = IngestManager( + vault_path=root / "vault", + project_root=root / "project", + runtime_root=root / "runtime", + max_file_bytes=200, + ) + mcp_server.create_server( + vault_path=root / "vault", + enable_ingest=True, + project_root=root / "project", + ingest_runtime_root=root / "runtime", + ingest_manager=manager, + ) + result = FakeMCP.tools["ingest_text_note"]( + text="Remote agent thought\n\nNeed to remember this.", + title="Customer thought: ACME / risk", + project="bjd", + doc_type_hint="thought", + note="captured from remote agent", + start_job=False, + ) + payload = Path(result["uploads"][0]["path"]).read_text(encoding="utf-8") + + self.assertEqual(result["job"], None) + self.assertEqual(len(result["uploads"]), 1) + upload = result["uploads"][0] + self.assertEqual(upload["state"], "complete") + self.assertEqual(upload["project"], "bjd") + self.assertEqual(upload["doc_type_hint"], "thought") + self.assertTrue(upload["file_name"].endswith(".md")) + self.assertNotIn("/", upload["file_name"]) + self.assertIn("# Customer thought: ACME / risk", payload) + self.assertIn("Remote agent thought", payload) + + def test_ingest_text_note_tool_rejects_empty_and_oversized_text(self): + class FakeMCP: + tools = {} + + def __init__(self, _name, **_kwargs): + pass + + def tool(self): + def decorator(func): + self.tools[func.__name__] = func + return func + + return decorator + + def resource(self, *_args, **_kwargs): + def decorator(func): + return func + + return decorator + + def prompt(self, *_args, **_kwargs): + def decorator(func): + return func + + return decorator + + with ( + tempfile.TemporaryDirectory() as tmp, + patch.dict("sys.modules", {"mcp.server.fastmcp": type("Mod", (), {"FastMCP": FakeMCP})()}), + ): + root = Path(tmp) + manager = IngestManager( + vault_path=root / "vault", + project_root=root / "project", + runtime_root=root / "runtime", + max_file_bytes=20, + ) + mcp_server.create_server( + vault_path=root / "vault", + enable_ingest=True, + project_root=root / "project", + ingest_runtime_root=root / "runtime", + ingest_manager=manager, + ) + with self.assertRaises(IngestError): + FakeMCP.tools["ingest_text_note"](text=" ", start_job=False) + with self.assertRaises(IngestError): + FakeMCP.tools["ingest_text_note"](text="x" * 21, start_job=False) def test_create_server_can_disable_file_asset_tools(self): class FakeMCP: