gitea-admin/obsidian-wiki
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mcp): support bundled text and attachment ingest

Browse Source 
Allow one multipart request to stage body text and repeated files as a Bundle with bundle-level metadata. Preserve Bundle roles through preview, job launch, prompts, cleanup, and text-note ingestion.\n\nValidated with the full unittest suite (188 tests).
This commit is contained in:
Codex
2026-07-16 14:07:58 +00:00
parent 4c5454da90
commit a2b4bc393b
5 changed files with 443 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_mcp_ingest.py
+61
View File
@@ -264,6 +264,9 @@ class McpIngestTests(unittest.TestCase):
project="bjd",
doc_type_hint="drawing",
note="customer uploaded drawing",
bundle_id="drawing-package-1",
source_type="report_package",
label="Customer drawing package",
start_job=False,
)
@@ -272,12 +275,16 @@ class McpIngestTests(unittest.TestCase):
self.assertEqual(ticket["project"], "bjd")
self.assertEqual(ticket["doc_type_hint"], "drawing")
self.assertEqual(ticket["note"], "customer uploaded drawing")
self.assertEqual(ticket["bundle_id"], "drawing-package-1")
self.assertEqual(ticket["source_type"], "report_package")
self.assertEqual(ticket["label"], "Customer drawing package")
self.assertFalse(ticket["start_job"])
self.assertEqual(ticket["used_at"], None)
used = self.manager.consume_upload_ticket(ticket["upload_token"])
self.assertEqual(used["project"], "bjd")
self.assertEqual(used["bundle_id"], "drawing-package-1")
self.assertIsNotNone(used["used_at"])
with self.assertRaises(IngestError):
self.manager.consume_upload_ticket(ticket["upload_token"])
@@ -378,6 +385,11 @@ class McpIngestTests(unittest.TestCase):
def test_start_ingest_job_constructs_restricted_opencode_command(self):
payload = b"source data"
upload = self._complete_upload("source.pdf", payload)
self.manager.create_ingest_bundle(
[upload["upload_id"]],
source_type="report_package",
label="Quality report",
)
self.manager.opencode_model = "openai/gpt-5.5"
self.manager.opencode_variant = "high"
popen = FakePopen()
@@ -398,6 +410,9 @@ class McpIngestTests(unittest.TestCase):
self.assertIn(f"MCP ingest {job['job_id']}", args)
prompt = args[-1]
self.assertIn("/wiki-ingest", prompt)
self.assertIn('"source_type": "report_package"', prompt)
self.assertIn('"label": "Quality report"', prompt)
self.assertIn('"bundle_role": "attachment"', prompt)
self.assertIn(str(self._upload_payload_path(upload["upload_id"])), prompt)
self.assertIn("temporary runtime staging files", prompt)
self.assertIn("Do not ingest any file outside this list.", prompt)
@@ -503,6 +518,48 @@ class McpIngestTests(unittest.TestCase):
self.assertEqual(repeated["state"], "running")
popen_mock.assert_called_once()
def test_create_bundle_persists_body_and_attachment_relationship(self):
body = self._complete_upload("thread.md", b"# Thread\n\nBody")
attachment = self._complete_upload("report.pdf", b"report")
bundle = self.manager.create_ingest_bundle(
[body["upload_id"], attachment["upload_id"]],
body_upload_id=body["upload_id"],
bundle_id="mail-thread-123",
source_type="email_thread",
label="Customer quality thread",
)
self.assertEqual(bundle["bundle_id"], "mail-thread-123")
self.assertEqual(bundle["source_type"], "email_thread")
self.assertEqual(bundle["label"], "Customer quality thread")
self.assertEqual(bundle["body_upload_id"], body["upload_id"])
self.assertEqual(bundle["attachment_upload_ids"], [attachment["upload_id"]])
self.assertEqual(
self.manager._load_upload(body["upload_id"])["bundle_role"],
"primary",
)
self.assertEqual(
self.manager._load_upload(attachment["upload_id"])["bundle_role"],
"attachment",
)
preview = self.manager.preview_upload_metadata(
[body["upload_id"], attachment["upload_id"]]
)
self.assertEqual(preview["bundle"], bundle)
def test_start_ingest_job_rejects_partial_bundle(self):
body = self._complete_upload("thread.md", b"body")
attachment = self._complete_upload("report.pdf", b"report")
self.manager.create_ingest_bundle(
[body["upload_id"], attachment["upload_id"]],
body_upload_id=body["upload_id"],
)
with self.assertRaisesRegex(IngestError, "complete bundle"):
self.manager.start_ingest_job([body["upload_id"]])
def test_start_ingest_job_rejects_duplicate_upload_ids(self):
upload = self._complete_upload("source.pdf", b"source")
@@ -529,16 +586,20 @@ class McpIngestTests(unittest.TestCase):
upload_path = self._upload_payload_path(upload["upload_id"])
upload_dir = upload_path.parent
job_dir = self.manager.jobs_root / job["job_id"]
bundle_dir = self.manager.bundles_root / job["bundle"]["bundle_id"]
self._touch_tree(upload_dir, old_epoch)
self._touch_tree(job_dir, old_epoch)
self._touch_tree(bundle_dir, old_epoch)
os.utime(ticket_path, (old_epoch, old_epoch))
result = self.manager.cleanup_runtime(max_age_days=30)
self.assertGreaterEqual(result["expired_tickets_deleted"], 1)
self.assertEqual(result["old_uploads_deleted"], 1)
self.assertEqual(result["old_bundles_deleted"], 1)
self.assertEqual(result["old_jobs_deleted"], 1)
self.assertFalse(upload_dir.exists())
self.assertFalse(bundle_dir.exists())
self.assertFalse(job_dir.exists())
self.assertFalse(ticket_path.exists())
tests/test_mcp_server_remote.py
+86 -3
View File
@@ -209,6 +209,68 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(data["uploads"][0]["sha256"], "sha256:" + hashlib.sha256(payload).hexdigest())
self.assertNotIn("path", data["uploads"][0])
def test_http_upload_endpoint_creates_bundle_from_body_and_attachments(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
manager = IngestManager(
vault_path=root / "vault",
project_root=root / "project",
runtime_root=root / "runtime",
)
app = mcp_server.create_http_app(
FakeApp(),
"secret-token",
ingest_manager=manager,
)
body, boundary = multipart_body(
{
"body_text": "First message\n\nSecond message",
"body_title": "Customer thread",
"bundle_id": "thread-123",
"source_type": "email_thread",
"label": "Customer quality thread",
"start_job": "false",
},
[
("files", "report.pdf", "application/pdf", b"report"),
("files", "photo.jpg", "image/jpeg", b"photo"),
],
)
messages = run_asgi_http_request(
app,
method="POST",
path="/mcp-ingest/uploads",
headers=[
(b"authorization", b"Bearer secret-token"),
(
b"content-type",
f"multipart/form-data; boundary={boundary}".encode("ascii"),
),
],
body=body,
)
self.assertEqual(messages[0]["status"], 200)
response = b"".join(
message.get("body", b"")
for message in messages
if message["type"] == "http.response.body"
)
data = json.loads(response.decode("utf-8"))
self.assertIsNone(data["job"])
self.assertEqual(data["bundle"]["bundle_id"], "thread-123")
self.assertEqual(data["bundle"]["source_type"], "email_thread")
self.assertEqual(data["bundle"]["label"], "Customer quality thread")
self.assertEqual(len(data["uploads"]), 3)
self.assertEqual(data["uploads"][0]["bundle_role"], "primary")
self.assertEqual(data["uploads"][1]["bundle_role"], "attachment")
self.assertEqual(data["uploads"][2]["bundle_role"], "attachment")
body_meta = manager._load_upload(data["bundle"]["body_upload_id"])
body_payload = Path(body_meta["path"]).read_text(encoding="utf-8")
self.assertIn("# Customer thread", body_payload)
self.assertIn("First message", body_payload)
def test_http_upload_endpoint_streams_multipart_without_buffering_whole_body(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -346,6 +408,9 @@ class RemoteMcpServerTests(unittest.TestCase):
max_files=1,
project="bjd",
doc_type_hint="drawing",
bundle_id="drawing-package-1",
source_type="report_package",
label="Customer drawing package",
start_job=False,
)
app = mcp_server.create_http_app(FakeApp(), "secret-token", ingest_manager=manager)
@@ -970,9 +1035,9 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(capabilities["server_name"], "wiki-file-assets")
self.assertIn("server_version", capabilities)
self.assertEqual(capabilities["schema_version"], 2)
self.assertEqual(capabilities["capabilities_schema_version"], 2)
self.assertEqual(capabilities["tools_version"], 2)
self.assertEqual(capabilities["schema_version"], 3)
self.assertEqual(capabilities["capabilities_schema_version"], 3)
self.assertEqual(capabilities["tools_version"], 3)
self.assertEqual(capabilities["enabled_tools"], ["file-assets", "ingest", "query"])
self.assertIn("file_index", capabilities)
self.assertIn("record_count", capabilities["file_index"])
@@ -988,6 +1053,10 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(capabilities["ingest"]["http_upload_path"], "/mcp-ingest/uploads")
self.assertEqual(capabilities["ingest"]["http_upload_url"], "https://wiki.example.test/mcp-ingest/uploads")
self.assertEqual(capabilities["ingest"]["preferred_text_tool"], "ingest_text_note")
self.assertTrue(capabilities["ingest"]["supports_bundle_upload"])
self.assertTrue(capabilities["ingest"]["supports_body_text"])
self.assertIn("body_text", capabilities["ingest"]["multipart_optional_fields"])
self.assertIn("source_type", capabilities["ingest"]["multipart_optional_fields"])
self.assertIn("validate_upload", capabilities["ingest"]["tools"])
self.assertIn("preview_upload_metadata", capabilities["ingest"]["tools"])
self.assertIn("queue", capabilities["ingest"])
@@ -1321,6 +1390,9 @@ class RemoteMcpServerTests(unittest.TestCase):
max_files=1,
project="bjd",
doc_type_hint="drawing",
bundle_id="drawing-package-1",
source_type="report_package",
label="Customer drawing package",
start_job=False,
)
instructions = FakeMCP.tools["get_ingest_upload_instructions"]()
@@ -1332,9 +1404,14 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertIn("X-Obsidian-Wiki-Upload-Token", ticket["curl_example"])
self.assertNotIn("Authorization: Bearer", ticket["curl_example"])
self.assertEqual(ticket["max_files"], 1)
self.assertEqual(ticket["bundle_id"], "drawing-package-1")
self.assertEqual(ticket["source_type"], "report_package")
self.assertEqual(ticket["label"], "Customer drawing package")
self.assertFalse(ticket["start_job"])
self.assertEqual(instructions["preferred_upload_auth"], "upload_ticket")
self.assertEqual(instructions["http_upload"]["url"], "https://wiki.example.test/mcp-ingest/uploads")
self.assertIn("body_text", instructions["http_upload"]["optional_fields"])
self.assertIn("bundle_id", instructions["http_upload"]["optional_fields"])
self.assertIn("create_ingest_upload_ticket", instructions["preferred_method"])
self.assertEqual(instructions["http_upload"]["bearer_auth"], "admin_or_local_only")
self.assertNotIn("Authorization: Bearer", instructions["http_upload"]["curl_example"])
@@ -1514,10 +1591,16 @@ class RemoteMcpServerTests(unittest.TestCase):
self.assertEqual(result["job"], None)
self.assertEqual(len(result["uploads"]), 1)
self.assertEqual(result["bundle"]["source_type"], "text_note")
self.assertEqual(
result["bundle"]["body_upload_id"],
result["uploads"][0]["upload_id"],
)
upload = result["uploads"][0]
self.assertEqual(upload["state"], "complete")
self.assertEqual(upload["project"], "bjd")
self.assertEqual(upload["doc_type_hint"], "thought")
self.assertEqual(upload["bundle_role"], "primary")
self.assertNotIn("path", upload)
self.assertTrue(upload["file_name"].endswith(".md"))
self.assertNotIn("/", upload["file_name"])