From e943bb329eb119b3f148fc0df5da79db5119580c Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 7 Nov 2026 21:49:41 +0000 Subject: [PATCH] feat(wiki): port upstream knowledge quality improvements Selectively port source coverage and redirect lint, ignore-aware provenance, staging and trust checks from upstream 055d720. Preserve local archive metadata and adapt source-date and defuddle skill guidance. Keep the existing skills directory, file index and MCP interfaces. Record per-module upstream baselines and exclusions without migrating vault data or changing services. Validation: 625 tests passed; git diff --check passed. --- .skills/ingest-url/SKILL.md | 2 +- .skills/wiki-ingest/SKILL.md | 8 +++ .skills/wiki-lint/SKILL.md | 6 ++ .skills/wiki-research/SKILL.md | 2 +- README.md | 2 +- docs/upstream-sync.json | 29 +++++++- docs/upstream-sync.md | 20 +++++- obsidian_wiki/lint.py | 106 ++++++++++++++++++++++++---- obsidian_wiki/provenance.py | 4 +- obsidian_wiki/staging.py | 21 ++++-- obsidian_wiki/trust.py | 19 ++++- obsidian_wiki/vault.py | 11 +++ tests/test_lint.py | 124 +++++++++++++++++++++++++++++++++ tests/test_provenance.py | 11 +++ tests/test_staging.py | 25 +++++++ tests/test_trust.py | 48 +++++++++++++ tests/test_upstream_compat.py | 28 ++++++++ 17 files changed, 437 insertions(+), 29 deletions(-) diff --git a/.skills/ingest-url/SKILL.md b/.skills/ingest-url/SKILL.md index ddfa4db..d668d2c 100644 --- a/.skills/ingest-url/SKILL.md +++ b/.skills/ingest-url/SKILL.md @@ -59,7 +59,7 @@ Before fetching, check whether the `defuddle` CLI is available: which defuddle ``` -- **If available:** Use `defuddle ` (via Bash) to retrieve a clean, stripped-down markdown version of the page. This removes ads, navbars, cookie banners, and related-content sidebars — reducing token usage by ~40-60% on typical articles. Use the `defuddle` output as your content source for Step 4 instead of the raw WebFetch result. +- **If available:** Use `defuddle parse --markdown` (via Bash) to retrieve a clean, stripped-down markdown version of the page. This removes ads, navbars, cookie banners, and related-content sidebars — reducing token usage by ~40-60% on typical articles. Use the `defuddle` output as your content source for Step 4 instead of the raw WebFetch result. - **If not available:** Fall back to `WebFetch` as normal. No action needed. ## Step 1: Fetch the URL diff --git a/.skills/wiki-ingest/SKILL.md b/.skills/wiki-ingest/SKILL.md index 2bde7cd..fe841d2 100644 --- a/.skills/wiki-ingest/SKILL.md +++ b/.skills/wiki-ingest/SKILL.md @@ -408,6 +408,14 @@ For each page in your plan: - Add the new source to the `sources` list - Resolve any contradictions between old and new information (note them if unresolvable) +**Source dates (applies to live and staged pages).** Keep `created`/`updated` as ingestion timestamps from `obsidian-wiki memory now`. For a new page, set optional `valid_from: YYYY-MM-DD` from the source’s own date, taking the first available value in this order: + +1. Source frontmatter `created:` or `date:` (including a raw note’s capture date). +2. A leading ISO date in the original filename, such as `2026-03-14-topic.md`. For archived files, consult the original filename in registry/manifest metadata; the archive directory date and hash prefix are not event dates. +3. A dated origin such as `conversation:2026-03-14` or ` session (2026-03-14)`. + +Use only a valid date no later than today (`obsidian-wiki memory now --date`). If absent, invalid or future-dated, omit `valid_from`; never guess from file mtime or the archive month. For an existing page, only lower an existing `valid_from` when the new source has an earlier valid date; never raise it or add it to a page that lacks it. Do not set `valid_until` during ingest. These fields preserve temporal provenance; this fork does not include upstream’s GraphRAG `query --as-of` command. Keep the local archive citations and curated registry fields unchanged. + **Populate `relationships:` when context is clear** — if Step 2 identified typed relationships between this page and another, add a `relationships:` block to the frontmatter (defined in `llm-wiki/SKILL.md`, Typed Relationships section). Only add entries where the source text makes the direction and type unambiguous. When in doubt, use `related_to` or omit the block. Example: ```yaml diff --git a/.skills/wiki-lint/SKILL.md b/.skills/wiki-lint/SKILL.md index 6f9985d..8055564 100644 --- a/.skills/wiki-lint/SKILL.md +++ b/.skills/wiki-lint/SKILL.md @@ -26,6 +26,12 @@ You are performing a health check on an Obsidian wiki. Your goal is to find and First run `obsidian-wiki lint "$OBSIDIAN_VAULT_PATH" --json` using this repository's CLI. Use its findings for structural checks below, then inspect only implicated pages and perform semantic contradiction review. It resolves project-relative and duplicate-name links, ignores code examples, and excludes source archives from knowledge pages. Trust-ledger gaps are warnings in legacy mode; do not invent reviews or change scores to silence them. Follow `llm-wiki/references/CORE-MAINTENANCE.md` for any authorized repairs and tracking-file updates. If the CLI is unavailable, use the manual checks below with the same source-directory exclusions. +Interpret the additional CLI findings before manual repairs: + +- `uncovered_sources`: a manifest source has an explicit empty page list, or all listed pages are missing. Check renames, dedup merges and unfinished ingest; repair the actual association or re-ingest instead of inventing pages or dropping source records. Legacy count-only entries are not evidence of missing pages. This check reports complete loss of coverage, not every missing page in a partially surviving list. +- `redirect_issues`: a `redirects_to` stub has a missing target, points to itself, or points to another stub. Resolve it to a real canonical page. Redirect stubs are excluded from duplicate-title/stem findings; actual duplicate content pages still appear. +- `.okignore` also applies to archive snapshot checks and staged/live listing. An ignored source is not automatically a deleted source; preserve the local source registry and archive. + Run these checks in order. Report findings as you go. ### 1. Orphaned Pages diff --git a/.skills/wiki-research/SKILL.md b/.skills/wiki-research/SKILL.md index 95fa248..312a2a8 100644 --- a/.skills/wiki-research/SKILL.md +++ b/.skills/wiki-research/SKILL.md @@ -38,7 +38,7 @@ If the file doesn't exist, proceed with defaults. 1. Decompose the topic into **3-5 distinct angles** (e.g., for "vector databases": what they are, when to use them, leading implementations, trade-offs, production gotchas) 2. For each angle, run **2-3 `WebSearch` queries** using varied phrasing -3. For the top 2-3 results per angle, use `WebFetch` (or `defuddle ` if available — cleaner extraction) to get content +3. For the top 2-3 results per angle, use `WebFetch` (or `defuddle parse --markdown` if available — cleaner extraction) to get content 4. From each fetched page, extract: - **Key claims** — what the source explicitly states - **Concepts** — ideas, terms, frameworks introduced diff --git a/README.md b/README.md index 0f68203..5a5433e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ We took that and built a framework around it. Markdown skill files guide AI codi ## Quick Start -This fork selectively includes the core maintenance updates from upstream **v2026.10.4**: hash-based source tracking, locked memory writes, path-aware lint/graph analysis, and revision-checked staging. See [the sync scope and migration notes](docs/upstream-sync.md). Existing source-file lookup and MCP deployment conventions remain local; upgrading the code does not migrate vault contents or enable services. +This fork selectively includes the core maintenance updates from upstream **v2026.10.4**: hash-based source tracking, locked memory writes, path-aware lint/graph analysis, and revision-checked staging. A further quality-only update adds source-coverage and redirect checks, consistent ignore rules, and source-date guidance. See [the sync scope and migration notes](docs/upstream-sync.md). Existing source-file lookup and MCP deployment conventions remain local; upgrading the code does not migrate vault contents or enable services. ### Install via pip (recommended) diff --git a/docs/upstream-sync.json b/docs/upstream-sync.json index 4ad66b2..80376cf 100644 --- a/docs/upstream-sync.json +++ b/docs/upstream-sync.json @@ -109,5 +109,32 @@ "Resolve _sources/archive and _sources/catalog citations without changing file extensions", "Preserve pages_produced when cache-update omits --pages", "Keep local CLI, MCP dependency range, service gating and curated source-registry metadata" - ] + ], + "quality_sync": { + "date": "2026-11-07", + "commit": "055d720950f7137954e36633b1a79c641ea5e735", + "base_commit": "4f457e154653756b04802d5aeb25c20f42b4770c", + "scope": "Wiki quality fixes only; no skill-directory migration or server changes", + "source_sha256": { + "obsidian_wiki/vault.py": "12e76c819bbed6514c6370df47c001b97cd63f6b13f90875d23c819ec8b185ab", + "obsidian_wiki/lint.py": "e6c6592a000487f4f8a67fa45d72a3ff528af90b70c1d61431bcc7c43b7db5df", + "obsidian_wiki/provenance.py": "c1cb69296aed2b7a5669191340c6d2e66819f595b1bc202eaeb88ca6c859725a", + "obsidian_wiki/trust.py": "3acc4899a2dfe28dfbebf518031fb68cd1e3f873c3244f17967a3713a08ea285", + "obsidian_wiki/staging.py": "ab1b1d7dbe8a024d2bdf9e17541b444e7bc6d5e06806334418d22ae4f587a8ee" + }, + "skill_changes": [ + "source event dates with local archive metadata adaptation", + "defuddle parse --markdown", + "lint source coverage and redirect guidance" + ], + "test_exclusions_outside_scope": [ + "test_recording_one_page_prunes_ignored_entries", + "test_recording_all_reports_the_ignored_entries_it_dropped" + ], + "notes": [ + "Excluded tests require upstream trust-record/trust-check CLI commands not ported locally; partial ledger pruning covered through the local Python API.", + "Original commit/source_sha256 remain the initial baseline; quality_sync.source_sha256 supersedes that baseline for its five listed modules.", + "Preserve local archive/catalog citations, registry identifiers, MCP implementation and .skills layout." + ] + } } diff --git a/docs/upstream-sync.md b/docs/upstream-sync.md index 975b708..bb7b803 100644 --- a/docs/upstream-sync.md +++ b/docs/upstream-sync.md @@ -1,5 +1,23 @@ # 上游关键更新同步 +## 2026-11-07：Wiki 质量增量同步 + +基于上次 v2026.10.4 移植，选择性同步到上游提交 [`055d720`](https://github.com/Ar9av/obsidian-wiki/commit/055d720950f7137954e36633b1a79c641ea5e735) 中的质量相关改动，不表示追平整个主分支。 + +- 来源覆盖检查：发现 manifest 中明确没有产出页面、或产出页面已全部丢失的来源；兼容列表/字典清单及旧版仅记录数量的条目。 +- 重定向检查：合并后留下的跳转页不再误报重复标题/文件名；报告目标丢失、自引用和重定向链。 +- 忽略规则：归档快照核对、URL 来源索引、暂存列表及信任记录检查遵守 `.okignore`；保留本地 `_sources/archive` / `_sources/catalog` 引用适配。 +- 入库时间信息：新增页面可从来源自身日期填写 `valid_from`；本地归档资料使用原文件名/元数据，避免误用归档月份、mtime 或哈希前缀。已有页面只允许提前已有的有效日期，不批量回填历史知识。 +- 网页正文提取：修正本地 ingest-url 与 wiki-research 中的 `defuddle parse --markdown` 调用。 + +按本次范围保留 `.skills/`、安装配置、文件索引、MCP 和现有服务。上游自身仍有 HTTP/MCP 服务，本地 MCP 在订单功能剥离后仍支持文件检索、Wiki 入库与问答，因此“上游没有 MCP 时删除”的条件不成立。本次不引入上游服务、GraphRAG、会话 hooks 或新增技能，也不执行实际知识库迁移。 + +新增回归覆盖来源丢失、重定向、忽略规则、暂存目录软链接、本地归档字段保留和 CLI 严格模式。完整回归结果：**625 项测试通过**（`PYTHONPATH="$PWD" .venv/bin/python -m pytest tests -q`）；`git diff --check` 通过。测试使用临时知识库，本次没有写入实际 vault，也未启动、停止或重新配置服务。 + +后续比较采用 `upstream-sync.json` 的 `quality_sync` 五个模块哈希；未变化模块沿用最初基线。技能为按语义移植，不可整体覆盖。 + +## 2026-10-31：初始核心维护同步 + 本次选择性同步 [Ar9av/obsidian-wiki v2026.10.4](https://github.com/Ar9av/obsidian-wiki/releases/tag/v2026.10.4)，固定提交 `4f457e154653756b04802d5aeb25c20f42b4770c`。这是核心维护能力的移植，不表示本分支完整等同于该版本。来源哈希、移植的 CLI 函数和未纳入的测试范围记录在 `upstream-sync.json`。 | 纳入能力 | 实际作用 | @@ -46,4 +64,4 @@ PYTHONPATH="$PWD" .venv/bin/python -m pytest tests -q 本次验证结果：**609 项测试通过**。实际知识库只执行了 `memory status` 只读检查，index/hot/log、manifest、registry、SQLite 的前后 SHA-256 一致。Wiki MCP 保持 inactive，独立订单 MCP 保持 active。 -下次以 `upstream-sync.json` 的提交和源文件哈希作比较基线，逐模块检查新提交，再重放列出的本地适配。不要直接覆盖 `.skills`、运行上游 setup、切换服务依赖或批量改写知识库。上游未纳入模块的测试明确列入排除清单，不能把它们的缺席解释为整版上游已通过验证。 +初始同步以 `upstream-sync.json` 的提交和源文件哈希作比较基线，逐模块检查新提交，再重放列出的本地适配。不要直接覆盖 `.skills`、运行上游 setup、切换服务依赖或批量改写知识库。上游未纳入模块的测试明确列入排除清单，不能把它们的缺席解释为整版上游已通过验证。 diff --git a/obsidian_wiki/lint.py b/obsidian_wiki/lint.py index 24c287b..8daca4e 100644 --- a/obsidian_wiki/lint.py +++ b/obsidian_wiki/lint.py @@ -20,7 +20,7 @@ from obsidian_wiki.provenance import ( parse_snapshots_field, ) from obsidian_wiki.vault import SKIP_DIRS as VAULT_SKIP_DIRS -from obsidian_wiki.vault import iter_instruction_files, iter_md, split_frontmatter +from obsidian_wiki.vault import iter_instruction_files, iter_md, okignore_patterns, okignored, split_frontmatter from obsidian_wiki.temporal import ( SUPERSEDED_FIELD, superseded_target, @@ -350,6 +350,7 @@ def _parse_page(path: Path, vault: Path) -> dict[str, Any]: "slug": _slug(path.stem), "title": values.get("title", "").strip() or path.stem, "summary": values.get("summary", "").strip(), + "redirects_to": values.get("redirects_to", "").strip(), "fields": fields, "links": links, "relationships": _parse_relationships(frontmatter), @@ -361,6 +362,11 @@ def _parse_page(path: Path, vault: Path) -> dict[str, Any]: } +# Manifest fields that list the pages a source produced (provenance.py reads +# the first two; pages_updated is the same list for an update run). +_COVERAGE_PAGE_KEYS = ("pages_produced", "pages_created", "pages_updated") + + def lint_vault( vault: Path, *, @@ -432,11 +438,18 @@ def lint_vault( # `Vector Search.md` beside `vector-search.md` still share one, since # nothing distinguishes them. Key this check with the graph's own page # selection so it reports exactly the pages the graph sees. + # Redirect stubs (wiki-dedup): a merged page keeps the secondary page's + # title and filename so old links stay usable, but it is not a content + # page. Stub paths are excluded from the duplicate checks below, and the + # pointers themselves are validated further down instead. + stub_paths = {page["path"] for page in pages if page["redirects_to"]} + stem_index: dict[str, list[str]] = defaultdict(list) for graph_page in iter_graph_pages(vault): - stem_index[graph_page_slug(graph_page, vault)].append( - graph_page.relative_to(vault).as_posix() - ) + graph_rel = graph_page.relative_to(vault).as_posix() + if graph_rel in stub_paths: + continue + stem_index[graph_page_slug(graph_page, vault)].append(graph_rel) duplicate_stems = [ {"stem": stem, "pages": sorted(paths)} for stem, paths in sorted(stem_index.items()) @@ -445,6 +458,8 @@ def lint_vault( title_index: dict[str, list[str]] = defaultdict(list) for page in pages: + if page["path"] in stub_paths: + continue title_index[page["title"].strip().lower()].append(page["path"]) duplicate_titles = [ {"title": title, "pages": paths} @@ -491,20 +506,27 @@ def lint_vault( (key or "").startswith("url:") for key, _ in _iter_entries(manifest_sources) ) url_index = clip_url_index(vault) if need_urls else None + okignore = okignore_patterns(vault) snapshot_mismatch = [] for page in pages: if page["slug"] in RESERVED_PAGE_STEMS: continue - expected = expected_snapshots_for_page( - vault, - page["path"], - manifest_sources, - inverted=inverted, - url_index=url_index, - ) + # An ignored archive neither appears in the report nor counts as a mismatch, + # whether the manifest or the page is the side still naming it. + expected = [ + rel + for rel in expected_snapshots_for_page( + vault, + page["path"], + manifest_sources, + inverted=inverted, + url_index=url_index, + ) + if not okignored(Path(rel), okignore) + ] if not expected: continue - actual = page["snapshots"] + actual = [rel for rel in page["snapshots"] if not okignored(Path(rel), okignore)] if set(expected) != set(actual): snapshot_mismatch.append( { @@ -515,6 +537,39 @@ def lint_vault( ) snapshot_mismatch.sort(key=lambda item: item["page"]) + # Source coverage: every ingested source should still map to a page that + # exists. A source whose page list is empty or points only at missing + # files was dropped somewhere between ingest and now (a failed write, a + # rename, a dedup), and nothing else would surface it. Entries with no + # page list at all (older count-only shapes) are skipped, not guessed at. + uncovered_sources: list[dict[str, Any]] = [] + for key, entry in _iter_entries(manifest_sources): + listed = [ + str(raw).strip().strip("[]") + for field in _COVERAGE_PAGE_KEYS + if isinstance(entry.get(field), list) + for raw in entry[field] + if raw + ] + if not any(isinstance(entry.get(field), list) for field in _COVERAGE_PAGE_KEYS): + continue + missing = sorted( + { + page + for page in listed + if not (vault / page).is_file() and not (vault / f"{page}.md").is_file() + } + ) + if not listed or len(missing) == len(set(listed)): + uncovered_sources.append( + { + "source": key or "", + "issue": "no_pages" if not listed else "pages_missing", + "missing": missing, + } + ) + uncovered_sources.sort(key=lambda item: item["source"]) + typed_relationship_issues: list[dict[str, Any]] = [] for page in pages: for index, relationship in enumerate(page["relationships"]): @@ -601,6 +656,29 @@ def lint_vault( {"page": page["path"], "target": successor, "issue": "missing_target"} ) + # Redirect pointers get the same treatment as `superseded_by`: a stub + # whose target is gone strands every old link that relied on it, and a + # stub pointing at another stub (or itself) never reaches content. + redirect_issues: list[dict[str, str]] = [] + pages_by_id = {page["id"]: page for page in pages} + for page in pages: + if page["path"] not in stub_paths: + continue + target = _relationship_link(page["redirects_to"]) + resolved_redirect = page_index.resolve(target, page["path"]) if target else None + if resolved_redirect is None: + redirect_issues.append( + {"page": page["path"], "target": target, "issue": "missing_target"} + ) + elif resolved_redirect == page["id"]: + redirect_issues.append( + {"page": page["path"], "target": target, "issue": "self_reference"} + ) + elif pages_by_id[resolved_redirect]["path"] in stub_paths: + redirect_issues.append( + {"page": page["path"], "target": target, "issue": "redirect_chain"} + ) + ledger_path = vault / TRUST_LEDGER_RELATIVE_PATH trust_report = ( check_trust_ledger( @@ -632,12 +710,14 @@ def lint_vault( "missing_summaries": sorted(missing_summaries), "machine_path_sources": machine_path_sources, "snapshot_mismatch": snapshot_mismatch, + "uncovered_sources": uncovered_sources, "orphan_pages": sorted(orphan_pages), "undated_claims": undated_claims, "typed_relationship_issues": typed_relationship_issues, # Sorted by page: these findings get diffed between CI runs. "temporal_errors": sorted(temporal_errors, key=lambda item: item["page"]), "superseded_dangling": sorted(superseded_dangling, key=lambda item: item["page"]), + "redirect_issues": sorted(redirect_issues, key=lambda item: item["page"]), "confidence_missing_fields": confidence_missing_fields, "trust_metadata_errors": trust_metadata_errors, "confidence_review_stale": trust_report["stale"] if trust_report else [], @@ -690,10 +770,12 @@ def lint_vault( "missing_summaries", "machine_path_sources", "snapshot_mismatch", + "uncovered_sources", "orphan_pages", "undated_claims", "typed_relationship_issues", "superseded_dangling", + "redirect_issues", ) ) or trust_findings_present diff --git a/obsidian_wiki/provenance.py b/obsidian_wiki/provenance.py index 81f19a8..ba447b7 100644 --- a/obsidian_wiki/provenance.py +++ b/obsidian_wiki/provenance.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any from obsidian_wiki.cache import _is_file_key, _iter_entries -from obsidian_wiki.vault import split_frontmatter +from obsidian_wiki.vault import iter_md_under, split_frontmatter _PAGE_LIST_KEYS = ("pages_produced", "pages_created") _URL_FIELDS = ("url", "source", "source_url") @@ -62,7 +62,7 @@ def clip_url_index(vault: Path) -> dict[str, list[str]]: root = _archived_dir(vault) if not root.is_dir(): return {} - for path in sorted(root.rglob("*.md")): + for path in iter_md_under(vault, root): text = path.read_text(encoding="utf-8", errors="replace") frontmatter = split_frontmatter(text)[0] rel = path.relative_to(vault).as_posix() diff --git a/obsidian_wiki/staging.py b/obsidian_wiki/staging.py index 94c04cc..3cefe1d 100644 --- a/obsidian_wiki/staging.py +++ b/obsidian_wiki/staging.py @@ -21,6 +21,8 @@ from datetime import date from pathlib import Path from typing import Any +from obsidian_wiki.vault import iter_md_under, okignore_patterns, okignored + STAGING_DIR = "_staging" RAW_DIR = "_raw" PATCH_SUFFIX = ".patch.md" @@ -92,7 +94,7 @@ def _live_relative(staged_rel: Path) -> str: if name.endswith(PATCH_SUFFIX): # `concepts/foo.patch.md` describes an edit to `concepts/foo.md`. name = name[: -len(PATCH_SUFFIX)] + ".md" - return str(staged_rel.with_name(name)) + return staged_rel.with_name(name).as_posix() def list_staged(vault: Path) -> list[StagedEntry]: @@ -102,10 +104,15 @@ def list_staged(vault: Path) -> list[StagedEntry]: if not root.is_dir(): return [] + patterns = okignore_patterns(vault) entries: list[StagedEntry] = [] - for staged in sorted(root.rglob("*.md")): + for staged in iter_md_under(vault, root): staged_rel = staged.relative_to(root) live_rel = _live_relative(staged_rel) + # The server diffs against the live page, which would print an ignored + # page's text for anything staged over it. + if okignored(Path(live_rel), patterns): + continue live = vault / live_rel if staged.name.endswith(PATCH_SUFFIX): kind = "patch" @@ -114,7 +121,7 @@ def list_staged(vault: Path) -> list[StagedEntry]: stat = staged.stat() entries.append( StagedEntry( - staged_path=str(staged.relative_to(vault)), + staged_path=staged.relative_to(vault).as_posix(), live_path=live_rel, kind=kind, staged_revision=revision(staged), @@ -189,7 +196,7 @@ def promote( staged = _staged_file(vault, rel) if staged.name.endswith(PATCH_SUFFIX): raise StagingError( - f"{staged.relative_to(vault)} is a patch, not a whole page — " + f"{staged.relative_to(vault).as_posix()} is a patch, not a whole page — " "merge it with the wiki-stage-commit skill, then promote the result" ) @@ -209,7 +216,7 @@ def promote( return { "action": "promote", - "staged_path": str(staged.relative_to(vault)), + "staged_path": staged.relative_to(vault).as_posix(), "live_path": live_rel, "kind": "update" if replaced else "new", "revision": revision(live), @@ -243,8 +250,8 @@ def discard(vault: Path, rel: str) -> dict[str, Any]: return { "action": "discard", - "staged_path": str(staged.relative_to(vault)), - "raw_path": str(raw.relative_to(vault)), + "staged_path": staged.relative_to(vault).as_posix(), + "raw_path": raw.relative_to(vault).as_posix(), } diff --git a/obsidian_wiki/trust.py b/obsidian_wiki/trust.py index f2923f9..316f346 100644 --- a/obsidian_wiki/trust.py +++ b/obsidian_wiki/trust.py @@ -22,7 +22,7 @@ from pathlib import Path, PurePosixPath from typing import Any from obsidian_wiki.vault import FRONTMATTER_RE as _FRONTMATTER_RE -from obsidian_wiki.vault import SKIP_DIRS, iter_md +from obsidian_wiki.vault import SKIP_DIRS, iter_md, okignore_patterns, okignored TRUST_LEDGER_RELATIVE_PATH = Path("_meta/trust-ledger.json") TRUST_LEDGER_SCHEMA_VERSION = 1 @@ -432,6 +432,10 @@ def update_trust_ledger( removed_not_applicable = sorted(set(ledger["pages"]) & set(not_applicable)) for rel in removed_not_applicable: del ledger["pages"][rel] + patterns = okignore_patterns(vault) + removed_ignored = [rel for rel in ledger["pages"] if okignored(Path(rel), patterns)] + for rel in removed_ignored: + del ledger["pages"][rel] selected: list[str] = [] for raw in page_paths: @@ -455,6 +459,7 @@ def update_trust_ledger( ledger["reviewed_at"] = reviewed_at ledger["not_applicable"] = sorted(not_applicable) ledger["removed_not_applicable"] = removed_not_applicable + ledger["removed_ignored"] = len(removed_ignored) return ledger @@ -581,6 +586,7 @@ def _empty_report( "score_mismatches": [], "missing_pages": [], "errors": [], + "ignored_entries": 0, "counts": {}, } @@ -711,8 +717,14 @@ def check_trust_ledger( } ) + patterns = okignore_patterns(vault) for rel in sorted(set(entries) - set(current)): - report["missing_pages"].append({"page": rel, "reason": "reviewed_page_missing"}) + # Counted, never named: the page must stay out of the output, but its + # leaving the review gate must not go unnoticed. + if okignored(Path(rel), patterns): + report["ignored_entries"] += 1 + else: + report["missing_pages"].append({"page": rel, "reason": "reviewed_page_missing"}) return _finalise_report(report) @@ -727,9 +739,10 @@ def _finalise_report(report: dict[str, Any]) -> dict[str, Any]: "errors", ) report["counts"] = {key: len(report[key]) for key in keys} + report["counts"]["ignored_entries"] = report["ignored_entries"] if report["errors"] or report["score_mismatches"]: report["status"] = "fail" - elif report["stale"] or report["unreviewed"] or report["missing_pages"]: + elif report["stale"] or report["unreviewed"] or report["missing_pages"] or report["ignored_entries"]: report["status"] = "warn" else: report["status"] = "pass" diff --git a/obsidian_wiki/vault.py b/obsidian_wiki/vault.py index 43799db..aaa06dd 100644 --- a/obsidian_wiki/vault.py +++ b/obsidian_wiki/vault.py @@ -115,6 +115,17 @@ def iter_instruction_files(vault: Path, skip_dirs: Iterable[str] = SKIP_DIRS) -> ) +def iter_md_under(vault: Path, root: Path) -> list[Path]: + """Every `.md` under `root` (a directory in `vault`), sorted, minus `.okignore`. + + For the tool-owned subtrees `iter_md` skips, such as `_staging/` and `_raw/`. + `rglob` rather than `_walk_md`: a symlinked folder here isn't part of the vault + the way a mounted project folder is, and it could lead out of the vault. + """ + patterns = okignore_patterns(vault) + return sorted(path for path in root.rglob("*.md") if not okignored(path.relative_to(vault), patterns)) + + def iter_md(vault: Path, skip_dirs: Iterable[str] = SKIP_DIRS) -> list[Path]: """Every `.md` page under `vault`, sorted, minus hidden/skipped dirs, agent-instruction files, and `.okignore`.""" diff --git a/tests/test_lint.py b/tests/test_lint.py index 081ab1f..e8ad201 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1065,3 +1065,127 @@ def test_lint_vault_warns_on_undated_counts_and_skips_dated_ones(tmp_path: Path) lines = page.read_text(encoding="utf-8").splitlines() assert all(lines[claim["line"] - 1].strip() == claim["text"] for claim in claims) assert report["status"] == "warn" + + +def _write_ignored_archive(vault: Path) -> None: + """`_write_archive_and_manifest`, plus a second archive `.okignore` covers.""" + _write_archive_and_manifest(vault) + hidden = vault / "_raw" / "_archived" / "private" / "hidden.md" + hidden.parent.mkdir(parents=True, exist_ok=True) + hidden.write_text("# clip\n", encoding="utf-8") + manifest = json.loads((vault / ".manifest.json").read_text(encoding="utf-8")) + manifest["sources"]["_raw/_archived/private/hidden.md"] = {"pages_produced": ["concepts/alpha.md"]} + (vault / ".manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + (vault / ".okignore").write_text("_raw/_archived/private\n", encoding="utf-8") + + +def test_snapshot_mismatch_does_not_name_an_ignored_archive(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _clean_pair(vault) + _write_ignored_archive(vault) + + rows = lint_vault(vault)["findings"]["snapshot_mismatch"] + + assert [row["expected"] for row in rows] == [["_raw/_archived/a.md"]] + + +def test_a_page_citing_an_ignored_archive_is_not_a_mismatch(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _clean_pair(vault, alpha_snapshots="[_raw/_archived/a.md, _raw/_archived/private/hidden.md]") + _write_ignored_archive(vault) + + assert lint_vault(vault)["findings"]["snapshot_mismatch"] == [] + + +def test_uncovered_sources_reports_sources_with_no_surviving_page(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _clean_pair(vault) + (vault / ".manifest.json").write_text( + json.dumps( + { + "sources": [ + {"path": "docs/kept.md", "pages_produced": ["concepts/alpha.md"]}, + {"path": "docs/partly.md", "pages_produced": ["concepts/beta", "concepts/gone.md"]}, + {"path": "docs/empty.md", "pages_produced": []}, + {"path": "docs/lost.md", "pages_created": ["concepts/gone.md"]}, + {"path": "docs/counts-only.md", "pages_created": 3}, + ] + } + ), + encoding="utf-8", + ) + + report = lint_vault(vault) + + assert report["findings"]["uncovered_sources"] == [ + {"source": "docs/empty.md", "issue": "no_pages", "missing": []}, + {"source": "docs/lost.md", "issue": "pages_missing", "missing": ["concepts/gone.md"]}, + ] + assert report["status"] == "warn" + + +def _redirect_stub(vault: Path, relpath: str, target: str, *, title: str | None = None) -> Path: + """A wiki-dedup merge leftover: same title/stem as the canonical page, + with a `redirects_to:` pointer in its frontmatter.""" + path = _page(vault, relpath, title=title) + text = path.read_text(encoding="utf-8") + path.write_text( + text.replace("\n---\n", f'\nredirects_to: "[[{target}]]"\n---\n', 1), + encoding="utf-8", + ) + return path + + +def test_lint_redirect_stub_is_not_a_duplicate(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _page(vault, "concepts/example.md", title="Example") + _redirect_stub(vault, "entities/example.md", "concepts/example", title="Example") + + report = lint_vault(vault, require_trust_ledger=False) + + assert report["findings"]["duplicate_titles"] == [] + assert report["findings"]["duplicate_stems"] == [] + assert report["findings"]["redirect_issues"] == [] + + +def test_lint_redirect_stub_with_missing_target_is_reported(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _page(vault, "concepts/example.md", title="Example") + _redirect_stub(vault, "entities/example.md", "concepts/gone", title="Example") + + report = lint_vault(vault, require_trust_ledger=False) + + assert report["findings"]["redirect_issues"] == [ + {"page": "entities/example.md", "target": "concepts/gone", "issue": "missing_target"} + ] + # A broken stub still stays out of the duplicate checks. + assert report["findings"]["duplicate_titles"] == [] + assert report["findings"]["duplicate_stems"] == [] + + +def test_lint_redirect_chain_is_reported(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _page(vault, "concepts/example.md", title="Example") + _redirect_stub(vault, "entities/example.md", "concepts/example", title="Example") + _redirect_stub(vault, "references/example.md", "entities/example", title="Example") + + report = lint_vault(vault, require_trust_ledger=False) + + assert report["findings"]["redirect_issues"] == [ + {"page": "references/example.md", "target": "entities/example", "issue": "redirect_chain"} + ] + + +def test_lint_two_real_pages_with_same_title_still_duplicate(tmp_path: Path) -> None: + vault = tmp_path / "vault" + _page(vault, "concepts/example.md", title="Example") + _page(vault, "entities/example.md", title="Example") + + report = lint_vault(vault, require_trust_ledger=False) + + assert report["findings"]["duplicate_titles"] == [ + {"title": "example", "pages": ["concepts/example.md", "entities/example.md"]} + ] + assert report["findings"]["duplicate_stems"] == [ + {"stem": "example", "pages": ["concepts/example.md", "entities/example.md"]} + ] diff --git a/tests/test_provenance.py b/tests/test_provenance.py index b5459c6..43401b1 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -4,6 +4,7 @@ from pathlib import Path from obsidian_wiki.provenance import ( archive_wikilink_relpath, + clip_url_index, expected_snapshots_for_page, invert_pages, parse_snapshots_field, @@ -218,3 +219,13 @@ def test_archive_wikilink_relpath_rejects_parent_traversal(tmp_path: Path) -> No vault = tmp_path / "vault" vault.mkdir() assert archive_wikilink_relpath(vault, "_raw/_archived/../attention") is None + + +def test_clip_url_index_honors_okignore(tmp_path: Path) -> None: + for rel, url in (("kept", "https://example.test/kept"), ("private/hidden", "https://example.test/hidden")): + clip = tmp_path / "_raw" / "_archived" / f"{rel}.md" + clip.parent.mkdir(parents=True, exist_ok=True) + clip.write_text(f"---\nurl: {url}\n---\n\nclip\n", encoding="utf-8") + (tmp_path / ".okignore").write_text("_raw/_archived/private\n", encoding="utf-8") + + assert clip_url_index(tmp_path) == {"https://example.test/kept": ["_raw/_archived/kept.md"]} diff --git a/tests/test_staging.py b/tests/test_staging.py index e303bcf..6561f8d 100644 --- a/tests/test_staging.py +++ b/tests/test_staging.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import pytest from obsidian_wiki.staging import ( @@ -200,3 +202,26 @@ def test_log_records_one_line_per_batch(vault): def test_log_is_silent_when_nothing_happened(vault): log_decisions(vault, []) assert not (vault / "log.md").exists() + + +def test_list_staged_honors_okignore(tmp_path): + write(tmp_path, "_staging/concepts/kept.md", "kept\n") + write(tmp_path, "_staging/private/hidden.md", "hidden\n") + # Staged over an ignored live page: listing it would diff against that page. + write(tmp_path, "notes/old/live.md", "secret\n") + write(tmp_path, "_staging/notes/old/live.md", "revised\n") + write(tmp_path, ".okignore", "_staging/private\n/notes/old\n") + + assert [Path(entry.live_path).as_posix() for entry in list_staged(tmp_path)] == ["concepts/kept.md"] + + +def test_list_staged_does_not_follow_a_symlinked_directory(tmp_path): + vault = tmp_path / "vault" + write(vault, "_staging/concepts/kept.md", "kept\n") + write(tmp_path, "outside/secret.md", "secret\n") + try: + (vault / "_staging" / "link").symlink_to(tmp_path / "outside", target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("symlinks unavailable on this platform") + + assert [Path(entry.live_path).as_posix() for entry in list_staged(vault)] == ["concepts/kept.md"] diff --git a/tests/test_trust.py b/tests/test_trust.py index 34e2feb..0a5070e 100644 --- a/tests/test_trust.py +++ b/tests/test_trust.py @@ -86,6 +86,7 @@ def test_reviewed_ledger_is_authoritative_instead_of_reclassifying_sources(tmp_p "score_mismatches": 0, "missing_pages": 0, "errors": 0, + "ignored_entries": 0, } assert {item["page"] for item in report["reviewed"]} == { "concepts/alpha.md", @@ -881,3 +882,50 @@ def test_required_trust_ledger_fails_closed_when_missing(tmp_path: Path) -> None strict_report = lint_vault(vault, require_trust_ledger=True, strict_trust=True) assert strict_report["status"] == "fail" + + +def _reviewed_then_ignored(tmp_path: Path) -> Path: + vault = tmp_path / "vault" + _page(vault, "concepts/alpha.md") + _page(vault, "private/hidden.md") + _write_ledger(vault) + (vault / ".okignore").write_text("private/\n", encoding="utf-8") + return vault + + +def test_check_counts_a_reviewed_page_ignored_later_without_naming_it(tmp_path: Path) -> None: + vault = _reviewed_then_ignored(tmp_path) + + report = check_trust_ledger(vault) + + # Still a warning: a page leaving the review gate has to stay visible. + assert report["status"] == "warn" + assert report["ignored_entries"] == 1 + assert report["counts"]["ignored_entries"] == 1 + assert report["missing_pages"] == [] + assert "hidden" not in json.dumps(report) + + +def test_a_deleted_page_is_still_reported_missing_by_name(tmp_path: Path) -> None: + vault = _reviewed_then_ignored(tmp_path) + (vault / "concepts" / "alpha.md").unlink() + + report = check_trust_ledger(vault) + + assert report["missing_pages"] == [{"page": "concepts/alpha.md", "reason": "reviewed_page_missing"}] + assert report["ignored_entries"] == 1 + + +def test_partial_review_prunes_ignored_entries_without_changing_other_reviews(tmp_path): + vault = _reviewed_then_ignored(tmp_path) + ledger_path = vault / "_meta/trust-ledger.json" + before = json.loads(ledger_path.read_text()) + updated = update_trust_ledger( + vault, ledger_path, reviewed_at="2026-11-07T00:00:00Z", + page_paths=["concepts/alpha.md"], + ) + assert updated["removed_ignored"] == 1 + assert set(updated["pages"]) == {"concepts/alpha.md"} + assert updated["pages"]["concepts/alpha.md"]["material_fingerprint"] == before["pages"]["concepts/alpha.md"]["material_fingerprint"] + write_trust_ledger(ledger_path, updated) + assert check_trust_ledger(vault)["ignored_entries"] == 0 diff --git a/tests/test_upstream_compat.py b/tests/test_upstream_compat.py index 4408b17..3d5aa43 100644 --- a/tests/test_upstream_compat.py +++ b/tests/test_upstream_compat.py @@ -83,3 +83,31 @@ def test_staging_cli_conflict_keeps_reviewed_and_live_files(tmp_path, capsys): ]) == 9 assert live.read_text() == "Concurrent live update" assert staged.read_text() == "Reviewed proposal" + + +@pytest.mark.parametrize("shape", ["dict", "list"]) +def test_source_coverage_accepts_local_archive_metadata_and_preserves_manifest(tmp_path, shape): + page = tmp_path / "concepts/a.md" + page.parent.mkdir() + page.write_text("---\ntitle: A\n---\n") + entries = [ + {"path": "original.pdf", "archive_path": "_sources/archive/a.pdf", + "pages_produced": ["concepts/a.md"], "bjd_part_numbers": ["BJD-123"]}, + {"path": "lost.pdf", "pages_updated": ["concepts/deleted.md"]}, + {"path": "legacy.pdf", "pages_created": 2}, + ] + manifest = tmp_path / ".manifest.json" + manifest.write_text(json.dumps({"sources": {e["path"]: e for e in entries} if shape == "dict" else entries})) + before = manifest.read_bytes() + report = lint_vault(tmp_path, require_trust_ledger=False) + assert report["findings"]["uncovered_sources"] == [ + {"source": "lost.pdf", "issue": "pages_missing", "missing": ["concepts/deleted.md"]} + ] + assert manifest.read_bytes() == before + + +def test_lint_cli_exposes_source_coverage_in_json_and_strict_exit(tmp_path, capsys): + (tmp_path / ".manifest.json").write_text(json.dumps({"sources": {"lost.pdf": {"pages_produced": []}}})) + assert main(["lint", str(tmp_path), "--json", "--strict"]) == 1 + report = json.loads(capsys.readouterr().out) + assert report["stats"]["findings"]["uncovered_sources"] == 1