nac
777607dc29
Track the source-of-truth code, skills, bootstrap rules, docs, and packaging files before adding new features. Exclude local vault config, generated skill mirrors, temporary exports, virtual environments, and business data files from version control. Validation: inspected staged file list and git ignored status before commit.
650 lines
22 KiB
Python
650 lines
22 KiB
Python
"""SQLite-backed file asset index for archived wiki sources.
|
|
|
|
The markdown wiki remains the source of distilled knowledge. This module keeps a
|
|
small, portable SQLite database for the operational question "where is the
|
|
original file?" so agents can find drawings, 3D files, spreadsheets, reports,
|
|
and other source assets without scanning the whole vault.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
DB_RELATIVE_PATH = Path("_meta") / "file-assets.sqlite"
|
|
SOURCE_REGISTRY_RELATIVE_PATH = Path("_meta") / "source-registry.jsonl"
|
|
|
|
TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+/-]*|[\u4e00-\u9fff]+")
|
|
BJD_PART_RE = re.compile(r"\b\d{6}-\d{6}(?:-[A-Za-z0-9]+)?\b")
|
|
GENERIC_PART_RE = re.compile(r"\b[A-Z]{1,6}\d{2,}[A-Z0-9-]*\b", re.IGNORECASE)
|
|
PITCH_RE = re.compile(
|
|
r"(?<!\d)(?:P\s*)?(\d+(?:\.\d+)?)\s*(?:mm)?\s*(?:pitch|间距)"
|
|
r"|(?:pitch|间距)\s*(?:P\s*)?(\d+(?:\.\d+)?)\s*(?:mm)?",
|
|
re.IGNORECASE,
|
|
)
|
|
PIN_RE = re.compile(r"(?<!\d)(\d{1,3})\s*(?:pin|p|位|芯)", re.IGNORECASE)
|
|
|
|
|
|
def clean_config_value(value: str) -> str:
|
|
"""Normalize config values that may be quoted once or twice."""
|
|
value = value.strip()
|
|
changed = True
|
|
while changed and len(value) >= 2:
|
|
changed = False
|
|
if (value[0], value[-1]) in {('"', '"'), ("'", "'")}:
|
|
value = value[1:-1].strip()
|
|
changed = True
|
|
return value
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def db_path_for_vault(vault_path: str | Path) -> Path:
|
|
vault = Path(clean_config_value(str(vault_path))).expanduser()
|
|
return vault / DB_RELATIVE_PATH
|
|
|
|
|
|
def registry_path_for_vault(vault_path: str | Path) -> Path:
|
|
vault = Path(clean_config_value(str(vault_path))).expanduser()
|
|
return vault / SOURCE_REGISTRY_RELATIVE_PATH
|
|
|
|
|
|
def connect(db_path: Path) -> sqlite3.Connection:
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
return conn
|
|
|
|
|
|
def init_db(db_path: Path) -> None:
|
|
with connect(db_path) as conn:
|
|
conn.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS file_assets (
|
|
file_id TEXT PRIMARY KEY,
|
|
status TEXT NOT NULL DEFAULT 'active',
|
|
sha256 TEXT NOT NULL,
|
|
archive_path TEXT NOT NULL,
|
|
original_path TEXT,
|
|
original_name TEXT,
|
|
file_ext TEXT,
|
|
mime_type TEXT,
|
|
source_type TEXT,
|
|
doc_type TEXT,
|
|
project TEXT,
|
|
title TEXT,
|
|
summary TEXT,
|
|
visibility TEXT,
|
|
business_categories TEXT,
|
|
product_lines TEXT,
|
|
customers TEXT,
|
|
competitors TEXT,
|
|
standards TEXT,
|
|
market_tags TEXT,
|
|
bjd_part_numbers TEXT,
|
|
competitor_part_numbers TEXT,
|
|
series TEXT,
|
|
pitch TEXT,
|
|
pin_count TEXT,
|
|
related_pages TEXT,
|
|
catalog_paths TEXT,
|
|
extracted_text TEXT,
|
|
classification_confidence REAL,
|
|
classification_evidence TEXT,
|
|
needs_review INTEGER NOT NULL DEFAULT 0,
|
|
size_bytes INTEGER,
|
|
modified_at TEXT,
|
|
archived_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_file_assets_status
|
|
ON file_assets(status);
|
|
CREATE INDEX IF NOT EXISTS idx_file_assets_doc_type
|
|
ON file_assets(doc_type);
|
|
CREATE INDEX IF NOT EXISTS idx_file_assets_project
|
|
ON file_assets(project);
|
|
CREATE INDEX IF NOT EXISTS idx_file_assets_original_name
|
|
ON file_assets(original_name);
|
|
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS file_assets_fts USING fts5(
|
|
file_id UNINDEXED,
|
|
original_name,
|
|
title,
|
|
summary,
|
|
archive_path,
|
|
doc_type,
|
|
product_lines,
|
|
business_categories,
|
|
customers,
|
|
competitors,
|
|
standards,
|
|
bjd_part_numbers,
|
|
competitor_part_numbers,
|
|
series,
|
|
pitch,
|
|
pin_count,
|
|
extracted_text
|
|
);
|
|
"""
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RebuildResult:
|
|
db_path: Path
|
|
registry_path: Path
|
|
indexed: int
|
|
skipped: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class StageResult:
|
|
file_id: str
|
|
path: Path
|
|
db_path: Path
|
|
|
|
|
|
def rebuild_from_registry(vault_path: str | Path, *, db_path: Path | None = None) -> RebuildResult:
|
|
vault = Path(clean_config_value(str(vault_path))).expanduser()
|
|
db_path = db_path or db_path_for_vault(vault)
|
|
registry_path = registry_path_for_vault(vault)
|
|
init_db(db_path)
|
|
|
|
if not registry_path.is_file():
|
|
raise FileNotFoundError(f"source registry not found: {registry_path}")
|
|
|
|
indexed = 0
|
|
skipped = 0
|
|
with connect(db_path) as conn:
|
|
for raw in registry_path.read_text(encoding="utf-8").splitlines():
|
|
if not raw.strip():
|
|
continue
|
|
try:
|
|
record = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
skipped += 1
|
|
continue
|
|
if record.get("record_type") != "source_archive":
|
|
continue
|
|
upsert_asset(conn, asset_from_registry_record(record), status="active")
|
|
indexed += 1
|
|
return RebuildResult(db_path=db_path, registry_path=registry_path, indexed=indexed, skipped=skipped)
|
|
|
|
|
|
def asset_from_registry_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
source_id = str(record.get("source_id") or "")
|
|
sha256 = source_id.removeprefix("sha256:")
|
|
archive_path = str(record.get("archive_path") or "")
|
|
original_name = str(record.get("original_name") or Path(archive_path).name)
|
|
text_blob = " ".join(
|
|
str(record.get(key) or "")
|
|
for key in ("original_name", "title", "summary", "archive_path", "original_path")
|
|
)
|
|
text_blob += " " + " ".join(_flatten_to_strings(record.get(key)) for key in _CLASSIFICATION_KEYS)
|
|
derived = derive_identifiers(text_blob)
|
|
file_ext = Path(original_name).suffix.lower()
|
|
confidence = _float_or_none(record.get("classification_confidence"))
|
|
|
|
return {
|
|
"file_id": source_id or f"sha256:{sha256}",
|
|
"sha256": sha256,
|
|
"archive_path": archive_path,
|
|
"original_path": record.get("original_path"),
|
|
"original_name": original_name,
|
|
"file_ext": file_ext,
|
|
"mime_type": mimetypes.guess_type(original_name)[0],
|
|
"source_type": record.get("source_type") or guess_source_type(original_name),
|
|
"doc_type": record.get("doc_type") or "other",
|
|
"project": record.get("project"),
|
|
"title": record.get("title") or Path(original_name).stem,
|
|
"summary": record.get("summary"),
|
|
"visibility": record.get("visibility") or "internal",
|
|
"business_categories": _json_list(record.get("business_categories")),
|
|
"product_lines": _json_list(record.get("product_lines")),
|
|
"customers": _json_list(record.get("customers")),
|
|
"competitors": _json_list(record.get("competitors")),
|
|
"standards": _json_list(record.get("standards")),
|
|
"market_tags": _json_list(record.get("market_tags")),
|
|
"bjd_part_numbers": _json_list(derived["bjd_part_numbers"]),
|
|
"competitor_part_numbers": _json_list(derived["competitor_part_numbers"]),
|
|
"series": _json_list(derived["series"]),
|
|
"pitch": _json_list(derived["pitch"]),
|
|
"pin_count": _json_list(derived["pin_count"]),
|
|
"related_pages": _json_list(record.get("related_pages")),
|
|
"catalog_paths": _json_list(record.get("catalog_paths")),
|
|
"extracted_text": None,
|
|
"classification_confidence": confidence,
|
|
"classification_evidence": _json_list(record.get("classification_evidence")),
|
|
"needs_review": 1 if confidence is None or confidence < 0.7 or record.get("doc_type") in (None, "other") else 0,
|
|
"size_bytes": _int_or_none(record.get("size_bytes")),
|
|
"modified_at": record.get("modified_at"),
|
|
"archived_at": record.get("archived_at"),
|
|
}
|
|
|
|
|
|
_CLASSIFICATION_KEYS = (
|
|
"business_categories",
|
|
"product_lines",
|
|
"customers",
|
|
"competitors",
|
|
"standards",
|
|
"market_tags",
|
|
)
|
|
|
|
|
|
def derive_identifiers(text: str) -> dict[str, list[str]]:
|
|
bjd_parts = sorted(set(match.group(0).upper() for match in BJD_PART_RE.finditer(text)))
|
|
generic_parts = sorted(set(match.group(0).upper() for match in GENERIC_PART_RE.finditer(text)))
|
|
competitor_parts = [part for part in generic_parts if part not in bjd_parts]
|
|
pitch = sorted(
|
|
set(
|
|
value
|
|
for match in PITCH_RE.finditer(text)
|
|
for value in (match.group(1), match.group(2))
|
|
if value and _looks_like_pitch(value)
|
|
)
|
|
)
|
|
pin_count = sorted(set(match.group(1) for match in PIN_RE.finditer(text)))
|
|
series = sorted(
|
|
set(
|
|
token.upper()
|
|
for token in generic_parts
|
|
if token.upper().startswith(("X", "FH", "DF", "BM", "MINI", "HSD", "FAKRA"))
|
|
)
|
|
)
|
|
return {
|
|
"bjd_part_numbers": bjd_parts,
|
|
"competitor_part_numbers": competitor_parts[:40],
|
|
"series": series[:40],
|
|
"pitch": pitch[:20],
|
|
"pin_count": pin_count[:20],
|
|
}
|
|
|
|
|
|
def _looks_like_pitch(value: str) -> bool:
|
|
try:
|
|
numeric = float(value)
|
|
except ValueError:
|
|
return False
|
|
return 0 < numeric <= 20
|
|
|
|
|
|
def upsert_asset(conn: sqlite3.Connection, asset: dict[str, Any], *, status: str | None = None) -> None:
|
|
now = utc_now()
|
|
asset = dict(asset)
|
|
asset.setdefault("created_at", now)
|
|
asset["updated_at"] = now
|
|
asset["status"] = status or asset.get("status") or "active"
|
|
|
|
columns = [
|
|
"file_id",
|
|
"status",
|
|
"sha256",
|
|
"archive_path",
|
|
"original_path",
|
|
"original_name",
|
|
"file_ext",
|
|
"mime_type",
|
|
"source_type",
|
|
"doc_type",
|
|
"project",
|
|
"title",
|
|
"summary",
|
|
"visibility",
|
|
"business_categories",
|
|
"product_lines",
|
|
"customers",
|
|
"competitors",
|
|
"standards",
|
|
"market_tags",
|
|
"bjd_part_numbers",
|
|
"competitor_part_numbers",
|
|
"series",
|
|
"pitch",
|
|
"pin_count",
|
|
"related_pages",
|
|
"catalog_paths",
|
|
"extracted_text",
|
|
"classification_confidence",
|
|
"classification_evidence",
|
|
"needs_review",
|
|
"size_bytes",
|
|
"modified_at",
|
|
"archived_at",
|
|
"created_at",
|
|
"updated_at",
|
|
]
|
|
placeholders = ", ".join("?" for _ in columns)
|
|
update_clause = ", ".join(f"{col}=excluded.{col}" for col in columns if col not in {"file_id", "created_at"})
|
|
conn.execute(
|
|
f"""
|
|
INSERT INTO file_assets ({", ".join(columns)})
|
|
VALUES ({placeholders})
|
|
ON CONFLICT(file_id) DO UPDATE SET {update_clause}
|
|
""",
|
|
[asset.get(col) for col in columns],
|
|
)
|
|
refresh_fts(conn, asset["file_id"])
|
|
|
|
|
|
def refresh_fts(conn: sqlite3.Connection, file_id: str) -> None:
|
|
conn.execute("DELETE FROM file_assets_fts WHERE file_id = ?", (file_id,))
|
|
row = conn.execute("SELECT * FROM file_assets WHERE file_id = ?", (file_id,)).fetchone()
|
|
if row is None:
|
|
return
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO file_assets_fts (
|
|
file_id, original_name, title, summary, archive_path, doc_type,
|
|
product_lines, business_categories, customers, competitors, standards,
|
|
bjd_part_numbers, competitor_part_numbers, series, pitch, pin_count, extracted_text
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
[
|
|
row["file_id"],
|
|
row["original_name"],
|
|
row["title"],
|
|
row["summary"],
|
|
row["archive_path"],
|
|
row["doc_type"],
|
|
row["product_lines"],
|
|
row["business_categories"],
|
|
row["customers"],
|
|
row["competitors"],
|
|
row["standards"],
|
|
row["bjd_part_numbers"],
|
|
row["competitor_part_numbers"],
|
|
row["series"],
|
|
row["pitch"],
|
|
row["pin_count"],
|
|
row["extracted_text"],
|
|
],
|
|
)
|
|
|
|
|
|
def stage_files(
|
|
vault_path: str | Path,
|
|
paths: Iterable[str | Path],
|
|
*,
|
|
project: str = "misc",
|
|
doc_type: str = "other",
|
|
visibility: str = "internal",
|
|
db_path: Path | None = None,
|
|
) -> list[StageResult]:
|
|
vault = Path(clean_config_value(str(vault_path))).expanduser()
|
|
db_path = db_path or db_path_for_vault(vault)
|
|
init_db(db_path)
|
|
staged_root = vault / "_sources" / "inbox" / project / datetime.now().strftime("%Y-%m")
|
|
staged_root.mkdir(parents=True, exist_ok=True)
|
|
results: list[StageResult] = []
|
|
|
|
with connect(db_path) as conn:
|
|
for raw_path in paths:
|
|
source = Path(raw_path).expanduser().resolve()
|
|
if not source.is_file():
|
|
raise FileNotFoundError(f"not a file: {source}")
|
|
digest = sha256_file(source)
|
|
safe = safe_basename(source.name)
|
|
staged_path = staged_root / f"{digest[:12]}--{safe}"
|
|
if not staged_path.exists():
|
|
shutil.copy2(source, staged_path)
|
|
stat = source.stat()
|
|
derived = derive_identifiers(f"{source.name} {source}")
|
|
asset = {
|
|
"file_id": f"sha256:{digest}",
|
|
"sha256": digest,
|
|
"archive_path": str(staged_path.relative_to(vault)),
|
|
"original_path": str(source),
|
|
"original_name": source.name,
|
|
"file_ext": source.suffix.lower(),
|
|
"mime_type": mimetypes.guess_type(source.name)[0],
|
|
"source_type": guess_source_type(source.name),
|
|
"doc_type": doc_type,
|
|
"project": project,
|
|
"title": source.stem,
|
|
"summary": "Staged upload waiting for wiki ingest/classification.",
|
|
"visibility": visibility,
|
|
"business_categories": _json_list([]),
|
|
"product_lines": _json_list([]),
|
|
"customers": _json_list([]),
|
|
"competitors": _json_list([]),
|
|
"standards": _json_list([]),
|
|
"market_tags": _json_list([]),
|
|
"bjd_part_numbers": _json_list(derived["bjd_part_numbers"]),
|
|
"competitor_part_numbers": _json_list(derived["competitor_part_numbers"]),
|
|
"series": _json_list(derived["series"]),
|
|
"pitch": _json_list(derived["pitch"]),
|
|
"pin_count": _json_list(derived["pin_count"]),
|
|
"related_pages": _json_list([]),
|
|
"catalog_paths": _json_list([]),
|
|
"extracted_text": None,
|
|
"classification_confidence": 0.0,
|
|
"classification_evidence": _json_list(["staged from CLI upload"]),
|
|
"needs_review": 1,
|
|
"size_bytes": stat.st_size,
|
|
"modified_at": _mtime_iso(stat.st_mtime),
|
|
"archived_at": utc_now(),
|
|
}
|
|
upsert_asset(conn, asset, status="staged")
|
|
results.append(StageResult(file_id=f"sha256:{digest}", path=staged_path, db_path=db_path))
|
|
return results
|
|
|
|
|
|
def search_assets(
|
|
vault_path: str | Path,
|
|
query: str,
|
|
*,
|
|
db_path: Path | None = None,
|
|
limit: int = 10,
|
|
doc_type: str | None = None,
|
|
source_type: str | None = None,
|
|
product_line: str | None = None,
|
|
customer: str | None = None,
|
|
competitor: str | None = None,
|
|
status: str | None = None,
|
|
include_staged: bool = True,
|
|
) -> list[dict[str, Any]]:
|
|
db_path = db_path or db_path_for_vault(vault_path)
|
|
if not db_path.is_file():
|
|
raise FileNotFoundError(f"file asset index not found: {db_path}")
|
|
|
|
filters: list[str] = []
|
|
params: list[Any] = []
|
|
if not include_staged:
|
|
filters.append("status != 'staged'")
|
|
if status:
|
|
filters.append("status = ?")
|
|
params.append(status)
|
|
if doc_type:
|
|
filters.append("doc_type = ?")
|
|
params.append(doc_type)
|
|
if source_type:
|
|
filters.append("source_type = ?")
|
|
params.append(source_type)
|
|
|
|
where = f"WHERE {' AND '.join(filters)}" if filters else ""
|
|
rows: list[dict[str, Any]] = []
|
|
with connect(db_path) as conn:
|
|
for row in conn.execute(f"SELECT * FROM file_assets {where}", params).fetchall():
|
|
record = dict(row)
|
|
if product_line and product_line.lower() not in _searchable(record.get("product_lines")):
|
|
continue
|
|
if customer and customer.lower() not in _searchable(record.get("customers")):
|
|
continue
|
|
if competitor and competitor.lower() not in _searchable(record.get("competitors")):
|
|
continue
|
|
score = score_asset(record, query)
|
|
if score <= 0 and query.strip():
|
|
continue
|
|
record["score"] = score
|
|
record["absolute_path"] = absolute_asset_path(vault_path, record["archive_path"])
|
|
rows.append(record)
|
|
|
|
rows.sort(key=lambda item: (item["score"], item.get("updated_at") or ""), reverse=True)
|
|
return rows[:limit]
|
|
|
|
|
|
def score_asset(record: dict[str, Any], query: str) -> int:
|
|
query = query.strip()
|
|
if not query:
|
|
return 1
|
|
haystack = _searchable(
|
|
[
|
|
record.get("original_name"),
|
|
record.get("title"),
|
|
record.get("summary"),
|
|
record.get("archive_path"),
|
|
record.get("doc_type"),
|
|
record.get("source_type"),
|
|
record.get("business_categories"),
|
|
record.get("product_lines"),
|
|
record.get("customers"),
|
|
record.get("competitors"),
|
|
record.get("standards"),
|
|
record.get("market_tags"),
|
|
record.get("bjd_part_numbers"),
|
|
record.get("competitor_part_numbers"),
|
|
record.get("series"),
|
|
record.get("pitch"),
|
|
record.get("pin_count"),
|
|
record.get("extracted_text"),
|
|
]
|
|
)
|
|
q = query.lower()
|
|
tokens = [token.lower() for token in TOKEN_RE.findall(query)]
|
|
score = 0
|
|
if q in haystack:
|
|
score += 25
|
|
for token in tokens:
|
|
if token in haystack:
|
|
score += 7 if len(token) >= 4 else 3
|
|
if tokens and all(token in haystack for token in tokens):
|
|
score += 12
|
|
if score > 0 and record.get("doc_type") in {"drawing", "test_report", "product_parameter", "cross_reference"}:
|
|
score += 1
|
|
return score
|
|
|
|
|
|
def absolute_asset_path(vault_path: str | Path, archive_path: str) -> str:
|
|
archive = Path(archive_path)
|
|
if archive.is_absolute():
|
|
return str(archive)
|
|
vault = Path(clean_config_value(str(vault_path))).expanduser()
|
|
return str((vault / archive).resolve())
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
hasher = hashlib.sha256()
|
|
with path.open("rb") as fh:
|
|
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
|
hasher.update(chunk)
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def safe_basename(name: str) -> str:
|
|
name = re.sub(r"[/\\:\x00-\x1f]", "_", name.strip())
|
|
return name or "unnamed"
|
|
|
|
|
|
def guess_source_type(name: str) -> str:
|
|
ext = Path(name).suffix.lower()
|
|
if ext == ".pdf":
|
|
return "pdf"
|
|
if ext in {".xlsx", ".xls", ".csv", ".tsv"}:
|
|
return "spreadsheet"
|
|
if ext in {".md", ".txt", ".doc", ".docx", ".ppt", ".pptx"}:
|
|
return "document"
|
|
if ext in {".png", ".jpg", ".jpeg", ".webp", ".gif", ".tif", ".tiff"}:
|
|
return "image"
|
|
if ext in {".step", ".stp", ".igs", ".iges", ".x_t", ".x_b", ".sldprt", ".sldasm", ".dwg", ".dxf"}:
|
|
return "cad"
|
|
if ext in {".json", ".jsonl", ".xml", ".yaml", ".yml"}:
|
|
return "data"
|
|
return "other"
|
|
|
|
|
|
def format_results_table(rows: list[dict[str, Any]]) -> str:
|
|
if not rows:
|
|
return "No matching file assets found."
|
|
lines = ["score type status file"]
|
|
lines.append("----- -------------- ------- ----")
|
|
for row in rows:
|
|
score = str(row.get("score", ""))
|
|
doc_type = _clip(str(row.get("doc_type") or row.get("source_type") or ""), 14)
|
|
status = _clip(str(row.get("status") or ""), 7)
|
|
file_name = row.get("original_name") or Path(str(row.get("archive_path") or "")).name
|
|
rel = row.get("archive_path") or ""
|
|
lines.append(f"{score:>5} {doc_type:<14} {status:<7} {file_name} ({rel})")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _clip(value: str, width: int) -> str:
|
|
return value[: width - 1] + "…" if len(value) > width else value
|
|
|
|
|
|
def _json_list(value: Any) -> str:
|
|
if value is None:
|
|
data: list[Any] = []
|
|
elif isinstance(value, list):
|
|
data = value
|
|
elif isinstance(value, tuple):
|
|
data = list(value)
|
|
else:
|
|
data = [value]
|
|
return json.dumps([item for item in data if item not in (None, "")], ensure_ascii=False)
|
|
|
|
|
|
def _flatten_to_strings(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, str):
|
|
return value
|
|
if isinstance(value, (list, tuple, set)):
|
|
return " ".join(_flatten_to_strings(item) for item in value)
|
|
if isinstance(value, dict):
|
|
return " ".join(_flatten_to_strings(item) for item in value.values())
|
|
return str(value)
|
|
|
|
|
|
def _searchable(value: Any) -> str:
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return value.lower()
|
|
return _flatten_to_strings(parsed).lower()
|
|
return _flatten_to_strings(value).lower()
|
|
|
|
|
|
def _float_or_none(value: Any) -> float | None:
|
|
try:
|
|
return None if value is None or value == "" else float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _int_or_none(value: Any) -> int | None:
|
|
try:
|
|
return None if value is None or value == "" else int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _mtime_iso(mtime: float) -> str:
|
|
return datetime.fromtimestamp(mtime, timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|