chore(repo): establish project baseline
Track the source-of-truth code, skills, bootstrap rules, docs, and packaging files before adding new features. Exclude local vault config, generated skill mirrors, temporary exports, virtual environments, and business data files from version control. Validation: inspected staged file list and git ignored status before commit.
This commit is contained in:
nac
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
import collections
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PART_RE = re.compile(
|
||||
r"\b(?:\d{6}-\d{4,6}(?:-HF)?|\d{3,}-\d{3,}(?:-HF)?|FH\d+[A-Z0-9-]*|C-\d{7}|\d{6,})\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
KEYWORDS = [
|
||||
"报价",
|
||||
"询价",
|
||||
"样品",
|
||||
"打样",
|
||||
"图纸",
|
||||
"承认书",
|
||||
"规格书",
|
||||
"替代",
|
||||
"替换",
|
||||
"竞品",
|
||||
"Hirose",
|
||||
"Molex",
|
||||
"TE",
|
||||
"泰科",
|
||||
"JAE",
|
||||
"Kyocera",
|
||||
"IRISO",
|
||||
"订单",
|
||||
"交期",
|
||||
"交货",
|
||||
"品质",
|
||||
"客诉",
|
||||
"投诉",
|
||||
"不良",
|
||||
"8D",
|
||||
"退货",
|
||||
"索赔",
|
||||
"对账",
|
||||
"付款",
|
||||
"发票",
|
||||
"合同",
|
||||
"送样",
|
||||
"开模",
|
||||
"开发",
|
||||
"零跑",
|
||||
"领跑",
|
||||
"维胜",
|
||||
"恒润",
|
||||
"涵润",
|
||||
"安悦",
|
||||
"先锋",
|
||||
]
|
||||
|
||||
|
||||
def safe_person(person):
|
||||
if not isinstance(person, dict):
|
||||
return None, None
|
||||
email = (person.get("email") or "").lower()
|
||||
name = (person.get("name") or "").strip()
|
||||
domain = email.split("@")[-1] if "@" in email else None
|
||||
return domain, name if name and len(name) <= 30 else None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: analyze_email_jsonl.py /path/to/llm_input.jsonl", file=sys.stderr)
|
||||
return 2
|
||||
path = Path(sys.argv[1])
|
||||
threads = msgs = attach = trunc = 0
|
||||
kw = collections.Counter()
|
||||
parts = collections.Counter()
|
||||
domains = collections.Counter()
|
||||
names = collections.Counter()
|
||||
years = collections.Counter()
|
||||
months = collections.Counter()
|
||||
attach_ext = collections.Counter()
|
||||
interesting = []
|
||||
|
||||
with path.open(encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
obj = json.loads(line)
|
||||
threads += 1
|
||||
trunc += bool(obj.get("body_truncated"))
|
||||
messages = obj.get("messages", [])
|
||||
subject = obj.get("suggested_title_from_subject", "")
|
||||
text = " ".join(
|
||||
[subject]
|
||||
+ [
|
||||
(m.get("subject") or "") + " " + (m.get("body_text") or "")
|
||||
for m in messages
|
||||
]
|
||||
)
|
||||
lowered = text.lower()
|
||||
for key in KEYWORDS:
|
||||
if key.lower() in lowered:
|
||||
kw[key] += 1
|
||||
found_parts = [x.upper() for x in PART_RE.findall(text)]
|
||||
for part in found_parts:
|
||||
if len(part) >= 6:
|
||||
parts[part] += 1
|
||||
for msg in messages:
|
||||
msgs += 1
|
||||
month = (msg.get("date") or "")[:7]
|
||||
if len(month) == 7:
|
||||
months[month] += 1
|
||||
years[month[:4]] += 1
|
||||
people = [msg.get("from", {})] + msg.get("to", []) + msg.get("cc", [])
|
||||
for person in people:
|
||||
domain, name = safe_person(person)
|
||||
if domain:
|
||||
domains[domain] += 1
|
||||
if name:
|
||||
names[name] += 1
|
||||
for item in msg.get("attachments", []) or []:
|
||||
attach += 1
|
||||
if isinstance(item, dict):
|
||||
filename = item.get("filename") or item.get("name") or ""
|
||||
else:
|
||||
filename = str(item)
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "[no_ext]"
|
||||
attach_ext[ext] += 1
|
||||
score = (
|
||||
sum(lowered.count(key.lower()) for key in KEYWORDS)
|
||||
+ len(found_parts) * 2
|
||||
+ len(messages) // 3
|
||||
+ len(obj.get("attachments", [])) * 2
|
||||
)
|
||||
if score > 8:
|
||||
interesting.append(
|
||||
{
|
||||
"score": score,
|
||||
"date_end": obj.get("date_end", ""),
|
||||
"thread_id": obj.get("thread_id", ""),
|
||||
"subject": subject[:120],
|
||||
"message_count": len(messages),
|
||||
"attachment_count": len(obj.get("attachments", [])),
|
||||
"parts": found_parts[:12],
|
||||
"keywords": [
|
||||
key for key in KEYWORDS if key.lower() in lowered
|
||||
][:12],
|
||||
}
|
||||
)
|
||||
|
||||
result = {
|
||||
"source": str(path),
|
||||
"threads": threads,
|
||||
"messages": msgs,
|
||||
"attachments": attach,
|
||||
"truncated_threads": trunc,
|
||||
"years": years.most_common(),
|
||||
"months_top": months.most_common(18),
|
||||
"keywords": kw.most_common(50),
|
||||
"parts_top": parts.most_common(80),
|
||||
"domains_top": domains.most_common(60),
|
||||
"names_top": names.most_common(60),
|
||||
"attachment_extensions": attach_ext.most_common(40),
|
||||
"interesting_threads": sorted(
|
||||
interesting, key=lambda x: (x["score"], x["date_end"]), reverse=True
|
||||
)[:120],
|
||||
}
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2)
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.obsidian-wiki.daily-update</string>
|
||||
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/bin/bash</string>
|
||||
<!-- Substituted automatically by /daily-update setup mode — do not edit manually -->
|
||||
<string>OBSIDIAN_WIKI_REPO/scripts/daily-update.sh</string>
|
||||
</array>
|
||||
|
||||
<!-- Run at 9:00 AM daily -->
|
||||
<key>StartCalendarInterval</key>
|
||||
<dict>
|
||||
<key>Hour</key>
|
||||
<integer>9</integer>
|
||||
<key>Minute</key>
|
||||
<integer>0</integer>
|
||||
</dict>
|
||||
|
||||
<!-- Also run immediately on load if the last run was missed (laptop was asleep) -->
|
||||
<key>RunAtLoad</key>
|
||||
<false/>
|
||||
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/obsidian-wiki-daily.log</string>
|
||||
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/obsidian-wiki-daily.err</string>
|
||||
</dict>
|
||||
</plist>
|
||||
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Daily wiki index update — called by launchd or directly.
|
||||
# Checks if any history sources are stale and writes vault-scoped state files
|
||||
# that the shell prompt reads on terminal open.
|
||||
#
|
||||
# Config resolution order (mirrors llm-wiki/SKILL.md protocol):
|
||||
# 1. Walk up from CWD looking for .env with OBSIDIAN_VAULT_PATH
|
||||
# 2. Fall back to ~/.obsidian-wiki/config
|
||||
# 3. Exit with error if neither found
|
||||
|
||||
_find_config() {
|
||||
local dir="$PWD"
|
||||
while [[ "$dir" != "$HOME" && "$dir" != "/" ]]; do
|
||||
if [[ -f "$dir/.env" ]] && grep -q "OBSIDIAN_VAULT_PATH" "$dir/.env" 2>/dev/null; then
|
||||
echo "$dir/.env"
|
||||
return
|
||||
fi
|
||||
dir="$(dirname "$dir")"
|
||||
done
|
||||
if [[ -f "$HOME/.obsidian-wiki/config" ]]; then
|
||||
echo "$HOME/.obsidian-wiki/config"
|
||||
return
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
CONFIG_FILE="$(_find_config)"
|
||||
|
||||
if [[ -z "$CONFIG_FILE" ]]; then
|
||||
echo "[wiki-daily] No config found. Run wiki-setup to initialize your wiki." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$CONFIG_FILE"
|
||||
|
||||
if [[ -z "${OBSIDIAN_VAULT_PATH:-}" ]]; then
|
||||
echo "[wiki-daily] OBSIDIAN_VAULT_PATH not set in $CONFIG_FILE — skipping" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Vault-scoped state dir (supports multiple vaults independently)
|
||||
VAULT_ID=$(echo "$OBSIDIAN_VAULT_PATH" | md5sum 2>/dev/null | cut -c1-8 || \
|
||||
md5 -q - <<< "$OBSIDIAN_VAULT_PATH" 2>/dev/null | cut -c1-8 || \
|
||||
echo "default")
|
||||
STATE_DIR="$HOME/.obsidian-wiki/state/$VAULT_ID"
|
||||
mkdir -p "$STATE_DIR"
|
||||
|
||||
# Write vault path so wiki-notify.sh can find this state dir
|
||||
echo "$OBSIDIAN_VAULT_PATH" > "$STATE_DIR/.vault_path"
|
||||
|
||||
MANIFEST="$OBSIDIAN_VAULT_PATH/.manifest.json"
|
||||
|
||||
# Count sources modified after last ingest
|
||||
stale_count=0
|
||||
if [[ -f "$MANIFEST" ]]; then
|
||||
last_updated=$(python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(open('$MANIFEST'))
|
||||
print(d.get('last_updated',''))
|
||||
except:
|
||||
print('')
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$last_updated" ]]; then
|
||||
stale_count=$(MANIFEST="$MANIFEST" OBSIDIAN_VAULT_PATH="$OBSIDIAN_VAULT_PATH" python3 - <<'PYEOF'
|
||||
import json, os, sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
manifest_path = os.environ["MANIFEST"]
|
||||
try:
|
||||
with open(manifest_path) as f:
|
||||
manifest = json.load(f)
|
||||
except Exception:
|
||||
print(0)
|
||||
sys.exit()
|
||||
|
||||
last_updated_str = manifest.get("last_updated", "")
|
||||
try:
|
||||
last_updated = datetime.fromisoformat(last_updated_str.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
print(0)
|
||||
sys.exit()
|
||||
|
||||
vault_path = os.environ.get("OBSIDIAN_VAULT_PATH", "")
|
||||
stale = 0
|
||||
for path, meta in manifest.get("sources", {}).items():
|
||||
expanded = os.path.expanduser(path)
|
||||
if not os.path.isabs(expanded) and vault_path:
|
||||
expanded = os.path.join(vault_path, expanded)
|
||||
if os.path.exists(expanded):
|
||||
mtime = datetime.fromtimestamp(os.path.getmtime(expanded), tz=timezone.utc)
|
||||
if mtime > last_updated:
|
||||
stale += 1
|
||||
|
||||
print(stale)
|
||||
PYEOF
|
||||
)
|
||||
fi
|
||||
fi
|
||||
|
||||
# Write vault-scoped state
|
||||
NOW=$(date +%s)
|
||||
echo "$NOW" > "$STATE_DIR/.last_update"
|
||||
echo "$stale_count" > "$STATE_DIR/.pending_delta"
|
||||
|
||||
if [[ "$stale_count" -gt 0 ]]; then
|
||||
echo "[wiki-daily] $stale_count source(s) have new content since last ingest. State: $STATE_DIR"
|
||||
else
|
||||
echo "[wiki-daily] Wiki is up to date. State: $STATE_DIR"
|
||||
fi
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path('/home/nac/smb/品质部/品质管理资料夹2009.11.3/PPAP资料/0.5-18 APQP资料 2022年8月修正')
|
||||
VAULT = Path('/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault')
|
||||
META = VAULT / '_meta'
|
||||
OUT = VAULT / 'projects/bjd/references'
|
||||
RUN_DATE = '20260611'
|
||||
|
||||
KEY_TERMS = ['规格','norm','USL','LSL','SL','UCL','LCL','CL','判定','judgment','OK','NG','测试目的','测试条件','参照标准','判定标准','测试设备','测试方法','测试结果','报告No','测试日','品名','本厂料号','试验数量']
|
||||
SPECIAL_HINTS = ['总间距','间距','直线度','平面度','插入深度','GAP','卡点','板厚','电镀','PIN孔','内腔长度','变形量','厚度','保持力','可焊性','接触电阻','绝缘','耐压','耐焊热','重复插拔']
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00','Z')
|
||||
|
||||
|
||||
def clean(v: Any) -> str:
|
||||
if v is None:
|
||||
return ''
|
||||
if isinstance(v, float) and math.isnan(v):
|
||||
return ''
|
||||
s=str(v).replace('\n',' ').replace('\r',' ')
|
||||
return re.sub(r'\s+', ' ', s).strip()
|
||||
|
||||
|
||||
def row_values(row) -> list[str]:
|
||||
return [clean(x) for x in row.tolist() if clean(x)]
|
||||
|
||||
|
||||
def read_excel(path: Path):
|
||||
try:
|
||||
return pd.ExcelFile(path)
|
||||
except Exception as e:
|
||||
return None
|
||||
|
||||
|
||||
def find_value_after(vals: list[str], labels: list[str]) -> str:
|
||||
def norm(x: str) -> str:
|
||||
return re.sub(r'\s+', '', x).lower()
|
||||
for i,v in enumerate(vals):
|
||||
nv=norm(v)
|
||||
for lab in labels:
|
||||
nl=norm(lab)
|
||||
exact = nv == nl or nv.startswith(nl + ':') or nv.startswith(nl + '：')
|
||||
contains = nl not in {'cl','sl'} and nl in nv
|
||||
if exact or contains:
|
||||
if i+1 < len(vals):
|
||||
return vals[i+1]
|
||||
m=re.search(re.escape(lab)+r'[:：]?\s*(.+)', v, re.I)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
return ''
|
||||
|
||||
|
||||
def extract_header_fields(df) -> dict[str,str]:
|
||||
vals=[]
|
||||
for _,row in df.head(20).iterrows():
|
||||
vals.extend(row_values(row))
|
||||
return {
|
||||
'part_name': find_value_after(vals, ['产品名称','Part Name','品名']),
|
||||
'item': find_value_after(vals, ['管制项目','Control the project']),
|
||||
'unit': find_value_after(vals, ['测量单位','Unit']),
|
||||
'spec': find_value_after(vals, ['规 格','规格','norm']),
|
||||
'usl': find_value_after(vals, ['上限USL','USL']),
|
||||
'sl': find_value_after(vals, ['中心限SL']),
|
||||
'lsl': find_value_after(vals, ['下限LSL','LSL']) if not re.search(r'\b(?:Max|Min)\b', find_value_after(vals, ['规 格','规格','norm']), re.I) else '',
|
||||
'ucl': find_value_after(vals, ['上限UCL','UCL']),
|
||||
'cl': find_value_after(vals, ['中心限CL']),
|
||||
'lcl': find_value_after(vals, ['下限LCL','LCL']),
|
||||
'report_no': find_value_after(vals, ['报告No']),
|
||||
'test_date': find_value_after(vals, ['测试日']),
|
||||
'bjd_pn': find_value_after(vals, ['本厂料号']),
|
||||
'sample_qty': find_value_after(vals, ['试验数量']),
|
||||
}
|
||||
|
||||
|
||||
def extract_judgement(df) -> str:
|
||||
for _,row in df.iterrows():
|
||||
vals=row_values(row)
|
||||
if not vals:
|
||||
continue
|
||||
line=' | '.join(vals)
|
||||
if re.search(r'判定|judgment', line, re.I):
|
||||
# prefer OK/合格 near end
|
||||
for v in reversed(vals):
|
||||
if re.search(r'合格|OK|NG|不合格', v, re.I):
|
||||
return v
|
||||
return line[:200]
|
||||
return ''
|
||||
|
||||
|
||||
def extract_key_lines(df, max_lines=12) -> list[str]:
|
||||
lines=[]
|
||||
for _,row in df.iterrows():
|
||||
vals=row_values(row)
|
||||
if not vals:
|
||||
continue
|
||||
line=' | '.join(vals)
|
||||
if any(k.lower() in line.lower() for k in KEY_TERMS) or any(k in line for k in SPECIAL_HINTS):
|
||||
if line not in lines:
|
||||
lines.append(line[:600])
|
||||
if len(lines)>=max_lines:
|
||||
break
|
||||
return lines
|
||||
|
||||
|
||||
def extract_numeric_result_rows(df, max_rows=8) -> list[str]:
|
||||
out=[]
|
||||
for _,row in df.iterrows():
|
||||
vals=row_values(row)
|
||||
if not vals:
|
||||
continue
|
||||
nums=[]
|
||||
for v in vals:
|
||||
if re.fullmatch(r'-?\d+(?:\.\d+)?', v):
|
||||
nums.append(v)
|
||||
if len(nums) >= 3 and re.search(r'S-\d|平均|AVG|Max|Min|Ave|OK|合格', ' '.join(vals), re.I):
|
||||
out.append(' | '.join(vals)[:500])
|
||||
if len(out)>=max_rows:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def classify_file(path: Path) -> str:
|
||||
s=path.as_posix()
|
||||
if 'PPK' in s or '过程能力' in s or 'CMK' in s:
|
||||
return 'process_capability'
|
||||
if 'MSA' in s or 'GRR' in s or 'GR&R' in s or '测量系统' in s:
|
||||
return 'msa'
|
||||
if '可靠性' in s or '测试' in s or any(x in path.name for x in ['保持力','可焊性','接触电阻','绝缘','耐压','耐焊热','重复插拔']):
|
||||
return 'reliability'
|
||||
if '全尺寸' in s or '尺寸' in s:
|
||||
return 'dimensional'
|
||||
if '试验测试工作计划' in s or 'DV' in s or 'PV' in s:
|
||||
return 'test_plan'
|
||||
return 'other'
|
||||
|
||||
|
||||
def extract_records():
|
||||
records=[]
|
||||
paths=[p for p in ROOT.rglob('*') if p.is_file() and p.suffix.lower() in {'.xls','.xlsx'}]
|
||||
for path in sorted(paths):
|
||||
kind=classify_file(path)
|
||||
if kind == 'other':
|
||||
continue
|
||||
xls=read_excel(path)
|
||||
if xls is None:
|
||||
continue
|
||||
for sheet in xls.sheet_names[:12]:
|
||||
try:
|
||||
df=pd.read_excel(path, sheet_name=sheet, header=None)
|
||||
except Exception:
|
||||
continue
|
||||
if df.empty:
|
||||
continue
|
||||
fields=extract_header_fields(df)
|
||||
key_lines=extract_key_lines(df)
|
||||
result_rows=extract_numeric_result_rows(df)
|
||||
judgement=extract_judgement(df)
|
||||
text=' '.join([path.name, sheet] + list(fields.values()) + key_lines + result_rows + [judgement])
|
||||
if not any(x in text for x in SPECIAL_HINTS) and kind not in {'process_capability','msa'}:
|
||||
continue
|
||||
rec={
|
||||
'relative_path': path.relative_to(ROOT).as_posix(),
|
||||
'filename': path.name,
|
||||
'sheet': sheet,
|
||||
'kind': kind,
|
||||
'fields': {k:v for k,v in fields.items() if v},
|
||||
'judgement': judgement,
|
||||
'key_lines': key_lines,
|
||||
'result_rows': result_rows,
|
||||
}
|
||||
records.append(rec)
|
||||
return records
|
||||
|
||||
|
||||
def summarize(records):
|
||||
special=[]
|
||||
seen=set()
|
||||
for r in records:
|
||||
item=r['fields'].get('item') or ''
|
||||
if not item:
|
||||
# from filename
|
||||
name=r['filename']
|
||||
for h in SPECIAL_HINTS:
|
||||
if h in name:
|
||||
item=name.rsplit('.',1)[0]
|
||||
break
|
||||
if item and item not in seen:
|
||||
seen.add(item)
|
||||
special.append(item)
|
||||
return special
|
||||
|
||||
|
||||
def esc(s):
|
||||
return str(s).replace('|','\\|').replace('\n',' ')
|
||||
|
||||
|
||||
def table(records, kind, limit=80):
|
||||
rows=[r for r in records if r['kind']==kind]
|
||||
lines=['| 项目/表 | 规格/标准 | 判定 | 关键数据/说明 | 来源 |','|---|---|---|---|---|']
|
||||
for r in rows[:limit]:
|
||||
f=r['fields']
|
||||
item=f.get('item') or r['sheet']
|
||||
spec=f.get('spec') or ''
|
||||
if f.get('usl') or f.get('lsl'):
|
||||
spec='; '.join(x for x in [spec, f"USL {f.get('usl','')}", f"SL {f.get('sl','')}", f"LSL {f.get('lsl','')}"] if x.strip())
|
||||
key='; '.join((r['result_rows'][:2] or r['key_lines'][:2]))
|
||||
lines.append(f"| {esc(item)} | {esc(spec)} | {esc(r.get('judgement') or '')} | {esc(key[:500])} | {esc(r['relative_path']+' :: '+r['sheet'])} |")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def write_outputs(records):
|
||||
ts=now_iso()
|
||||
META.mkdir(parents=True, exist_ok=True)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
jsonp=META/f'bjd-ppap-0.5-18-deep-extraction-{RUN_DATE}.json'
|
||||
jsonp.write_text(json.dumps(records, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
page=OUT/f'bjd-ppap-0.5-18-deep-extraction-{RUN_DATE}.md'
|
||||
special=summarize(records)
|
||||
counts=defaultdict(int)
|
||||
for r in records:
|
||||
counts[r['kind']]+=1
|
||||
lines=[
|
||||
'---','title: BJD 0.5-18 PPAP/APQP 深度解析','category: reference','project: bjd',
|
||||
'tags: [bjd, quality, ppap, apqp, 0.5-18, msa, ppk, reliability, visibility/internal]',
|
||||
'visibility: internal','business_domains: [quality, engineering, products]','data_classification: technical-confidential',f'updated: {ts}','---','',
|
||||
'# BJD 0.5-18 PPAP/APQP 深度解析','',
|
||||
f'来源目录：`{ROOT}`。本次用 pandas/openpyxl/xlrd 对 0.5-18 相关 `.xls/.xlsx` 做深度解析，抽取 `{len(records)}` 个工作表记录。记录分布：' + '；'.join(f'{k}={v}' for k,v in sorted(counts.items())) + '。','',
|
||||
'## 可识别的特殊特性/重点管控项目','',
|
||||
]
|
||||
for s in special[:80]:
|
||||
lines.append(f'- {s}')
|
||||
lines += ['', '## 过程能力 / PPK / CMK', '', table(records,'process_capability', limit=80), '', '## MSA / GR&R / 测量系统', '', table(records,'msa', limit=80), '', '## 可靠性 / DV / PV / 样品阶段测试', '', table(records,'reliability', limit=120), '', '## 全尺寸 / 尺寸测量', '', table(records,'dimensional', limit=80), '', '## 使用注意', '', '- 旧 Excel 中有些能力指数位于图表/公式区域，若文件未保存缓存，解析器可能只能抽到规格、控制限和判定，不能保证每个 PPK/GR&R 数值都完整。', '- 对客户正式回复时，应以原文件截图/导出 PDF 复核版本、签核、日期和适用料号。', f'- 结构化 JSON：`_meta/{jsonp.name}`']
|
||||
page.write_text('\n'.join(lines)+'\n', encoding='utf-8')
|
||||
update_meta(ts, page, jsonp, len(records))
|
||||
return page, jsonp
|
||||
|
||||
|
||||
def update_meta(ts, page, jsonp, n):
|
||||
idx=VAULT/'index.md'
|
||||
if idx.exists():
|
||||
text=idx.read_text(encoding='utf-8')
|
||||
rel=page.relative_to(VAULT).with_suffix('').as_posix()
|
||||
if rel not in text:
|
||||
text=text.replace('## 综合分析\n', f'## 综合分析\n- [[{rel}]] - 0.5-18 PPAP/APQP 深度解析，包含特殊特性、PPK/MSA/可靠性/全尺寸抽取。 ( #bjd #quality #ppap #0.5-18 )\n', 1)
|
||||
text=re.sub(r'最后更新：[^*]+', f'最后更新：{ts}', text)
|
||||
idx.write_text(text, encoding='utf-8')
|
||||
with (VAULT/'log.md').open('a', encoding='utf-8') as f:
|
||||
f.write(f'\n- [{ts}] DATA_INGEST source="{ROOT}" project="bjd" mode="ppap-0.5-18-deep-extraction" records={n} page="{page.relative_to(VAULT).as_posix()}" data="{jsonp.relative_to(VAULT).as_posix()}" bot_visible=false\n')
|
||||
mp=VAULT/'.manifest.json'
|
||||
data={}
|
||||
if mp.exists():
|
||||
try: data=json.loads(mp.read_text(encoding='utf-8'))
|
||||
except Exception: data={}
|
||||
data.setdefault('sources',{})[f'filesystem:ppap-0.5-18-deep-extraction-{RUN_DATE}']={
|
||||
'ingested_at':ts,'source_path':str(ROOT),'source_type':'filesystem_directory','doc_type':'ppap_deep_extraction','project':'bjd','records':n,
|
||||
'pages_created':[page.relative_to(VAULT).as_posix()],'data_paths':[jsonp.relative_to(VAULT).as_posix()],'bot_visible':False,
|
||||
'parser':'pandas/openpyxl/xlrd in /tmp/ppap-parse-venv'
|
||||
}
|
||||
mp.write_text(json.dumps(data, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
records=extract_records()
|
||||
page,jsonp=write_outputs(records)
|
||||
print(json.dumps({'records':len(records),'page':str(page),'json':str(jsonp)}, ensure_ascii=False))
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
ROOT = Path('/home/nac/smb/品质部/品质管理资料夹2009.11.3/PPAP资料')
|
||||
VAULT = Path('/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault')
|
||||
META = VAULT / '_meta'
|
||||
OUT = VAULT / 'projects/bjd/references'
|
||||
RUN_DATE = '20260611'
|
||||
MAX_TEXT = 20000
|
||||
MAX_FILES = int(os.environ.get('PPAP_DEEP_MAX_FILES', '0')) or None
|
||||
|
||||
QUALITY_TERMS = [
|
||||
'PPAP','APQP','PSW','零件提交保证书','全尺寸','寸法','MSA','GR&R','GRR','CPK','PPK','CMK',
|
||||
'可靠性','DV','PV','接触电阻','绝缘','耐压','保持力','可焊性','耐焊热','重复插拔','耐湿','盐雾',
|
||||
'PFMEA','DFMEA','控制计划','过程流程','特殊特性','规格书','SGS','RoHS','Halogen','判定标准','测试结果','合格','OK',
|
||||
]
|
||||
SERIES_RE = re.compile(r'(?<!\d)(?:0\.3|0\.5|0\.8|1\.0|1\.25|1\.8|2\.0|2\.2|3\.5|5\.0)-\d+(?:B)?|28\+4P|10PIN|18B966|VPKSJF|VPHNMF|VPDNNMF', re.I)
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00','Z')
|
||||
|
||||
|
||||
def clean(s: str) -> str:
|
||||
s = (s or '').replace('\x00', ' ')
|
||||
s = re.sub(r'[\t\r\f\v]+', ' ', s)
|
||||
s = re.sub(r'\n{3,}', '\n\n', s)
|
||||
s = re.sub(r' {2,}', ' ', s)
|
||||
return s.strip()
|
||||
|
||||
|
||||
def run(cmd: list[str], timeout=30) -> str:
|
||||
try:
|
||||
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=False)
|
||||
data = p.stdout or b''
|
||||
if not data and p.stderr:
|
||||
data = p.stderr[:500]
|
||||
return data.decode('utf-8', errors='ignore')
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def extract_pdf(path: Path) -> str:
|
||||
return run(['pdftotext', '-layout', str(path), '-'], timeout=45)
|
||||
|
||||
|
||||
def extract_doc(path: Path) -> str:
|
||||
txt = run(['antiword', str(path)], timeout=30)
|
||||
if len(txt.strip()) < 50:
|
||||
txt = run(['catdoc', str(path)], timeout=30)
|
||||
return txt
|
||||
|
||||
|
||||
def extract_unrtf(path: Path) -> str:
|
||||
return run(['unrtf', '--text', str(path)], timeout=30)
|
||||
|
||||
|
||||
def extract_xlsx(path: Path) -> str:
|
||||
# Dependency-free OOXML shared strings/cell extraction; handles xlsx without openpyxl.
|
||||
try:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
names=set(z.namelist())
|
||||
shared=[]
|
||||
if 'xl/sharedStrings.xml' in names:
|
||||
root=ET.fromstring(z.read('xl/sharedStrings.xml'))
|
||||
ns={'a':'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
||||
for si in root.findall('.//a:si', ns):
|
||||
parts=[t.text or '' for t in si.findall('.//a:t', ns)]
|
||||
if parts: shared.append(''.join(parts))
|
||||
lines=[]
|
||||
for name in sorted(n for n in names if n.startswith('xl/worksheets/sheet') and n.endswith('.xml'))[:8]:
|
||||
root=ET.fromstring(z.read(name))
|
||||
ns={'a':'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
||||
vals=[]
|
||||
for c in root.findall('.//a:c', ns)[:1000]:
|
||||
v=c.find('a:v', ns)
|
||||
if v is None or v.text is None: continue
|
||||
val=v.text
|
||||
if c.get('t') == 's':
|
||||
try: val=shared[int(val)]
|
||||
except Exception: pass
|
||||
vals.append(val)
|
||||
if vals:
|
||||
lines.append(name + '\n' + ' '.join(vals))
|
||||
return '\n'.join(lines)
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def extract_xls_with_ssconvert(path: Path) -> str:
|
||||
# Convert each workbook to csv in a temp dir. ssconvert may output one csv or multiple files depending workbook.
|
||||
with tempfile.TemporaryDirectory(prefix='ppap-xls-') as td:
|
||||
out = Path(td) / 'out.csv'
|
||||
txt = run(['ssconvert', '--export-type=Gnumeric_stf:stf_csv', str(path), str(out)], timeout=60)
|
||||
chunks=[]
|
||||
for f in Path(td).glob('*'):
|
||||
if f.is_file():
|
||||
try:
|
||||
chunks.append(f.read_text(encoding='utf-8', errors='ignore'))
|
||||
except Exception:
|
||||
pass
|
||||
if chunks:
|
||||
return '\n'.join(chunks)
|
||||
return txt
|
||||
|
||||
|
||||
def extract_libreoffice_text(path: Path) -> str:
|
||||
with tempfile.TemporaryDirectory(prefix='ppap-lo-') as td:
|
||||
run(['libreoffice', '--headless', '--convert-to', 'txt:Text', '--outdir', td, str(path)], timeout=90)
|
||||
chunks=[]
|
||||
for f in Path(td).glob('*.txt'):
|
||||
try: chunks.append(f.read_text(encoding='utf-8', errors='ignore'))
|
||||
except Exception: pass
|
||||
return '\n'.join(chunks)
|
||||
|
||||
|
||||
def extract_text(path: Path) -> tuple[str, str]:
|
||||
ext=path.suffix.lower()
|
||||
text=''; method=''
|
||||
if ext == '.pdf':
|
||||
text=extract_pdf(path); method='pdftotext'
|
||||
elif ext == '.doc':
|
||||
text=extract_doc(path); method='antiword/catdoc'
|
||||
if len(clean(text)) < 50:
|
||||
lo=extract_libreoffice_text(path)
|
||||
if len(clean(lo)) > len(clean(text)):
|
||||
text=lo; method='libreoffice-txt'
|
||||
elif ext == '.rtf':
|
||||
text=extract_unrtf(path); method='unrtf'
|
||||
elif ext == '.xlsx':
|
||||
text=extract_xlsx(path); method='xlsx-xml'
|
||||
elif ext == '.xls':
|
||||
text=extract_xls_with_ssconvert(path); method='ssconvert-csv'
|
||||
if len(clean(text)) < 50:
|
||||
lo=extract_libreoffice_text(path)
|
||||
if len(clean(lo)) > len(clean(text)):
|
||||
text=lo; method='libreoffice-txt'
|
||||
elif ext in {'.txt','.log','.csv'}:
|
||||
try: text=path.read_text(encoding='utf-8', errors='ignore'); method='plain'
|
||||
except Exception: text=''
|
||||
elif ext == '.ppt':
|
||||
text=extract_libreoffice_text(path); method='libreoffice-txt'
|
||||
return clean(text)[:MAX_TEXT], method
|
||||
|
||||
|
||||
def snippets(text: str, max_snips=8) -> list[str]:
|
||||
out=[]
|
||||
for term in QUALITY_TERMS:
|
||||
for m in re.finditer(re.escape(term), text, re.I):
|
||||
start=max(0, m.start()-160); end=min(len(text), m.end()+320)
|
||||
sn=clean(text[start:end])[:600]
|
||||
if sn and sn not in out:
|
||||
out.append(sn)
|
||||
if len(out) >= max_snips:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
def classify(text: str, path: Path) -> list[str]:
|
||||
hay=(path.as_posix()+'\n'+text)
|
||||
cats=[]
|
||||
patterns={
|
||||
'ppap_submission': r'PPAP提交清单|PPAP文件清单',
|
||||
'psw': r'零件提交保证书|PSW',
|
||||
'dimensional': r'全尺寸|寸法|尺寸测量|样品寸法',
|
||||
'msa': r'MSA|GR&R|GRR|测量系统|量测系统',
|
||||
'capability': r'CPK|PPK|CMK|过程能力|设备能力',
|
||||
'reliability': r'可靠性|DV|PV|接触电阻|绝缘|耐压|保持力|可焊性|耐焊热|重复插拔|耐湿|盐雾',
|
||||
'pfmea_dfmea': r'PFMEA|DFMEA|失效模式',
|
||||
'control_plan': r'控制计划|Control Plan',
|
||||
'process_flow': r'过程流程|工艺流程|流程图',
|
||||
'material': r'SGS|RoHS|Halogen|C5210|C5191|C2680|LCP|PA9T',
|
||||
'special_characteristics': r'特殊特性|关键特性|重要特性|Special Characteristics',
|
||||
}
|
||||
for k,pat in patterns.items():
|
||||
if re.search(pat, hay, re.I): cats.append(k)
|
||||
return cats or ['other']
|
||||
|
||||
|
||||
def main():
|
||||
records=[]
|
||||
files=[]
|
||||
for p in ROOT.rglob('*'):
|
||||
if p.is_file() and p.suffix.lower() in {'.pdf','.doc','.rtf','.xls','.xlsx','.txt','.log','.csv','.ppt'}:
|
||||
files.append(p)
|
||||
if MAX_FILES:
|
||||
files=files[:MAX_FILES]
|
||||
for i,p in enumerate(files,1):
|
||||
text, method = extract_text(p)
|
||||
rel=p.relative_to(ROOT).as_posix()
|
||||
cats=classify(text, p)
|
||||
rec={
|
||||
'relative_path': rel,
|
||||
'path': str(p),
|
||||
'top_package': rel.split('/')[0],
|
||||
'filename': p.name,
|
||||
'extension': p.suffix.lower().lstrip('.') or 'no_ext',
|
||||
'method': method,
|
||||
'text_length': len(text),
|
||||
'categories': cats,
|
||||
'series': sorted(set(SERIES_RE.findall(rel+'\n'+text))),
|
||||
'snippets': snippets(text),
|
||||
'text_sample': text[:1500],
|
||||
}
|
||||
records.append(rec)
|
||||
if i % 100 == 0:
|
||||
print(f'processed {i}/{len(files)}', flush=True)
|
||||
META.mkdir(parents=True, exist_ok=True)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
jsonl=META/f'bjd-ppap-apqp-deep-text-index-{RUN_DATE}.jsonl'
|
||||
with jsonl.open('w',encoding='utf-8') as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r,ensure_ascii=False)+'\n')
|
||||
# Summary CSV
|
||||
csvp=META/f'bjd-ppap-apqp-deep-text-index-{RUN_DATE}.csv'
|
||||
with csvp.open('w',encoding='utf-8-sig',newline='') as f:
|
||||
w=csv.DictWriter(f,fieldnames=['relative_path','top_package','extension','method','text_length','categories','series','snippet_count'])
|
||||
w.writeheader()
|
||||
for r in records:
|
||||
w.writerow({
|
||||
'relative_path':r['relative_path'], 'top_package':r['top_package'], 'extension':r['extension'], 'method':r['method'],
|
||||
'text_length':r['text_length'], 'categories':'; '.join(r['categories']), 'series':'; '.join(r['series']), 'snippet_count':len(r['snippets'])
|
||||
})
|
||||
write_page(records,jsonl,csvp)
|
||||
print(json.dumps({'files':len(records),'jsonl':str(jsonl),'csv':str(csvp)},ensure_ascii=False))
|
||||
|
||||
|
||||
def esc(s): return str(s).replace('|','\\|').replace('\n',' ')
|
||||
|
||||
|
||||
def write_page(records, jsonl, csvp):
|
||||
ts=now_iso()
|
||||
by_ext=Counter(r['extension'] for r in records)
|
||||
by_method=Counter(r['method'] or 'none' for r in records)
|
||||
by_cat=Counter(c for r in records for c in r['categories'])
|
||||
by_pkg=defaultdict(list)
|
||||
for r in records:
|
||||
by_pkg[r['top_package']].append(r)
|
||||
pkg_rows=[]
|
||||
for pkg,rs in by_pkg.items():
|
||||
pkg_rows.append((pkg,len(rs),sum(1 for r in rs if r['text_length']>50),Counter(c for r in rs for c in r['categories']).most_common(6)))
|
||||
pkg_rows.sort(key=lambda x:(-x[1],x[0]))
|
||||
page=OUT/f'bjd-ppap-apqp-deep-text-index-{RUN_DATE}.md'
|
||||
lines=['---','title: BJD PPAP/APQP 深度文本索引','category: reference','project: bjd','tags: [bjd, quality, ppap, apqp, deep-extraction, visibility/internal]','visibility: internal','data_classification: technical-confidential',f'updated: {ts}','---','', '# BJD PPAP/APQP 深度文本索引','', f'本页基于已安装工具 `pdftotext`、`antiword/catdoc`、`ssconvert`、`libreoffice` 对 PPAP 目录做二次文本抽取。共处理 `{len(records)}` 个可解析文件。', '', '## 抽取覆盖','', '- 文件类型：' + '；'.join(f'{k}={v}' for k,v in by_ext.most_common()), '- 抽取方法：' + '；'.join(f'{k}={v}' for k,v in by_method.most_common()), '- 质量类别：' + '；'.join(f'{k}={v}' for k,v in by_cat.most_common()), '', '## 资料包覆盖', '', '| 资料包 | 文件数 | 抽到文本文件数 | 主要质量类别 |', '|---|---:|---:|---|']
|
||||
for pkg,n,txtn,cats in pkg_rows[:80]:
|
||||
lines.append(f'| {esc(pkg)} | {n} | {txtn} | {esc("；".join(f"{k}={v}" for k,v in cats))} |')
|
||||
# selected snippets
|
||||
interesting=[r for r in records if r['snippets'] and any(c in r['categories'] for c in ['reliability','capability','msa','pfmea_dfmea','control_plan','special_characteristics'])]
|
||||
lines += ['', '## 代表性证据片段', '', '| 来源 | 类别 | 片段 |', '|---|---|---|']
|
||||
for r in interesting[:80]:
|
||||
lines.append(f'| {esc(r["relative_path"])} | {esc("; ".join(r["categories"]))} | {esc(r["snippets"][0][:500])} |')
|
||||
lines += ['', '## 数据文件', '', f'- JSONL：`_meta/{jsonl.name}`', f'- CSV：`_meta/{csvp.name}`', '', '## 使用注意', '', '- 这是文本抽取索引，不替代原始 PPAP 文件。正式对外提交仍需复核原文件签核、版本、日期和客户适用性。', '- 扫描版 PDF 或图片型图纸如果没有文本层，仍需要 OCR 才能抽出正文。']
|
||||
page.write_text('\n'.join(lines)+'\n',encoding='utf-8')
|
||||
idx=VAULT/'index.md'
|
||||
if idx.exists():
|
||||
text=idx.read_text(encoding='utf-8')
|
||||
rel=page.relative_to(VAULT).with_suffix('').as_posix()
|
||||
if rel not in text:
|
||||
text=text.replace('## 综合分析\n', f'## 综合分析\n- [[{rel}]] - PPAP/APQP 全目录深度文本索引，包含 PDF/DOC/XLS 二次抽取。 ( #bjd #quality #ppap #apqp )\n',1)
|
||||
text=re.sub(r'最后更新：[^*]+', f'最后更新：{ts}', text)
|
||||
idx.write_text(text,encoding='utf-8')
|
||||
with (VAULT/'log.md').open('a',encoding='utf-8') as f:
|
||||
f.write(f'\n- [{ts}] DATA_INGEST source="{ROOT}" project="bjd" mode="ppap-apqp-deep-text-index" files={len(records)} page="{page.relative_to(VAULT).as_posix()}" jsonl="{jsonl.relative_to(VAULT).as_posix()}" csv="{csvp.relative_to(VAULT).as_posix()}" bot_visible=false\n')
|
||||
mp=VAULT/'.manifest.json'
|
||||
data={}
|
||||
if mp.exists():
|
||||
try: data=json.loads(mp.read_text(encoding='utf-8'))
|
||||
except Exception: data={}
|
||||
data.setdefault('sources',{})[f'filesystem:ppap-apqp-deep-text-index-{RUN_DATE}']={'ingested_at':ts,'source_path':str(ROOT),'source_type':'filesystem_directory','doc_type':'ppap_apqp_deep_text_index','project':'bjd','files':len(records),'pages_created':[page.relative_to(VAULT).as_posix()],'data_paths':[jsonl.relative_to(VAULT).as_posix(),csvp.relative_to(VAULT).as_posix()],'bot_visible':False,'tools':'pdftotext, antiword/catdoc, ssconvert, libreoffice'}
|
||||
mp.write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+351
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Deploy an obsidian-wiki Linux migration bundle on a headless server.
|
||||
#
|
||||
# Run this from the extracted bundle root:
|
||||
# sudo bash deploy-linux-server.sh --install-dir /opt/obsidian-wiki --vault-dir /srv/bjd-vault
|
||||
|
||||
set -Eeuo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BUNDLE_DIR="$SCRIPT_DIR"
|
||||
|
||||
INSTALL_DIR="${INSTALL_DIR:-/opt/obsidian-wiki}"
|
||||
VAULT_DIR="${VAULT_DIR:-/srv/bjd-vault}"
|
||||
BOT_VIEW_DIR="${BOT_VIEW_DIR:-/srv/bjd-bot-views}"
|
||||
QMD_WIKI_COLLECTION="${QMD_WIKI_COLLECTION:-bjd-wiki}"
|
||||
QMD_CLI_SEARCH_MODE="${QMD_CLI_SEARCH_MODE:-balanced}"
|
||||
PROFILE_LIST="${PROFILE_LIST:-all}"
|
||||
SOURCE_FILE_PROFILES="${SOURCE_FILE_PROFILES:-quality-bot,engineering-bot}"
|
||||
CONFIG_FILE="${CONFIG_FILE:-$HOME/.obsidian-wiki/config}"
|
||||
RUN_DELETE=0
|
||||
SKIP_FILE_INDEX=0
|
||||
SKIP_BOT_VIEWS=0
|
||||
SKIP_QMD=0
|
||||
DRY_RUN=0
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
bash deploy-linux-server.sh [options]
|
||||
|
||||
Options:
|
||||
--install-dir DIR Destination for obsidian-wiki repo. Default: /opt/obsidian-wiki
|
||||
--vault-dir DIR Destination for mother vault. Default: /srv/bjd-vault
|
||||
--bot-view-dir DIR Destination for derived bot views. Default: /srv/bjd-bot-views
|
||||
--qmd-collection NAME Mother vault QMD collection name. Default: bjd-wiki
|
||||
--profiles LIST Comma-separated bot profiles to export, or "all". Default: all
|
||||
--source-file-profiles LIST Comma-separated profiles exported with --include-files.
|
||||
Default: quality-bot,engineering-bot
|
||||
--config-file PATH Config file to write. Default: ~/.obsidian-wiki/config
|
||||
--delete Use rsync --delete when copying repo/vault.
|
||||
--skip-file-index Do not rebuild _meta/file-assets.sqlite.
|
||||
--skip-bot-views Do not export bot views.
|
||||
--skip-qmd Do not create/update QMD collections.
|
||||
--dry-run Print planned actions only.
|
||||
-h, --help Show this help.
|
||||
|
||||
Environment variables with the same names as the defaults may also be used:
|
||||
INSTALL_DIR, VAULT_DIR, BOT_VIEW_DIR, QMD_WIKI_COLLECTION,
|
||||
QMD_CLI_SEARCH_MODE, PROFILE_LIST, SOURCE_FILE_PROFILES, CONFIG_FILE
|
||||
EOF
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
info() {
|
||||
echo "[INFO] $*"
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo "[WARN] $*" >&2
|
||||
}
|
||||
|
||||
expand_path() {
|
||||
local path="$1"
|
||||
case "$path" in
|
||||
"~") printf '%s' "$HOME" ;;
|
||||
"~/"*) printf '%s/%s' "$HOME" "${path#~/}" ;;
|
||||
*) printf '%s' "$path" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
trim() {
|
||||
local value="$1"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
csv_contains() {
|
||||
local needle="$1"
|
||||
local csv="$2"
|
||||
local item
|
||||
IFS=',' read -ra items <<< "$csv"
|
||||
for item in "${items[@]}"; do
|
||||
item="$(trim "$item")"
|
||||
[[ "$item" == "$needle" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
local cmd="$1"
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--install-dir)
|
||||
[[ $# -ge 2 ]] || die "--install-dir requires a directory"
|
||||
INSTALL_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--vault-dir)
|
||||
[[ $# -ge 2 ]] || die "--vault-dir requires a directory"
|
||||
VAULT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bot-view-dir)
|
||||
[[ $# -ge 2 ]] || die "--bot-view-dir requires a directory"
|
||||
BOT_VIEW_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--qmd-collection)
|
||||
[[ $# -ge 2 ]] || die "--qmd-collection requires a name"
|
||||
QMD_WIKI_COLLECTION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--profiles)
|
||||
[[ $# -ge 2 ]] || die "--profiles requires a comma-separated list or all"
|
||||
PROFILE_LIST="$2"
|
||||
shift 2
|
||||
;;
|
||||
--source-file-profiles)
|
||||
[[ $# -ge 2 ]] || die "--source-file-profiles requires a comma-separated list"
|
||||
SOURCE_FILE_PROFILES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--config-file)
|
||||
[[ $# -ge 2 ]] || die "--config-file requires a path"
|
||||
CONFIG_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--delete)
|
||||
RUN_DELETE=1
|
||||
shift
|
||||
;;
|
||||
--skip-file-index)
|
||||
SKIP_FILE_INDEX=1
|
||||
shift
|
||||
;;
|
||||
--skip-bot-views)
|
||||
SKIP_BOT_VIEWS=1
|
||||
shift
|
||||
;;
|
||||
--skip-qmd)
|
||||
SKIP_QMD=1
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
INSTALL_DIR="$(expand_path "$INSTALL_DIR")"
|
||||
VAULT_DIR="$(expand_path "$VAULT_DIR")"
|
||||
BOT_VIEW_DIR="$(expand_path "$BOT_VIEW_DIR")"
|
||||
CONFIG_FILE="$(expand_path "$CONFIG_FILE")"
|
||||
|
||||
BUNDLE_REPO="$BUNDLE_DIR/obsidian-wiki"
|
||||
BUNDLE_VAULT="$BUNDLE_DIR/vault"
|
||||
|
||||
[[ -d "$BUNDLE_REPO/obsidian_wiki" ]] || die "bundle repo missing: $BUNDLE_REPO"
|
||||
[[ -d "$BUNDLE_VAULT" ]] || die "bundle vault missing: $BUNDLE_VAULT"
|
||||
|
||||
require_command python3
|
||||
require_command rsync
|
||||
|
||||
info "bundle: $BUNDLE_DIR"
|
||||
info "install dir: $INSTALL_DIR"
|
||||
info "vault dir: $VAULT_DIR"
|
||||
info "bot views: $BOT_VIEW_DIR"
|
||||
info "config file: $CONFIG_FILE"
|
||||
info "profiles: $PROFILE_LIST"
|
||||
info "source files: $SOURCE_FILE_PROFILES"
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
info "dry run complete; no files copied."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$INSTALL_DIR" "$VAULT_DIR" "$BOT_VIEW_DIR" "$(dirname "$CONFIG_FILE")"
|
||||
|
||||
RSYNC_ARGS=(-a)
|
||||
if [[ "$RUN_DELETE" -eq 1 ]]; then
|
||||
RSYNC_ARGS+=(--delete)
|
||||
fi
|
||||
|
||||
info "copying obsidian-wiki repo..."
|
||||
rsync "${RSYNC_ARGS[@]}" "$BUNDLE_REPO/" "$INSTALL_DIR/"
|
||||
|
||||
info "copying mother vault..."
|
||||
rsync "${RSYNC_ARGS[@]}" "$BUNDLE_VAULT/" "$VAULT_DIR/"
|
||||
|
||||
cat > "$INSTALL_DIR/.env" <<EOF
|
||||
OBSIDIAN_VAULT_PATH="$VAULT_DIR"
|
||||
OBSIDIAN_WIKI_REPO="$INSTALL_DIR"
|
||||
WIKI_ARCHIVE_SOURCES=true
|
||||
OBSIDIAN_SOURCE_ARCHIVE_DIR=_sources/archive
|
||||
WIKI_SOURCE_CATALOG=true
|
||||
OBSIDIAN_SOURCE_CATALOG_DIR=_sources/catalog
|
||||
OBSIDIAN_LINK_FORMAT=wikilink
|
||||
OBSIDIAN_FILE_ASSET_DB=_meta/file-assets.sqlite
|
||||
QMD_WIKI_COLLECTION=$QMD_WIKI_COLLECTION
|
||||
QMD_TRANSPORT=cli
|
||||
QMD_CLI_SEARCH_MODE=$QMD_CLI_SEARCH_MODE
|
||||
QMD_CLI=qmd
|
||||
EOF
|
||||
|
||||
cat > "$CONFIG_FILE" <<EOF
|
||||
OBSIDIAN_VAULT_PATH="$VAULT_DIR"
|
||||
OBSIDIAN_WIKI_REPO="$INSTALL_DIR"
|
||||
OBSIDIAN_FILE_ASSET_DB="_meta/file-assets.sqlite"
|
||||
QMD_WIKI_COLLECTION=$QMD_WIKI_COLLECTION
|
||||
QMD_TRANSPORT=cli
|
||||
QMD_CLI_SEARCH_MODE=$QMD_CLI_SEARCH_MODE
|
||||
QMD_CLI=qmd
|
||||
EOF
|
||||
|
||||
mkdir -p "$INSTALL_DIR/bin"
|
||||
cat > "$INSTALL_DIR/bin/obsidian-wiki-local" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
INSTALL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
export PYTHONPATH="$INSTALL_DIR${PYTHONPATH:+:$PYTHONPATH}"
|
||||
exec python3 -m obsidian_wiki "$@"
|
||||
EOF
|
||||
chmod +x "$INSTALL_DIR/bin/obsidian-wiki-local"
|
||||
|
||||
ow() {
|
||||
PYTHONPATH="$INSTALL_DIR${PYTHONPATH:+:$PYTHONPATH}" python3 -m obsidian_wiki "$@"
|
||||
}
|
||||
|
||||
if [[ "$SKIP_FILE_INDEX" -eq 0 ]]; then
|
||||
info "rebuilding file asset SQLite index..."
|
||||
ow file-index rebuild --vault "$VAULT_DIR"
|
||||
fi
|
||||
|
||||
PROFILES=()
|
||||
if [[ "$PROFILE_LIST" == "all" ]]; then
|
||||
if PROFILE_OUTPUT="$(ow bot-view list --vault "$VAULT_DIR" 2>&1)"; then
|
||||
while IFS= read -r profile; do
|
||||
[[ -n "$profile" ]] && PROFILES+=("$profile")
|
||||
done <<< "$PROFILE_OUTPUT"
|
||||
else
|
||||
warn "bot profiles are not available; skipping bot-view audit/export."
|
||||
[[ -n "$PROFILE_OUTPUT" ]] && warn "$PROFILE_OUTPUT"
|
||||
SKIP_BOT_VIEWS=1
|
||||
fi
|
||||
else
|
||||
IFS=',' read -ra raw_profiles <<< "$PROFILE_LIST"
|
||||
for profile in "${raw_profiles[@]}"; do
|
||||
profile="$(trim "$profile")"
|
||||
[[ -n "$profile" ]] && PROFILES+=("$profile")
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "${#PROFILES[@]}" -gt 0 ]]; then
|
||||
info "auditing bot-view ACLs..."
|
||||
ow bot-view audit --vault "$VAULT_DIR" || warn "bot-view audit failed; continuing deployment."
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_BOT_VIEWS" -eq 0 ]]; then
|
||||
info "exporting bot views..."
|
||||
for profile in "${PROFILES[@]}"; do
|
||||
extra_args=()
|
||||
if csv_contains "$profile" "$SOURCE_FILE_PROFILES"; then
|
||||
extra_args+=(--include-files)
|
||||
fi
|
||||
ow bot-view export --vault "$VAULT_DIR" "$profile" --out "$BOT_VIEW_DIR" "${extra_args[@]}"
|
||||
done
|
||||
fi
|
||||
|
||||
qmd_collection_exists() {
|
||||
qmd collection show "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
qmd_remove_if_exists() {
|
||||
local name="$1"
|
||||
if qmd_collection_exists "$name"; then
|
||||
qmd collection remove "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
qmd_add_collection_named() {
|
||||
local name="$1"
|
||||
local path="$2"
|
||||
local pattern="${3:-**/*.md}"
|
||||
|
||||
qmd_remove_if_exists "$name"
|
||||
qmd collection add "$path" --name "$name" --mask "$pattern"
|
||||
}
|
||||
|
||||
if [[ "$SKIP_QMD" -eq 0 ]]; then
|
||||
if command -v qmd >/dev/null 2>&1; then
|
||||
info "creating/updating QMD mother-vault collection..."
|
||||
qmd_add_collection_named "$QMD_WIKI_COLLECTION" "$VAULT_DIR" '**/*.md'
|
||||
|
||||
if [[ "$SKIP_BOT_VIEWS" -eq 0 ]]; then
|
||||
for profile in "${PROFILES[@]}"; do
|
||||
bot_wiki="$BOT_VIEW_DIR/$profile/wiki"
|
||||
if [[ -d "$bot_wiki" ]]; then
|
||||
qmd_add_collection_named "$QMD_WIKI_COLLECTION-$profile" "$bot_wiki" '**/*.md'
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
info "refreshing QMD indexes..."
|
||||
qmd update
|
||||
info "refreshing QMD embeddings..."
|
||||
qmd embed -c "$QMD_WIKI_COLLECTION" || warn "QMD embed failed for $QMD_WIKI_COLLECTION"
|
||||
if [[ "$SKIP_BOT_VIEWS" -eq 0 ]]; then
|
||||
for profile in "${PROFILES[@]}"; do
|
||||
qmd embed -c "$QMD_WIKI_COLLECTION-$profile" || warn "QMD embed failed for $QMD_WIKI_COLLECTION-$profile"
|
||||
done
|
||||
fi
|
||||
else
|
||||
warn "qmd not found; skipped QMD collection rebuild. Install qmd, then rerun with --skip-file-index --skip-bot-views if only QMD is needed."
|
||||
fi
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
Deployment complete.
|
||||
|
||||
Mother vault:
|
||||
$VAULT_DIR
|
||||
|
||||
obsidian-wiki command wrapper:
|
||||
$INSTALL_DIR/bin/obsidian-wiki-local
|
||||
|
||||
Examples:
|
||||
$INSTALL_DIR/bin/obsidian-wiki-local file-index search "FH52K drawing" -n 10
|
||||
$INSTALL_DIR/bin/obsidian-wiki-local bot-view audit
|
||||
qmd search "FH52K replacement" -c "$QMD_WIKI_COLLECTION-sales-bot" -n 5
|
||||
|
||||
Important:
|
||||
Point each robot at its own bot-view directory or QMD collection, not at the mother vault.
|
||||
EOF
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
import collections
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
OEM_TERMS = [
|
||||
"零跑",
|
||||
"Leapmotor",
|
||||
"BYD",
|
||||
"比亚迪",
|
||||
"Toyota",
|
||||
"SVW",
|
||||
"上汽大众",
|
||||
"MZD",
|
||||
"Mazda",
|
||||
"VW",
|
||||
"STLA",
|
||||
"Stellantis",
|
||||
"Ford",
|
||||
"Honda",
|
||||
"Hyundai",
|
||||
"Kia",
|
||||
"CATARC",
|
||||
"Yazaki",
|
||||
"Marelli",
|
||||
"Valeo",
|
||||
"Visteon",
|
||||
]
|
||||
|
||||
APP_TERMS = [
|
||||
"吸顶屏",
|
||||
"显示",
|
||||
"显示屏",
|
||||
"CID",
|
||||
"FID",
|
||||
"Cluster",
|
||||
"仪表",
|
||||
"Full Digital Cluster",
|
||||
"氛围灯",
|
||||
"Ambient Lighting",
|
||||
"储能",
|
||||
"车灯",
|
||||
"lighting",
|
||||
"霍尔小板",
|
||||
"排线",
|
||||
"FPC",
|
||||
"FFC",
|
||||
"Fakra",
|
||||
"FAKRA",
|
||||
"Socket Header",
|
||||
"Pin Header",
|
||||
"SMB",
|
||||
"BTB",
|
||||
"WTB",
|
||||
"Wafer",
|
||||
]
|
||||
|
||||
PRODUCT_RE = re.compile(
|
||||
r"\b(?:\d{6}-\d{4,6}(?:-HF)?|\d{3,}-\d{3,}(?:-HF)?|FH\d+[A-Z0-9-]*|C-\d{7})\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
PROJECT_RE = re.compile(
|
||||
r"(?:项目名称[:：]\s*([^\n\r；;，,]{2,60})|项目车型[:：]\s*([^\n\r；;，,]{2,60})|"
|
||||
r"\b(?:eP|EP)\s*\d{4,6}\b|CU\d+|BR\d+|MZD[-\w]+|Scout|RUV|PICKUP)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def domain_of(person):
|
||||
if not isinstance(person, dict):
|
||||
return None
|
||||
email = (person.get("email") or "").lower()
|
||||
return email.split("@")[-1] if "@" in email else None
|
||||
|
||||
|
||||
def context_snippet(text, term, limit=130):
|
||||
idx = text.lower().find(term.lower())
|
||||
if idx < 0:
|
||||
return ""
|
||||
start = max(0, idx - 45)
|
||||
end = min(len(text), idx + len(term) + 85)
|
||||
s = re.sub(r"\s+", " ", text[start:end]).strip()
|
||||
return s[:limit]
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: extract_email_project_signals.py /path/to/llm_input.jsonl", file=sys.stderr)
|
||||
return 2
|
||||
path = Path(sys.argv[1])
|
||||
pairs = collections.Counter()
|
||||
oems = collections.Counter()
|
||||
apps = collections.Counter()
|
||||
products = collections.Counter()
|
||||
domains = collections.Counter()
|
||||
rows = []
|
||||
|
||||
with path.open(encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
obj = json.loads(line)
|
||||
messages = obj.get("messages", [])
|
||||
text = " ".join(
|
||||
[obj.get("suggested_title_from_subject", "")]
|
||||
+ [(m.get("subject") or "") + " " + (m.get("body_text") or "") for m in messages]
|
||||
)
|
||||
found_oems = [x for x in OEM_TERMS if x.lower() in text.lower()]
|
||||
found_apps = [x for x in APP_TERMS if x.lower() in text.lower()]
|
||||
found_products = [x.upper() for x in PRODUCT_RE.findall(text)]
|
||||
project_hits = []
|
||||
for m in PROJECT_RE.finditer(text):
|
||||
project_hits.append(next((g for g in m.groups() if g), m.group(0)))
|
||||
if not (found_oems or found_apps or project_hits):
|
||||
continue
|
||||
thread_domains = collections.Counter()
|
||||
for msg in messages:
|
||||
for person in [msg.get("from", {})] + msg.get("to", []) + msg.get("cc", []):
|
||||
d = domain_of(person)
|
||||
if d:
|
||||
thread_domains[d] += 1
|
||||
for d, n in thread_domains.items():
|
||||
domains[d] += n
|
||||
for x in found_oems:
|
||||
oems[x] += 1
|
||||
for x in found_apps:
|
||||
apps[x] += 1
|
||||
for x in found_products[:25]:
|
||||
products[x] += 1
|
||||
for oem in found_oems:
|
||||
for app in found_apps:
|
||||
pairs[(oem, app)] += 1
|
||||
score = len(found_oems) * 4 + len(found_apps) * 2 + len(project_hits) * 3 + len(found_products)
|
||||
if score >= 8:
|
||||
anchor = (found_oems + found_apps + project_hits + found_products)[:1]
|
||||
rows.append(
|
||||
{
|
||||
"thread_id": obj.get("thread_id"),
|
||||
"subject": obj.get("suggested_title_from_subject", "")[:160],
|
||||
"date_end": obj.get("date_end"),
|
||||
"customer_domains": [d for d, _ in thread_domains.most_common(5)],
|
||||
"oems": found_oems[:8],
|
||||
"applications": found_apps[:10],
|
||||
"projects": project_hits[:8],
|
||||
"products": found_products[:12],
|
||||
"snippet": context_snippet(text, anchor[0]) if anchor else "",
|
||||
}
|
||||
)
|
||||
|
||||
out = {
|
||||
"oems_top": oems.most_common(50),
|
||||
"applications_top": apps.most_common(50),
|
||||
"products_top": products.most_common(80),
|
||||
"oem_application_pairs_top": [
|
||||
{"oem": k[0], "application": k[1], "threads": v}
|
||||
for k, v in pairs.most_common(80)
|
||||
],
|
||||
"customer_domains_top": domains.most_common(50),
|
||||
"project_signal_threads": sorted(rows, key=lambda x: x["date_end"] or "", reverse=True)[:120],
|
||||
}
|
||||
json.dump(out, sys.stdout, ensure_ascii=False, indent=2)
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,360 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build BJD replacement/cross-reference opportunity database from email-derived meta files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
VAULT = Path('/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault')
|
||||
META = VAULT / '_meta'
|
||||
OUT_DIR = VAULT / 'projects/bjd/references'
|
||||
FAST = META / 'bjd-fast-product-lookup.jsonl'
|
||||
THREADS = META / 'bjd-email-internal-thread-evidence-registry-20260611.jsonl'
|
||||
SNIPPETS = META / 'bjd-email-attachment-extracted-snippets-20260611.jsonl'
|
||||
RUN_DATE = '20260611'
|
||||
|
||||
BRANDS = {
|
||||
'Molex': r'Molex|MOLEX|莫仕|莫莱克斯',
|
||||
'Hirose': r'Hirose|HRS|广濑|FH\d+',
|
||||
'IRISO': r'IRISO|意力速',
|
||||
'TE': r'TE|Tyco|TYCO|泰科|AMP',
|
||||
'Rosenberger': r'Rosenberger|罗森博格|罗森',
|
||||
'Samtec': r'Samtec|SAMTEC',
|
||||
'Harwin': r'Harwin',
|
||||
'JST': r'JST|日压',
|
||||
'Kyocera': r'Kyocera|京瓷|AVX',
|
||||
'KEL': r'\bKEL\b',
|
||||
'JAE': r'\bJAE\b',
|
||||
'I-PEX': r'I-PEX|IPEX',
|
||||
'Kostal': r'Kostal',
|
||||
'UJU': r'\bUJU\b',
|
||||
'SMK': r'\bSMK\b',
|
||||
'Aptiv': r'Aptiv|APTIV|安波福',
|
||||
'Wurth': r'Wurth|Würth|WE|伍尔特',
|
||||
'JAM': r'JAM Global|\bJAM\b',
|
||||
}
|
||||
BRAND_RE = {k: re.compile(v, re.I) for k, v in BRANDS.items()}
|
||||
REPLACEMENT_RE = re.compile(r'替代|对标|对照|对应|可替代|匹配|cross(?:ed)?|cross[- ]?reference|replacement|alternative|equivalent|against|matching information', re.I)
|
||||
NEGATIVE_RE = re.compile(r'没有可替代|没有可以对标|无替代|暂无替代|not available|no replacement|cannot replace|不建议|无法替代', re.I)
|
||||
NEED_DRAWING_RE = re.compile(r'提供.*图纸|请.*图纸|need.*drawing|datasheet|规格书|footprint|land pattern|确认.*图纸', re.I)
|
||||
BJD_RE = re.compile(r'\b(?:1[0-9]{5}|2[0-9]{5}|9[0-9]{5})-[0-9A-Z]{4,}(?:-[A-Z0-9]+)?\b|\bX4\d{4}X?\b|\b1410[0-9]{2}-[0-9X]{6}\b|\b2410[0-9X]{2}-[0-9X]{6}\b', re.I)
|
||||
# Part numbers near brand text; intentionally broad but filters common words later.
|
||||
PN_RE = re.compile(r'\b(?=[A-Z0-9-]*\d)[A-Z0-9][A-Z0-9-]{3,24}[A-Z0-9]\b', re.I)
|
||||
BAD_PN = {'BJD','PDF','SMT','DIP','SMD','CONN','PIN','PINS','FPC','FFC','WTB','BTB','RFQ','SOP','PPAP','VAVE','AUTO','DATE','PART','MPN','MFR','NO','PCS','RMB','USD','FOB','HTTP','HTTPS'}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00','Z')
|
||||
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
with path.open(encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line=line.strip()
|
||||
if line:
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def clean_text(s: str) -> str:
|
||||
s = re.sub(r'<[^>]+>', ' ', s or '')
|
||||
s = re.sub(r' |&', ' ', s)
|
||||
s = re.sub(r'\s+', ' ', s).strip()
|
||||
return s
|
||||
|
||||
|
||||
def infer_brand(text: str) -> list[str]:
|
||||
return [brand for brand, rx in BRAND_RE.items() if rx.search(text)]
|
||||
|
||||
|
||||
def part_numbers(text: str) -> list[str]:
|
||||
out=[]
|
||||
for m in PN_RE.finditer(text):
|
||||
pn=m.group(0).strip('.,;:()[]{}<>')
|
||||
up=pn.upper()
|
||||
if up in BAD_PN or len(pn)<5:
|
||||
continue
|
||||
if re.fullmatch(r'20\d{2}[-/]?\d{1,2}[-/]?\d{1,2}', pn):
|
||||
continue
|
||||
if re.fullmatch(r'\d{4,}', pn) and len(pn) < 6:
|
||||
continue
|
||||
out.append(pn)
|
||||
# stable unique
|
||||
seen=set(); res=[]
|
||||
for x in out:
|
||||
key=x.upper()
|
||||
if key not in seen:
|
||||
seen.add(key); res.append(x)
|
||||
return res[:20]
|
||||
|
||||
|
||||
def bjd_numbers(text: str) -> list[str]:
|
||||
seen=set(); out=[]
|
||||
for m in BJD_RE.finditer(text):
|
||||
pn=m.group(0).strip('.,;:()[]{}<>')
|
||||
key=pn.upper()
|
||||
if key not in seen:
|
||||
seen.add(key); out.append(pn)
|
||||
return out[:12]
|
||||
|
||||
|
||||
def context_window(text: str, max_len: int = 700) -> str:
|
||||
text=clean_text(text)
|
||||
if len(text)<=max_len:
|
||||
return text
|
||||
m=REPLACEMENT_RE.search(text)
|
||||
if not m:
|
||||
return text[:max_len]
|
||||
start=max(0,m.start()-260); end=min(len(text),m.end()+440)
|
||||
return text[start:end]
|
||||
|
||||
|
||||
def confidence(text: str, brands: list[str], bjds: list[str], comp_pns: list[str], source_type: str) -> str:
|
||||
if NEGATIVE_RE.search(text):
|
||||
return 'negative'
|
||||
if source_type == 'fast_lookup':
|
||||
return 'high'
|
||||
if brands and bjds and comp_pns and REPLACEMENT_RE.search(text):
|
||||
return 'high'
|
||||
if bjds and REPLACEMENT_RE.search(text):
|
||||
return 'medium'
|
||||
if brands and comp_pns and REPLACEMENT_RE.search(text):
|
||||
return 'medium'
|
||||
return 'low'
|
||||
|
||||
|
||||
def stage_label(conf: str, text: str) -> str:
|
||||
if conf == 'negative':
|
||||
return '明确无替代/不建议'
|
||||
if NEED_DRAWING_RE.search(text):
|
||||
return '需图纸/工程复核'
|
||||
if re.search(r'报价|quote|RFQ|询价', text, re.I):
|
||||
return 'RFQ/报价评估'
|
||||
if re.search(r'样品|sample|承认|PPAP|SOP', text, re.I):
|
||||
return '样品/PPAP/项目推进'
|
||||
return '替代对标线索'
|
||||
|
||||
|
||||
def record_id(payload: dict[str, Any]) -> str:
|
||||
raw=json.dumps(payload,ensure_ascii=False,sort_keys=True)
|
||||
return hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]
|
||||
|
||||
|
||||
def add_record(records: list[dict[str, Any]], **kw):
|
||||
text=kw.get('evidence','') + ' ' + kw.get('subject','') + ' ' + kw.get('filename','')
|
||||
brands=kw.get('competitor_brands') or infer_brand(text)
|
||||
bjds=kw.get('bjd_candidates') or bjd_numbers(text)
|
||||
pns=kw.get('competitor_pns') or []
|
||||
if not pns:
|
||||
# remove BJD pns from generic pn list
|
||||
bjd_upper={x.upper() for x in bjds}
|
||||
pns=[x for x in part_numbers(text) if x.upper() not in bjd_upper][:12]
|
||||
conf=confidence(text, brands, bjds, pns, kw.get('source_type',''))
|
||||
if not (brands or bjds or pns or REPLACEMENT_RE.search(text)):
|
||||
return
|
||||
rec={
|
||||
'source_type': kw.get('source_type'),
|
||||
'source_ref': kw.get('source_ref'),
|
||||
'date': kw.get('date'),
|
||||
'customer_domains': kw.get('customer_domains') or [],
|
||||
'customers': kw.get('customers') or [],
|
||||
'subject': kw.get('subject') or '',
|
||||
'filename': kw.get('filename') or '',
|
||||
'product_families': kw.get('product_families') or [],
|
||||
'applications': kw.get('applications') or [],
|
||||
'competitor_brands': brands,
|
||||
'competitor_pns': pns,
|
||||
'bjd_candidates': bjds,
|
||||
'status': stage_label(conf, text),
|
||||
'confidence': conf,
|
||||
'evidence': context_window(kw.get('evidence') or text),
|
||||
}
|
||||
rec['id']=record_id(rec)
|
||||
records.append(rec)
|
||||
|
||||
|
||||
def build_from_fast(records):
|
||||
for r in read_jsonl(FAST):
|
||||
if r.get('kind')!='competitor_cross_reference':
|
||||
continue
|
||||
comp=' '.join(filter(None,[str(r.get('competitor') or ''), str(r.get('competitor_pn') or '')]))
|
||||
text=' '.join([str(r.get('model') or ''), str(r.get('description') or ''), comp, str(r.get('bjd_pn') or ''), str(r.get('notes') or '')])
|
||||
add_record(
|
||||
records,
|
||||
source_type='fast_lookup', source_ref=r.get('source_page'), date='2026-04-22',
|
||||
subject=f"{r.get('source_slug')}:{r.get('source_sheet')} row {r.get('row_number')}",
|
||||
product_families=[r.get('product_line')] if r.get('product_line') else [],
|
||||
competitor_brands=infer_brand(comp or text), competitor_pns=[r.get('competitor_pn')] if r.get('competitor_pn') else part_numbers(comp),
|
||||
bjd_candidates=[r.get('bjd_pn')] if r.get('bjd_pn') else [], evidence=text,
|
||||
)
|
||||
|
||||
|
||||
def build_from_threads(records):
|
||||
for r in read_jsonl(THREADS):
|
||||
subject=clean_text(r.get('subject',''))
|
||||
ev=' '.join(clean_text(x) for x in r.get('evidence') or [])
|
||||
text=subject+' '+ev
|
||||
if not REPLACEMENT_RE.search(text) and not infer_brand(text):
|
||||
continue
|
||||
# keep signal-heavy rows only
|
||||
if not (REPLACEMENT_RE.search(text) or bjd_numbers(text) or any(b in text for b in ['Molex','Hirose','IRISO','TE','Samtec','Harwin','Rosenberger'])):
|
||||
continue
|
||||
add_record(records, source_type='email_thread', source_ref=r.get('thread_id'), date=(r.get('date_end') or '')[:10],
|
||||
customers=r.get('customers') or [], customer_domains=r.get('domains') or [], subject=subject,
|
||||
product_families=r.get('families') or [], applications=r.get('applications') or [], evidence=text)
|
||||
|
||||
|
||||
def build_from_snippets(records):
|
||||
for r in read_jsonl(SNIPPETS):
|
||||
subject='; '.join(r.get('subjects') or [])
|
||||
filename=r.get('filename') or ''
|
||||
text=clean_text(' '.join([filename, subject, r.get('text_excerpt') or '']))
|
||||
if not (REPLACEMENT_RE.search(text) or re.search(r'\b(MPN|Matching Information|Crossed competitor|Alternative to)\b', text, re.I)):
|
||||
continue
|
||||
if not (infer_brand(text) or bjd_numbers(text)):
|
||||
continue
|
||||
add_record(records, source_type='email_attachment', source_ref=r.get('absolute_path') or r.get('content_sha1'), date=(r.get('date_end') or '')[:10],
|
||||
customer_domains=r.get('top_domains') or [], subject=subject, filename=filename,
|
||||
product_families=[r.get('class')] if r.get('class') else [], evidence=text)
|
||||
|
||||
|
||||
def dedupe(records):
|
||||
best={}
|
||||
rank={'high':3,'medium':2,'low':1,'negative':2}
|
||||
for r in records:
|
||||
key=(tuple(sorted(x.upper() for x in r['competitor_brands'])), tuple(x.upper() for x in r['competitor_pns'][:5]), tuple(x.upper() for x in r['bjd_candidates'][:5]), r['subject'][:80])
|
||||
if key not in best or rank.get(r['confidence'],0)>rank.get(best[key]['confidence'],0):
|
||||
best[key]=r
|
||||
out=list(best.values())
|
||||
out.sort(key=lambda r: ({'high':0,'medium':1,'negative':2,'low':3}.get(r['confidence'],4), r.get('date') or '', r.get('subject') or ''))
|
||||
return out
|
||||
|
||||
|
||||
def write_jsonl(path, records):
|
||||
with path.open('w',encoding='utf-8') as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r,ensure_ascii=False)+'\n')
|
||||
|
||||
|
||||
def write_csv(path, records):
|
||||
fields=['id','date','source_type','customer_domains','subject','filename','product_families','applications','competitor_brands','competitor_pns','bjd_candidates','status','confidence','source_ref','evidence']
|
||||
with path.open('w',encoding='utf-8-sig',newline='') as f:
|
||||
w=csv.DictWriter(f,fieldnames=fields)
|
||||
w.writeheader()
|
||||
for r in records:
|
||||
row={k:r.get(k,'') for k in fields}
|
||||
for k in ['customer_domains','product_families','applications','competitor_brands','competitor_pns','bjd_candidates']:
|
||||
row[k]='; '.join(row[k] or [])
|
||||
w.writerow(row)
|
||||
|
||||
|
||||
def md_table(rows):
|
||||
lines=['| 原厂/品牌 | 原厂料号 | BJD候选 | 状态 | 证据 |','|---|---|---|---|---|']
|
||||
for r in rows:
|
||||
lines.append('| ' + ' | '.join([
|
||||
esc('; '.join(r['competitor_brands']) or '未提取'),
|
||||
esc('; '.join(r['competitor_pns'][:4]) or '未提取'),
|
||||
esc('; '.join(r['bjd_candidates'][:4]) or '未提取'),
|
||||
esc(r['status']+' / '+r['confidence']),
|
||||
esc((r.get('subject') or r.get('filename') or r.get('evidence',''))[:180]),
|
||||
]) + ' |')
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def esc(s):
|
||||
return str(s).replace('|','\\|').replace('\n',' ')
|
||||
|
||||
|
||||
def write_page(records, jsonl, csvp):
|
||||
ts=now_iso()
|
||||
by_conf=Counter(r['confidence'] for r in records)
|
||||
by_brand=Counter(b for r in records for b in r['competitor_brands'])
|
||||
by_family=Counter(f for r in records for f in r['product_families'])
|
||||
high=[r for r in records if r['confidence']=='high'][:40]
|
||||
negative=[r for r in records if r['confidence']=='negative'][:25]
|
||||
samtec=[r for r in records if any(b in {'Samtec','Harwin'} for b in r['competitor_brands'])][:25]
|
||||
molex=[r for r in records if 'Molex' in r['competitor_brands']][:35]
|
||||
page=OUT_DIR/f'bjd-email-replacement-cross-reference-database-{RUN_DATE}.md'
|
||||
lines=[
|
||||
'---',
|
||||
'title: BJD 邮件替代对标机会数据库',
|
||||
'category: reference',
|
||||
'project: bjd',
|
||||
'tags: [bjd, emails, replacement, cross-reference, rfq, visibility/internal]',
|
||||
'visibility: internal',
|
||||
'business_domains: [sales, engineering, products]',
|
||||
'data_classification: technical-confidential',
|
||||
f'updated: {ts}',
|
||||
'---','',
|
||||
'# BJD 邮件替代对标机会数据库','',
|
||||
'本页把跟单邮件线程、附件摘录和既有 fast lookup 中的替代/对标信息归并成结构化检索入口。它用于销售工程初筛，不等同于最终 drop-in 承诺；正式推荐仍需工程按图纸、land pattern、pitch、高度、pin 数、接点方向、包装和验证状态复核。','',
|
||||
'## 数据规模','',
|
||||
f'- 结构化记录：{len(records)} 条。',
|
||||
f'- 置信度分布：' + '；'.join(f'{k}={v}' for k,v in by_conf.most_common()),
|
||||
f'- 主要品牌：' + '；'.join(f'{k}={v}' for k,v in by_brand.most_common(12)),
|
||||
f'- 主要产品/来源族：' + '；'.join(f'{k}={v}' for k,v in by_family.most_common(10)),
|
||||
'',
|
||||
'## 使用口径','',
|
||||
'- `high`：有直接对照表、明确邮件语句，或同时出现原厂料号与 BJD 候选，可作为优先候选。',
|
||||
'- `medium`：有替代/RFQ语境和原厂品牌/料号，但 BJD 候选不完整，需要二次查图纸或产品族。',
|
||||
'- `negative`：邮件明确说没有可替代、不能对标或需要客户提供图纸；这类记录很重要，避免销售误推。',
|
||||
'- `low`：只有弱线索，适合放入待验证池。','',
|
||||
'## 高置信直接对照样例','', md_table(high), '',
|
||||
'## Molex 相关线索','', md_table(molex), '',
|
||||
'## Samtec / Harwin 相关线索','', md_table(samtec), '',
|
||||
'## 明确无替代或需谨慎的线索','', md_table(negative), '',
|
||||
'## 数据文件','',
|
||||
f'- JSONL：`_meta/{jsonl.name}`',
|
||||
f'- CSV：`_meta/{csvp.name}`',
|
||||
'- 主要来源：`_meta/bjd-email-internal-thread-evidence-registry-20260611.jsonl`、`_meta/bjd-email-attachment-extracted-snippets-20260611.jsonl`、`_meta/bjd-fast-product-lookup.jsonl`。',
|
||||
]
|
||||
page.write_text('\n'.join(lines)+'\n',encoding='utf-8')
|
||||
update_meta(ts, page, jsonl, csvp, len(records))
|
||||
return page
|
||||
|
||||
|
||||
def update_meta(ts, page, jsonl, csvp, n):
|
||||
rel=page.relative_to(VAULT).with_suffix('').as_posix()
|
||||
idx=VAULT/'index.md'
|
||||
if idx.exists():
|
||||
text=idx.read_text(encoding='utf-8')
|
||||
text=re.sub(r'最后更新：[^*]+', f'最后更新：{ts}', text)
|
||||
entry=f'- [[{rel}]] - 从邮件线程、附件摘录和 fast lookup 归并的替代对标机会数据库。 ( #bjd #replacement #emails #cross-reference )'
|
||||
if rel not in text:
|
||||
marker='## 综合分析\n'
|
||||
text=text.replace(marker, marker+entry+'\n', 1) if marker in text else text+'\n## 综合分析\n'+entry+'\n'
|
||||
idx.write_text(text,encoding='utf-8')
|
||||
with (VAULT/'log.md').open('a',encoding='utf-8') as f:
|
||||
f.write(f'\n- [{ts}] DATA_INGEST source="email-derived replacement signals" project="bjd" mode="replacement-cross-reference-database" records={n} page="{page.relative_to(VAULT).as_posix()}" jsonl="{jsonl.relative_to(VAULT).as_posix()}" csv="{csvp.relative_to(VAULT).as_posix()}" bot_visible=false\n')
|
||||
mp=VAULT/'.manifest.json'
|
||||
data={}
|
||||
if mp.exists():
|
||||
try: data=json.loads(mp.read_text(encoding='utf-8'))
|
||||
except Exception: data={}
|
||||
data.setdefault('sources',{})['derived:email-replacement-cross-reference-20260611']={
|
||||
'ingested_at':ts,'source_type':'derived_email_replacement_database','doc_type':'replacement_cross_reference','project':'bjd','records':n,
|
||||
'pages_created':[page.relative_to(VAULT).as_posix()],'data_paths':[jsonl.relative_to(VAULT).as_posix(), csvp.relative_to(VAULT).as_posix()], 'bot_visible':False
|
||||
}
|
||||
mp.write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
records=[]
|
||||
build_from_fast(records)
|
||||
build_from_threads(records)
|
||||
build_from_snippets(records)
|
||||
records=dedupe(records)
|
||||
jsonl=META/f'bjd-email-replacement-cross-reference-db-{RUN_DATE}.jsonl'
|
||||
csvp=META/f'bjd-email-replacement-cross-reference-db-{RUN_DATE}.csv'
|
||||
write_jsonl(jsonl,records)
|
||||
write_csv(csvp,records)
|
||||
page=write_page(records,jsonl,csvp)
|
||||
print(json.dumps({'records':len(records),'jsonl':str(jsonl),'csv':str(csvp),'page':str(page)},ensure_ascii=False))
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,385 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build BJD replacement/cross-reference opportunity database from email-derived meta files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
VAULT = Path('/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault')
|
||||
META = VAULT / '_meta'
|
||||
OUT_DIR = VAULT / 'projects/bjd/references'
|
||||
FAST = META / 'bjd-fast-product-lookup.jsonl'
|
||||
THREADS = META / 'bjd-email-internal-thread-evidence-registry-20260611.jsonl'
|
||||
SNIPPETS = META / 'bjd-email-attachment-extracted-snippets-20260611.jsonl'
|
||||
RUN_DATE = '20260611'
|
||||
VERSION = 'v2'
|
||||
|
||||
BRANDS = {
|
||||
'Molex': r'Molex|MOLEX|莫仕|莫莱克斯',
|
||||
'Hirose': r'Hirose|HRS|广濑|FH\d+',
|
||||
'IRISO': r'IRISO|意力速',
|
||||
'TE': r'(?<![A-Za-z0-9])(?:TE|Tyco|TYCO|泰科|AMP)(?![A-Za-z0-9])',
|
||||
'Rosenberger': r'Rosenberger|罗森博格|罗森',
|
||||
'Samtec': r'Samtec|SAMTEC',
|
||||
'Harwin': r'Harwin',
|
||||
'JST': r'JST|日压',
|
||||
'Kyocera': r'Kyocera|京瓷|AVX',
|
||||
'KEL': r'\bKEL\b',
|
||||
'JAE': r'\bJAE\b',
|
||||
'I-PEX': r'I-PEX|IPEX',
|
||||
'Kostal': r'Kostal',
|
||||
'UJU': r'\bUJU\b',
|
||||
'SMK': r'\bSMK\b',
|
||||
'Aptiv': r'Aptiv|APTIV|安波福',
|
||||
'Wurth': r'Wurth|Würth|伍尔特',
|
||||
'JAM': r'JAM Global|\bJAM\b',
|
||||
}
|
||||
BRAND_RE = {k: re.compile(v, re.I) for k, v in BRANDS.items()}
|
||||
REPLACEMENT_RE = re.compile(r'替代|对标|对照|对应|可替代|匹配|cross(?:ed)?|cross[- ]?reference|replacement|alternative|equivalent|against|matching information', re.I)
|
||||
NEGATIVE_RE = re.compile(r'没有可替代|没有可以对标|无替代|暂无替代|无对应|not available|no replacement|cannot replace|无法替代|不能替代|not cross|no cross', re.I)
|
||||
NEED_DRAWING_RE = re.compile(r'提供.*图纸|请.*图纸|need.*drawing|datasheet|规格书|footprint|land pattern|确认.*图纸', re.I)
|
||||
BJD_RE = re.compile(r'\b(?:1[0-9]{5}|2[0-9]{5}|9[0-9]{5})-[0-9A-Z]{4,}(?:-[A-Z0-9]+)?\b|\bX4\d{4}X?\b|\b1410[0-9]{2}-[0-9X]{6}\b|\b2410[0-9X]{2}-[0-9X]{6}\b', re.I)
|
||||
# Part numbers near brand text; intentionally broad but filters common words later.
|
||||
PN_RE = re.compile(r'\b(?=[A-Z0-9-]*\d)[A-Z0-9][A-Z0-9-]{3,24}[A-Z0-9]\b', re.I)
|
||||
BAD_PN = {'BJD','PDF','PDF-1','SMT','DIP','SMD','CONN','CONNECTOR','PIN','PINS','FPC','FFC','WTB','BTB','RFQ','SOP','PPAP','VAVE','AUTO','DATE','PART','MPN','MFR','NO','PCS','RMB','USD','FOB','HTTP','HTTPS','IMAGE','FONTFILE','FONT','CID','OBJ','ENDOBJ','STREAM','ENDSTREAM'}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00','Z')
|
||||
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
with path.open(encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line=line.strip()
|
||||
if line:
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def clean_text(s: str) -> str:
|
||||
s = re.sub(r'<[^>]+>', ' ', s or '')
|
||||
s = re.sub(r' |&', ' ', s)
|
||||
s = re.sub(r'\s+', ' ', s).strip()
|
||||
return s
|
||||
|
||||
|
||||
def infer_brand(text: str) -> list[str]:
|
||||
return [brand for brand, rx in BRAND_RE.items() if rx.search(text)]
|
||||
|
||||
|
||||
def part_numbers(text: str) -> list[str]:
|
||||
out=[]
|
||||
for m in PN_RE.finditer(text):
|
||||
pn=m.group(0).strip('.,;:()[]{}<>')
|
||||
up=pn.upper()
|
||||
if up in BAD_PN or len(pn)<5:
|
||||
continue
|
||||
if up.startswith(('IMAGE','FONT','CID','PDF')):
|
||||
continue
|
||||
if re.fullmatch(r'[0-9]+(?:MM|PCS|P)?', up):
|
||||
continue
|
||||
if re.fullmatch(r'[0-9]+(?:\.[0-9]+)?', up):
|
||||
continue
|
||||
if re.fullmatch(r'20\d{2}[-/]?\d{1,2}[-/]?\d{1,2}', pn):
|
||||
continue
|
||||
if re.fullmatch(r'\d{4,}', pn) and len(pn) < 6:
|
||||
continue
|
||||
out.append(pn)
|
||||
# stable unique
|
||||
seen=set(); res=[]
|
||||
for x in out:
|
||||
key=x.upper()
|
||||
if key not in seen:
|
||||
seen.add(key); res.append(x)
|
||||
return res[:20]
|
||||
|
||||
|
||||
def bjd_numbers(text: str) -> list[str]:
|
||||
seen=set(); out=[]
|
||||
for m in BJD_RE.finditer(text):
|
||||
pn=m.group(0).strip('.,;:()[]{}<>')
|
||||
key=pn.upper()
|
||||
if key not in seen:
|
||||
seen.add(key); out.append(pn)
|
||||
return out[:12]
|
||||
|
||||
|
||||
def context_window(text: str, max_len: int = 700) -> str:
|
||||
text=clean_text(text)
|
||||
if len(text)<=max_len:
|
||||
return text
|
||||
m=REPLACEMENT_RE.search(text)
|
||||
if not m:
|
||||
return text[:max_len]
|
||||
start=max(0,m.start()-260); end=min(len(text),m.end()+440)
|
||||
return text[start:end]
|
||||
|
||||
|
||||
def confidence(text: str, brands: list[str], bjds: list[str], comp_pns: list[str], source_type: str) -> str:
|
||||
if NEGATIVE_RE.search(text) and (REPLACEMENT_RE.search(text) or brands or bjds or comp_pns):
|
||||
return 'negative'
|
||||
if source_type == 'fast_lookup':
|
||||
return 'high'
|
||||
if brands and bjds and comp_pns and REPLACEMENT_RE.search(text):
|
||||
return 'high'
|
||||
if bjds and REPLACEMENT_RE.search(text):
|
||||
return 'medium'
|
||||
if brands and comp_pns and REPLACEMENT_RE.search(text):
|
||||
return 'medium'
|
||||
return 'low'
|
||||
|
||||
|
||||
def stage_label(conf: str, text: str) -> str:
|
||||
if conf == 'negative':
|
||||
return '明确无替代/不建议'
|
||||
if NEED_DRAWING_RE.search(text):
|
||||
return '需图纸/工程复核'
|
||||
if re.search(r'报价|quote|RFQ|询价', text, re.I):
|
||||
return 'RFQ/报价评估'
|
||||
if re.search(r'样品|sample|承认|PPAP|SOP', text, re.I):
|
||||
return '样品/PPAP/项目推进'
|
||||
return '替代对标线索'
|
||||
|
||||
|
||||
def record_id(payload: dict[str, Any]) -> str:
|
||||
raw=json.dumps(payload,ensure_ascii=False,sort_keys=True)
|
||||
return hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]
|
||||
|
||||
|
||||
def add_record(records: list[dict[str, Any]], **kw):
|
||||
text=kw.get('evidence','') + ' ' + kw.get('subject','') + ' ' + kw.get('filename','')
|
||||
brands=kw.get('competitor_brands') or infer_brand(text)
|
||||
bjds=kw.get('bjd_candidates') or bjd_numbers(text)
|
||||
pns=kw.get('competitor_pns') or []
|
||||
if not pns:
|
||||
# remove BJD pns from generic pn list
|
||||
bjd_upper={x.upper() for x in bjds}
|
||||
pns=[x for x in part_numbers(text) if x.upper() not in bjd_upper][:12]
|
||||
conf=confidence(text, brands, bjds, pns, kw.get('source_type',''))
|
||||
if not (brands or bjds or pns or REPLACEMENT_RE.search(text)):
|
||||
return
|
||||
rec={
|
||||
'source_type': kw.get('source_type'),
|
||||
'source_ref': kw.get('source_ref'),
|
||||
'date': kw.get('date'),
|
||||
'customer_domains': kw.get('customer_domains') or [],
|
||||
'customers': kw.get('customers') or [],
|
||||
'subject': kw.get('subject') or '',
|
||||
'filename': kw.get('filename') or '',
|
||||
'product_families': kw.get('product_families') or [],
|
||||
'applications': kw.get('applications') or [],
|
||||
'competitor_brands': brands,
|
||||
'competitor_pns': pns,
|
||||
'bjd_candidates': bjds,
|
||||
'status': stage_label(conf, text),
|
||||
'confidence': conf,
|
||||
'evidence': context_window(kw.get('evidence') or text),
|
||||
}
|
||||
rec['id']=record_id(rec)
|
||||
records.append(rec)
|
||||
|
||||
|
||||
def build_from_fast(records):
|
||||
for r in read_jsonl(FAST):
|
||||
if r.get('kind')!='competitor_cross_reference':
|
||||
continue
|
||||
comp=' '.join(filter(None,[str(r.get('competitor') or ''), str(r.get('competitor_pn') or '')]))
|
||||
text=' '.join([str(r.get('model') or ''), str(r.get('description') or ''), comp, str(r.get('bjd_pn') or ''), str(r.get('notes') or '')])
|
||||
add_record(
|
||||
records,
|
||||
source_type='fast_lookup', source_ref=r.get('source_page'), date='2026-04-22',
|
||||
subject=f"{r.get('source_slug')}:{r.get('source_sheet')} row {r.get('row_number')}",
|
||||
product_families=[r.get('product_line')] if r.get('product_line') else [],
|
||||
competitor_brands=infer_brand(comp or text), competitor_pns=[r.get('competitor_pn')] if r.get('competitor_pn') else part_numbers(comp),
|
||||
bjd_candidates=[r.get('bjd_pn')] if r.get('bjd_pn') else [], evidence=text,
|
||||
)
|
||||
|
||||
|
||||
def build_from_threads(records):
|
||||
for r in read_jsonl(THREADS):
|
||||
subject=clean_text(r.get('subject',''))
|
||||
ev=' '.join(clean_text(x) for x in r.get('evidence') or [])
|
||||
text=subject+' '+ev
|
||||
if not (REPLACEMENT_RE.search(text) or bjd_numbers(text)):
|
||||
continue
|
||||
if not (infer_brand(text) or bjd_numbers(text)):
|
||||
continue
|
||||
add_record(records, source_type='email_thread', source_ref=r.get('thread_id'), date=(r.get('date_end') or '')[:10],
|
||||
customers=r.get('customers') or [], customer_domains=r.get('domains') or [], subject=subject,
|
||||
product_families=r.get('families') or [], applications=r.get('applications') or [], evidence=text)
|
||||
|
||||
|
||||
def build_from_snippets(records):
|
||||
for r in read_jsonl(SNIPPETS):
|
||||
subject='; '.join(r.get('subjects') or [])
|
||||
filename=r.get('filename') or ''
|
||||
text=clean_text(' '.join([filename, subject, r.get('text_excerpt') or '']))
|
||||
strong_attachment = re.search(r'替代|对标|对照|cross|replacement|alternative|Matching Information|Crossed competitor|BJD P/?N|BJD料号|BJD 料号', text, re.I)
|
||||
if not strong_attachment:
|
||||
continue
|
||||
if not (infer_brand(text) or bjd_numbers(text)):
|
||||
continue
|
||||
if len(text) > 5000 and not REPLACEMENT_RE.search(filename + ' ' + subject):
|
||||
continue
|
||||
add_record(records, source_type='email_attachment', source_ref=r.get('absolute_path') or r.get('content_sha1'), date=(r.get('date_end') or '')[:10],
|
||||
customer_domains=r.get('top_domains') or [], subject=subject, filename=filename,
|
||||
product_families=[r.get('class')] if r.get('class') else [], evidence=text)
|
||||
|
||||
|
||||
def split_records(records):
|
||||
direct=[]; opportunity=[]; negative=[]
|
||||
for r in records:
|
||||
if r['confidence'] == 'negative':
|
||||
negative.append(r)
|
||||
elif r['confidence'] == 'high' or r['bjd_candidates']:
|
||||
direct.append(r)
|
||||
else:
|
||||
opportunity.append(r)
|
||||
return direct, opportunity, negative
|
||||
|
||||
|
||||
def dedupe(records):
|
||||
best={}
|
||||
rank={'high':3,'medium':2,'low':1,'negative':2}
|
||||
for r in records:
|
||||
key=(tuple(sorted(x.upper() for x in r['competitor_brands'])), tuple(x.upper() for x in r['competitor_pns'][:5]), tuple(x.upper() for x in r['bjd_candidates'][:5]), r['subject'][:80])
|
||||
if key not in best or rank.get(r['confidence'],0)>rank.get(best[key]['confidence'],0):
|
||||
best[key]=r
|
||||
out=list(best.values())
|
||||
out.sort(key=lambda r: ({'high':0,'medium':1,'negative':2,'low':3}.get(r['confidence'],4), r.get('date') or '', r.get('subject') or ''))
|
||||
return out
|
||||
|
||||
|
||||
def write_jsonl(path, records):
|
||||
with path.open('w',encoding='utf-8') as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r,ensure_ascii=False)+'\n')
|
||||
|
||||
|
||||
def write_csv(path, records):
|
||||
fields=['id','date','source_type','customer_domains','subject','filename','product_families','applications','competitor_brands','competitor_pns','bjd_candidates','status','confidence','source_ref','evidence']
|
||||
with path.open('w',encoding='utf-8-sig',newline='') as f:
|
||||
w=csv.DictWriter(f,fieldnames=fields)
|
||||
w.writeheader()
|
||||
for r in records:
|
||||
row={k:r.get(k,'') for k in fields}
|
||||
for k in ['customer_domains','product_families','applications','competitor_brands','competitor_pns','bjd_candidates']:
|
||||
row[k]='; '.join(row[k] or [])
|
||||
w.writerow(row)
|
||||
|
||||
|
||||
def md_table(rows):
|
||||
lines=['| 原厂/品牌 | 原厂料号 | BJD候选 | 状态 | 证据 |','|---|---|---|---|---|']
|
||||
for r in rows:
|
||||
lines.append('| ' + ' | '.join([
|
||||
esc('; '.join(r['competitor_brands']) or '未提取'),
|
||||
esc('; '.join(r['competitor_pns'][:4]) or '未提取'),
|
||||
esc('; '.join(r['bjd_candidates'][:4]) or '未提取'),
|
||||
esc(r['status']+' / '+r['confidence']),
|
||||
esc((r.get('subject') or r.get('filename') or r.get('evidence',''))[:180]),
|
||||
]) + ' |')
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def esc(s):
|
||||
return str(s).replace('|','\\|').replace('\n',' ')
|
||||
|
||||
|
||||
def write_page(records, jsonl, csvp):
|
||||
ts=now_iso()
|
||||
by_conf=Counter(r['confidence'] for r in records)
|
||||
by_brand=Counter(b for r in records for b in r['competitor_brands'])
|
||||
by_family=Counter(f for r in records for f in r['product_families'])
|
||||
high=[r for r in records if r['confidence']=='high'][:40]
|
||||
negative=[r for r in records if r['confidence']=='negative'][:25]
|
||||
samtec=[r for r in records if any(b in {'Samtec','Harwin'} for b in r['competitor_brands'])][:25]
|
||||
molex=[r for r in records if 'Molex' in r['competitor_brands']][:35]
|
||||
page=OUT_DIR/f'bjd-email-replacement-cross-reference-database-v2-{RUN_DATE}.md'
|
||||
lines=[
|
||||
'---',
|
||||
'title: BJD 邮件替代对标机会数据库',
|
||||
'category: reference',
|
||||
'project: bjd',
|
||||
'tags: [bjd, emails, replacement, cross-reference, rfq, visibility/internal]',
|
||||
'visibility: internal',
|
||||
'business_domains: [sales, engineering, products]',
|
||||
'data_classification: technical-confidential',
|
||||
f'updated: {ts}',
|
||||
'---','',
|
||||
'# BJD 邮件替代对标机会数据库 v2','',
|
||||
'本页把跟单邮件线程、附件摘录和既有 fast lookup 中的替代/对标信息归并成结构化检索入口。v2 版本修正了 TE/WE 等短词误匹配，并把负向语义限定在替代语境内。它用于销售工程初筛，不等同于最终 drop-in 承诺；正式推荐仍需工程按图纸、land pattern、pitch、高度、pin 数、接点方向、包装和验证状态复核。','',
|
||||
'## 数据规模','',
|
||||
f'- 结构化记录：{len(records)} 条。',
|
||||
f'- 置信度分布：' + '；'.join(f'{k}={v}' for k,v in by_conf.most_common()),
|
||||
f'- 主要品牌：' + '；'.join(f'{k}={v}' for k,v in by_brand.most_common(12)),
|
||||
f'- 主要产品/来源族：' + '；'.join(f'{k}={v}' for k,v in by_family.most_common(10)),
|
||||
'',
|
||||
'## 使用口径','',
|
||||
'- `high`：有直接对照表、明确邮件语句，或同时出现原厂料号与 BJD 候选，可作为优先候选。',
|
||||
'- `medium`：有替代/RFQ语境和原厂品牌/料号，但 BJD 候选不完整，需要二次查图纸或产品族。',
|
||||
'- `negative`：邮件明确说没有可替代、不能对标或需要客户提供图纸；这类记录很重要，避免销售误推。',
|
||||
'- `low`：只有弱线索，适合放入待验证池。','',
|
||||
'## 高置信直接对照样例','', md_table(high), '',
|
||||
'## Molex 相关线索','', md_table(molex), '',
|
||||
'## Samtec / Harwin 相关线索','', md_table(samtec), '',
|
||||
'## 明确无替代或需谨慎的线索','', md_table(negative), '',
|
||||
'## 数据文件','',
|
||||
f'- JSONL：`_meta/{jsonl.name}`',
|
||||
f'- CSV：`_meta/{csvp.name}`',
|
||||
'- 主要来源：`_meta/bjd-email-internal-thread-evidence-registry-20260611.jsonl`、`_meta/bjd-email-attachment-extracted-snippets-20260611.jsonl`、`_meta/bjd-fast-product-lookup.jsonl`。',
|
||||
]
|
||||
page.write_text('\n'.join(lines)+'\n',encoding='utf-8')
|
||||
update_meta(ts, page, jsonl, csvp, len(records))
|
||||
return page
|
||||
|
||||
|
||||
def update_meta(ts, page, jsonl, csvp, n):
|
||||
rel=page.relative_to(VAULT).with_suffix('').as_posix()
|
||||
idx=VAULT/'index.md'
|
||||
if idx.exists():
|
||||
text=idx.read_text(encoding='utf-8')
|
||||
text=re.sub(r'最后更新：[^*]+', f'最后更新：{ts}', text)
|
||||
entry=f'- [[{rel}]] - 从邮件线程、附件摘录和 fast lookup 归并的替代对标机会数据库。 ( #bjd #replacement #emails #cross-reference )'
|
||||
if rel not in text:
|
||||
marker='## 综合分析\n'
|
||||
text=text.replace(marker, marker+entry+'\n', 1) if marker in text else text+'\n## 综合分析\n'+entry+'\n'
|
||||
idx.write_text(text,encoding='utf-8')
|
||||
with (VAULT/'log.md').open('a',encoding='utf-8') as f:
|
||||
f.write(f'\n- [{ts}] DATA_INGEST source="email-derived replacement signals" project="bjd" mode="replacement-cross-reference-database" records={n} page="{page.relative_to(VAULT).as_posix()}" jsonl="{jsonl.relative_to(VAULT).as_posix()}" csv="{csvp.relative_to(VAULT).as_posix()}" bot_visible=false\n')
|
||||
mp=VAULT/'.manifest.json'
|
||||
data={}
|
||||
if mp.exists():
|
||||
try: data=json.loads(mp.read_text(encoding='utf-8'))
|
||||
except Exception: data={}
|
||||
data.setdefault('sources',{})['derived:email-replacement-cross-reference-20260611']={
|
||||
'ingested_at':ts,'source_type':'derived_email_replacement_database','doc_type':'replacement_cross_reference','project':'bjd','records':n,
|
||||
'pages_created':[page.relative_to(VAULT).as_posix()],'data_paths':[jsonl.relative_to(VAULT).as_posix(), csvp.relative_to(VAULT).as_posix()], 'bot_visible':False
|
||||
}
|
||||
mp.write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
records=[]
|
||||
build_from_fast(records)
|
||||
build_from_threads(records)
|
||||
build_from_snippets(records)
|
||||
records=dedupe(records)
|
||||
jsonl=META/f'bjd-email-replacement-cross-reference-db-v2-{RUN_DATE}.jsonl'
|
||||
csvp=META/f'bjd-email-replacement-cross-reference-db-v2-{RUN_DATE}.csv'
|
||||
direct, opportunity, negative = split_records(records)
|
||||
write_jsonl(jsonl,records)
|
||||
write_csv(csvp,records)
|
||||
write_jsonl(META/f'bjd-email-replacement-cross-reference-db-v2-direct-{RUN_DATE}.jsonl', direct)
|
||||
write_jsonl(META/f'bjd-email-replacement-cross-reference-db-v2-opportunity-{RUN_DATE}.jsonl', opportunity)
|
||||
write_jsonl(META/f'bjd-email-replacement-cross-reference-db-v2-negative-{RUN_DATE}.jsonl', negative)
|
||||
page=write_page(records,jsonl,csvp)
|
||||
print(json.dumps({'records':len(records),'direct':len(direct),'opportunity':len(opportunity),'negative':len(negative),'jsonl':str(jsonl),'csv':str(csvp),'page':str(page)},ensure_ascii=False))
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EMAIL_RE = re.compile(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}")
|
||||
PHONE_RE = re.compile(r"(?<!\d)(?:\+?\d[\d -]{6,}\d)(?!\d)")
|
||||
PRICE_RE = re.compile(r"(?i)(?:rmb|usd|eur|cny|￥|\$)?\s*\d+(?:\.\d+)?\s*(?:元|块|usd|rmb|eur|cny|/pcs|/pc|/k|pcs)?")
|
||||
|
||||
|
||||
def redact(text: str) -> str:
|
||||
text = EMAIL_RE.sub("[email]", text)
|
||||
text = PHONE_RE.sub("[number]", text)
|
||||
text = PRICE_RE.sub("[number]", text)
|
||||
return text
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 3:
|
||||
print("usage: extract_email_thread_snippets.py file.jsonl thread_id...", file=sys.stderr)
|
||||
return 2
|
||||
path = Path(sys.argv[1])
|
||||
wanted = set(sys.argv[2:])
|
||||
with path.open(encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
obj = json.loads(line)
|
||||
if obj.get("thread_id") not in wanted:
|
||||
continue
|
||||
print("=" * 80)
|
||||
print("thread_id:", obj.get("thread_id"))
|
||||
print("subject:", redact(obj.get("suggested_title_from_subject", "")))
|
||||
print("date:", obj.get("date_start"), "->", obj.get("date_end"))
|
||||
print("messages:", len(obj.get("messages", [])), "attachments:", len(obj.get("attachments", [])))
|
||||
for i, msg in enumerate(obj.get("messages", [])[:8], 1):
|
||||
body = redact(msg.get("body_text") or "")
|
||||
body = re.sub(r"\n{3,}", "\n\n", body).strip()
|
||||
if len(body) > 1400:
|
||||
body = body[:1400] + "\n[...truncated...]"
|
||||
print("-" * 40)
|
||||
print("message", i, "date", msg.get("date", ""))
|
||||
print("subject:", redact(msg.get("subject", "")))
|
||||
print(body)
|
||||
if len(obj.get("messages", [])) > 8:
|
||||
print("[remaining messages omitted]")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,508 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract FFC/flat-cable PFMEA signals from indexed email attachments.
|
||||
|
||||
This intentionally uses only the Python standard library. The current host
|
||||
does not have openpyxl/xlrd/libreoffice installed, so .xlsx files are parsed
|
||||
from their OOXML parts and legacy .xls files are indexed with a conservative
|
||||
string scan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
|
||||
VAULT = Path("/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault")
|
||||
META = VAULT / "_meta"
|
||||
OUT_DIR = VAULT / "projects/bjd/references"
|
||||
INVENTORY = META / "bjd-email-attachment-inventory-20260611.jsonl"
|
||||
|
||||
TARGET_RE = re.compile(
|
||||
r"(PFMEA|P-FMEA|Process[_ ]?FMEA|过程失效|失效模式|AIAG.*VDA)",
|
||||
re.I,
|
||||
)
|
||||
FFC_RE = re.compile(r"(FFC|排线|柔性扁平|flat cable|VPMASF|VPNYGF|VPRASF|Backlight FFC)", re.I)
|
||||
|
||||
RISK_TERMS = [
|
||||
"失效",
|
||||
"不良",
|
||||
"断",
|
||||
"短路",
|
||||
"开路",
|
||||
"错",
|
||||
"反",
|
||||
"偏",
|
||||
"翘",
|
||||
"破",
|
||||
"裂",
|
||||
"压伤",
|
||||
"划伤",
|
||||
"污染",
|
||||
"脏污",
|
||||
"毛刺",
|
||||
"尺寸",
|
||||
"导通",
|
||||
"绝缘",
|
||||
"阻抗",
|
||||
"耐压",
|
||||
"接地",
|
||||
"GND",
|
||||
"pad",
|
||||
"焊盘",
|
||||
"端子",
|
||||
"线材",
|
||||
"线缆",
|
||||
"贴合",
|
||||
"冲切",
|
||||
"折弯",
|
||||
"包装",
|
||||
"标签",
|
||||
"混料",
|
||||
"Failure",
|
||||
"Defect",
|
||||
"Open",
|
||||
"Short",
|
||||
"Damage",
|
||||
"Dimension",
|
||||
"Continuity",
|
||||
"Insulation",
|
||||
]
|
||||
|
||||
CONTROL_TERMS = [
|
||||
"全检",
|
||||
"巡检",
|
||||
"首检",
|
||||
"自检",
|
||||
"抽检",
|
||||
"防错",
|
||||
"治具",
|
||||
"CCD",
|
||||
"AOI",
|
||||
"测试",
|
||||
"检验",
|
||||
"确认",
|
||||
"控制",
|
||||
"Control",
|
||||
"Inspection",
|
||||
"Test",
|
||||
"Poka",
|
||||
"Fixture",
|
||||
]
|
||||
|
||||
NS = {
|
||||
"s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main",
|
||||
"r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
|
||||
}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def xml_text(elem: ET.Element) -> str:
|
||||
return "".join(elem.itertext())
|
||||
|
||||
|
||||
def col_to_idx(cell_ref: str) -> int:
|
||||
m = re.match(r"([A-Z]+)", cell_ref or "")
|
||||
if not m:
|
||||
return 0
|
||||
n = 0
|
||||
for ch in m.group(1):
|
||||
n = n * 26 + (ord(ch) - 64)
|
||||
return n - 1
|
||||
|
||||
|
||||
def load_shared_strings(zf: zipfile.ZipFile) -> list[str]:
|
||||
try:
|
||||
data = zf.read("xl/sharedStrings.xml")
|
||||
except KeyError:
|
||||
return []
|
||||
root = ET.fromstring(data)
|
||||
out = []
|
||||
for si in root.findall("s:si", NS):
|
||||
out.append(xml_text(si))
|
||||
return out
|
||||
|
||||
|
||||
def workbook_sheets(zf: zipfile.ZipFile) -> list[tuple[str, str]]:
|
||||
wb = ET.fromstring(zf.read("xl/workbook.xml"))
|
||||
rels = ET.fromstring(zf.read("xl/_rels/workbook.xml.rels"))
|
||||
rid_to_target = {}
|
||||
for rel in rels:
|
||||
rid = rel.attrib.get("Id")
|
||||
target = rel.attrib.get("Target")
|
||||
if rid and target:
|
||||
rid_to_target[rid] = "xl/" + target.lstrip("/")
|
||||
sheets = []
|
||||
for sheet in wb.findall("s:sheets/s:sheet", NS):
|
||||
name = sheet.attrib.get("name", "Sheet")
|
||||
rid = sheet.attrib.get(f"{{{NS['r']}}}id")
|
||||
target = rid_to_target.get(rid)
|
||||
if target:
|
||||
sheets.append((name, target))
|
||||
return sheets
|
||||
|
||||
|
||||
def parse_xlsx(path: Path) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
"parser": "xlsx-ooxml-stdlib",
|
||||
"sheets": [],
|
||||
"candidate_rows": [],
|
||||
"term_counts": Counter(),
|
||||
"control_counts": Counter(),
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
shared = load_shared_strings(zf)
|
||||
for sheet_name, sheet_path in workbook_sheets(zf):
|
||||
try:
|
||||
root = ET.fromstring(zf.read(sheet_path))
|
||||
except KeyError:
|
||||
continue
|
||||
rows_seen = 0
|
||||
nonempty_rows = 0
|
||||
candidate_count = 0
|
||||
for row in root.findall(".//s:sheetData/s:row", NS):
|
||||
rows_seen += 1
|
||||
cells: dict[int, str] = {}
|
||||
for c in row.findall("s:c", NS):
|
||||
idx = col_to_idx(c.attrib.get("r", ""))
|
||||
typ = c.attrib.get("t")
|
||||
v = c.find("s:v", NS)
|
||||
text = ""
|
||||
if typ == "inlineStr":
|
||||
is_elem = c.find("s:is", NS)
|
||||
text = xml_text(is_elem) if is_elem is not None else ""
|
||||
elif v is not None:
|
||||
raw = v.text or ""
|
||||
if typ == "s":
|
||||
try:
|
||||
text = shared[int(raw)]
|
||||
except Exception:
|
||||
text = raw
|
||||
else:
|
||||
text = raw
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if text:
|
||||
cells[idx] = text
|
||||
if not cells:
|
||||
continue
|
||||
nonempty_rows += 1
|
||||
row_text = " | ".join(cells[i] for i in sorted(cells))
|
||||
for term in RISK_TERMS:
|
||||
if re.search(re.escape(term), row_text, re.I):
|
||||
result["term_counts"][term] += 1
|
||||
for term in CONTROL_TERMS:
|
||||
if re.search(re.escape(term), row_text, re.I):
|
||||
result["control_counts"][term] += 1
|
||||
is_candidate = (
|
||||
any(re.search(re.escape(term), row_text, re.I) for term in RISK_TERMS)
|
||||
or any(re.search(re.escape(term), row_text, re.I) for term in CONTROL_TERMS)
|
||||
or TARGET_RE.search(row_text)
|
||||
or FFC_RE.search(row_text)
|
||||
)
|
||||
if is_candidate:
|
||||
candidate_count += 1
|
||||
if len(result["candidate_rows"]) < 260:
|
||||
result["candidate_rows"].append(
|
||||
{
|
||||
"sheet": sheet_name,
|
||||
"row": int(row.attrib.get("r", rows_seen)),
|
||||
"text": row_text[:1200],
|
||||
}
|
||||
)
|
||||
result["sheets"].append(
|
||||
{
|
||||
"name": sheet_name,
|
||||
"rows_seen": rows_seen,
|
||||
"nonempty_rows": nonempty_rows,
|
||||
"candidate_rows": candidate_count,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - written to ingest report
|
||||
result["error"] = repr(exc)
|
||||
result["term_counts"] = dict(result["term_counts"].most_common())
|
||||
result["control_counts"] = dict(result["control_counts"].most_common())
|
||||
return result
|
||||
|
||||
|
||||
def scan_legacy_xls(path: Path) -> dict[str, Any]:
|
||||
data = path.read_bytes()
|
||||
latin = re.findall(rb"[\x20-\x7e]{4,}", data)
|
||||
utf16 = re.findall((rb"(?:[\x20-\x7e]\x00){4,}"), data)
|
||||
texts = []
|
||||
for b in latin[:3000]:
|
||||
texts.append(b.decode("latin1", "ignore"))
|
||||
for b in utf16[:3000]:
|
||||
texts.append(b.decode("utf-16le", "ignore"))
|
||||
candidates = []
|
||||
for text in texts:
|
||||
if TARGET_RE.search(text) or FFC_RE.search(text) or any(t.lower() in text.lower() for t in ["failure", "control", "inspection"]):
|
||||
candidates.append(text[:300])
|
||||
return {
|
||||
"parser": "xls-string-scan",
|
||||
"sheets": [],
|
||||
"candidate_rows": [{"sheet": "binary-string-scan", "row": i + 1, "text": t} for i, t in enumerate(candidates[:80])],
|
||||
"term_counts": {},
|
||||
"control_counts": {},
|
||||
"error": "legacy .xls not structurally parsed; string scan only",
|
||||
}
|
||||
|
||||
|
||||
def classify_file(rec: dict[str, Any]) -> bool:
|
||||
fn = rec.get("filename", "")
|
||||
subj = " ".join(rec.get("subjects") or [])
|
||||
text = f"{fn} {subj}"
|
||||
return bool(TARGET_RE.search(text) and FFC_RE.search(text))
|
||||
|
||||
|
||||
def summarize_patterns(parsed: list[dict[str, Any]]) -> tuple[list[tuple[str, int]], list[tuple[str, int]]]:
|
||||
risk = Counter()
|
||||
control = Counter()
|
||||
for item in parsed:
|
||||
risk.update(item["parsed"].get("term_counts", {}))
|
||||
control.update(item["parsed"].get("control_counts", {}))
|
||||
return risk.most_common(24), control.most_common(18)
|
||||
|
||||
|
||||
def md_escape(s: str) -> str:
|
||||
return s.replace("|", "\\|").replace("\n", " ")
|
||||
|
||||
|
||||
def write_outputs(records: list[dict[str, Any]], parsed: list[dict[str, Any]]) -> None:
|
||||
ts = now_iso()
|
||||
data_path = META / "bjd-ffc-pfmea-extraction-20260611.json"
|
||||
page_path = OUT_DIR / "bjd-ffc-pfmea-knowledge-20260611.md"
|
||||
|
||||
risk, control = summarize_patterns(parsed)
|
||||
|
||||
with data_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"generated_at": ts,
|
||||
"target_records": records,
|
||||
"parsed_files": parsed,
|
||||
"aggregate_risk_terms": risk,
|
||||
"aggregate_control_terms": control,
|
||||
},
|
||||
f,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"title: BJD FFC/排线 PFMEA 知识整理",
|
||||
"category: reference",
|
||||
"project: bjd",
|
||||
"tags: [bjd, ffc, pfmea, fmea, quality, ppap, visibility/internal]",
|
||||
"visibility: internal",
|
||||
"business_domains: [quality, products]",
|
||||
"data_classification: quality-confidential",
|
||||
f"updated: {ts}",
|
||||
"---",
|
||||
"",
|
||||
"# BJD FFC/排线 PFMEA 知识整理",
|
||||
"",
|
||||
"本页从邮件附件索引中的 FFC/排线 PFMEA、AIAG & VDA PFMEA、PPAP 资料中抽取。由于当前环境没有 `openpyxl/xlrd/LibreOffice`，`.xlsx` 使用 OOXML 原始 XML 解析，旧版 `.xls` 仅做文件级定位和字符串级弱提取；正式提交客户资料前仍应打开原始 Excel 复核。",
|
||||
"",
|
||||
"## 结论",
|
||||
"",
|
||||
"- 母库中已经存在多份排线/FFC PFMEA 源文件，不是只有口头线索。",
|
||||
"- 2026 年审核资料指出过一个体系性问题：柔性扁平线缆 NTF 清单相关知识没有完整传递到新项目 PFMEA，新项目 PFMEA 编制没有强制对标 NTF 文件。",
|
||||
"- 整改方向已经在资料中出现：用最新 NTF 清单逐项修正新项目 PFMEA，补全新技术对应的失效风险、预防控制和探测控制。",
|
||||
"- FFC PFMEA 的客户使用场景主要集中在 PPAP、客户审核、规格变更、样品/封样、背光/车载显示相关线缆项目。",
|
||||
"",
|
||||
"## 已定位的 FFC/排线 PFMEA 源文件",
|
||||
"",
|
||||
"| 文件 | 日期 | 邮件主题 | 解析状态 |",
|
||||
"|---|---:|---|---|",
|
||||
]
|
||||
for item in parsed:
|
||||
rec = item["record"]
|
||||
p = item["parsed"]
|
||||
status = p["parser"]
|
||||
if p.get("error"):
|
||||
status += f"; {p['error']}"
|
||||
lines.append(
|
||||
f"| `{md_escape(rec.get('filename',''))}` | {rec.get('date_end','未提取')[:10]} | {md_escape('; '.join(rec.get('subjects') or []))} | {md_escape(status)} |"
|
||||
)
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"## 聚合风险信号",
|
||||
"",
|
||||
"| 风险/失效关键词 | 出现行数 |",
|
||||
"|---|---:|",
|
||||
]
|
||||
for term, count in risk:
|
||||
lines.append(f"| {md_escape(term)} | {count} |")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"## 聚合控制信号",
|
||||
"",
|
||||
"| 控制/检验关键词 | 出现行数 |",
|
||||
"|---|---:|",
|
||||
]
|
||||
for term, count in control:
|
||||
lines.append(f"| {md_escape(term)} | {count} |")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"## 审核发现和整改要点",
|
||||
"",
|
||||
"客户审核 Action list 中对 FFC/PFMEA 的核心发现是：",
|
||||
"",
|
||||
"- 客户项目输入没有在内部充分整合。",
|
||||
"- 柔性扁平线缆 NTF 清单相关知识没有完整传递到新项目 PFMEA。",
|
||||
"- 部分 FFC 原材料及供应商管理不够系统，尤其是接地焊盘/GND pad 类物料，涉及供应商类别定义、图纸电气性能参数、PPAP Level 3、安全库存、货架有效期等。",
|
||||
"",
|
||||
"对应的 PFMEA 改进动作：",
|
||||
"",
|
||||
"- 新项目 PFMEA 必须强制对标最新 FFC NTF 清单。",
|
||||
"- 将 NTF 中的新技术、新材料、新工艺风险转成 PFMEA 的失效模式、失效影响、失效原因、预防控制和探测控制。",
|
||||
"- 对 GND pad、线材、端子、贴合/冲切/折弯/包装/标签等关键对象建立专项管控清单。",
|
||||
"- 图纸评审、供应商准入、PPAP 审核、库存和效期管理要形成闭环，避免 PFMEA 写了但控制计划、检验规范和供应商管理没有同步。",
|
||||
"",
|
||||
"## 可复用的问答口径",
|
||||
"",
|
||||
"- 如果客户问有没有 FFC/排线 PFMEA：可以说有，BJD 内部已有 AIAG & VDA 版本和项目版本的 FFC PFMEA 资料，可按客户项目/料号提交受控版本。",
|
||||
"- 如果客户追问是否覆盖新技术清单：应谨慎回答，内部审核资料曾指出 NTF 到新项目 PFMEA 的联动需要加强；现在正确做法是按最新 NTF 清单逐项复核后再提交。",
|
||||
"- 如果是 PPAP 提交：不要直接用通用版替代项目版，应确认客户项目、料号、工艺路线、特殊特性、控制计划和检验规范是否一致。",
|
||||
"",
|
||||
"## 抽取样例行",
|
||||
"",
|
||||
]
|
||||
|
||||
for item in parsed:
|
||||
rec = item["record"]
|
||||
rows = item["parsed"].get("candidate_rows", [])[:20]
|
||||
if not rows:
|
||||
continue
|
||||
lines += [f"### {rec.get('filename','')}", ""]
|
||||
for row in rows[:12]:
|
||||
lines.append(f"- `{row.get('sheet')}` R{row.get('row')}: {md_escape(row.get('text','')[:420])}")
|
||||
lines.append("")
|
||||
|
||||
lines += [
|
||||
"## 数据文件",
|
||||
"",
|
||||
f"- 结构化抽取结果：`_meta/{data_path.name}`",
|
||||
"- 原始附件路径保存在 `_meta/bjd-email-attachment-inventory-20260611.jsonl` 中。",
|
||||
]
|
||||
|
||||
page_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
update_vault_metadata(ts, records, parsed, page_path, data_path)
|
||||
|
||||
|
||||
def update_vault_metadata(
|
||||
ts: str,
|
||||
records: list[dict[str, Any]],
|
||||
parsed: list[dict[str, Any]],
|
||||
page_path: Path,
|
||||
data_path: Path,
|
||||
) -> None:
|
||||
rel_page = page_path.relative_to(VAULT).with_suffix("")
|
||||
index_path = VAULT / "index.md"
|
||||
if index_path.exists():
|
||||
text = index_path.read_text(encoding="utf-8")
|
||||
text = re.sub(r"最后更新：[^*]+", f"最后更新：{ts}", text)
|
||||
entry = (
|
||||
f"- [[{rel_page.as_posix()}]] - FFC/排线 PFMEA 源文件、AIAG & VDA FMEA、"
|
||||
"审核发现和整改口径整理。 ( #bjd #ffc #pfmea #quality )"
|
||||
)
|
||||
if str(rel_page) not in text:
|
||||
marker = "## 综合分析\n"
|
||||
if marker in text:
|
||||
text = text.replace(marker, marker + entry + "\n", 1)
|
||||
else:
|
||||
text += "\n## 综合分析\n" + entry + "\n"
|
||||
index_path.write_text(text, encoding="utf-8")
|
||||
|
||||
hot_path = VAULT / "hot.md"
|
||||
if hot_path.exists():
|
||||
hot = hot_path.read_text(encoding="utf-8")
|
||||
hot_entry = f"- [[{rel_page.as_posix()}]] - FFC/排线 PFMEA 知识整理。"
|
||||
if str(rel_page) not in hot:
|
||||
hot += "\n" + hot_entry + "\n"
|
||||
hot_path.write_text(hot, encoding="utf-8")
|
||||
|
||||
log_path = VAULT / "log.md"
|
||||
with log_path.open("a", encoding="utf-8") as f:
|
||||
f.write(
|
||||
"\n"
|
||||
f"- [{ts}] DATA_INGEST source=\"_meta/bjd-email-attachment-inventory-20260611.jsonl\" "
|
||||
f"project=\"bjd\" mode=\"ffc-pfmea-deep-extraction\" records={len(records)} "
|
||||
f"parsed={len(parsed)} page=\"{page_path.relative_to(VAULT).as_posix()}\" "
|
||||
f"data=\"{data_path.relative_to(VAULT).as_posix()}\" bot_visible=false\n"
|
||||
)
|
||||
|
||||
manifest_path = VAULT / ".manifest.json"
|
||||
manifest: dict[str, Any] = {}
|
||||
if manifest_path.exists():
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
manifest = {}
|
||||
source_key = "derived:ffc-pfmea-extraction-20260611"
|
||||
inv_hash = hashlib.sha256(INVENTORY.read_bytes()).hexdigest() if INVENTORY.exists() else ""
|
||||
manifest.setdefault("sources", {})[source_key] = {
|
||||
"ingested_at": ts,
|
||||
"original_path": str(INVENTORY),
|
||||
"source_type": "derived_email_attachment_pfmea",
|
||||
"doc_type": "quality_pfmea",
|
||||
"project": "bjd",
|
||||
"content_hash": "sha256:" + inv_hash if inv_hash else "",
|
||||
"records": len(records),
|
||||
"parsed": len(parsed),
|
||||
"pages_created": [page_path.relative_to(VAULT).as_posix()],
|
||||
"data_path": data_path.relative_to(VAULT).as_posix(),
|
||||
"bot_visible": False,
|
||||
}
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not INVENTORY.exists():
|
||||
print(f"missing inventory: {INVENTORY}", file=sys.stderr)
|
||||
return 2
|
||||
records = [r for r in read_jsonl(INVENTORY) if classify_file(r)]
|
||||
records.sort(key=lambda r: (r.get("date_end") or "", r.get("filename") or ""))
|
||||
parsed = []
|
||||
for rec in records:
|
||||
path = Path(rec["absolute_path"])
|
||||
if rec.get("ext") == ".xlsx":
|
||||
info = parse_xlsx(path)
|
||||
elif rec.get("ext") == ".xls":
|
||||
info = scan_legacy_xls(path)
|
||||
else:
|
||||
continue
|
||||
parsed.append({"record": rec, "parsed": info})
|
||||
write_outputs(records, parsed)
|
||||
print(json.dumps({"records": len(records), "parsed": len(parsed)}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,552 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ingest archived public competitor sources into the BJD mother vault."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
VAULT = Path("/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault")
|
||||
RESULTS = Path("/tmp/bjd_competitor_download_results.json")
|
||||
NOW = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def load_results() -> list[dict]:
|
||||
return json.loads(RESULTS.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def source_type(item: dict) -> str:
|
||||
if item.get("type"):
|
||||
return item["type"]
|
||||
name = item["name"].lower()
|
||||
if name.endswith(".pdf"):
|
||||
return "pdf"
|
||||
if name.endswith(".zip"):
|
||||
return "cad"
|
||||
return "web"
|
||||
|
||||
|
||||
def doc_type(item: dict) -> str:
|
||||
name = item["name"].lower()
|
||||
if "drawing" in name or "dxf" in name:
|
||||
return "drawing"
|
||||
if "step" in name:
|
||||
return "cad_model"
|
||||
if "catalog" in name or "guide" in name:
|
||||
return "catalog"
|
||||
if "manual" in name or "instruction" in name:
|
||||
return "instruction_manual"
|
||||
if "datasheet" in name or "spec-sheet" in name or "whitepaper" in name:
|
||||
return "spec_sheet"
|
||||
if "page" in name:
|
||||
return "product_page"
|
||||
return "product_parameter"
|
||||
|
||||
|
||||
def product_line(item: dict) -> list[str]:
|
||||
series = item.get("series", "")
|
||||
brand = item.get("brand", "")
|
||||
if brand in {"Rosenberger"} or "hfm" in item["name"].lower():
|
||||
return ["Mini FAKRA", "High Speed"]
|
||||
if brand == "TE":
|
||||
return ["I/O", "WTB", "Hybrid connector"]
|
||||
if "mini50" in item["name"].lower():
|
||||
return ["WTB", "Automotive wire-to-board"]
|
||||
if series in {"FH52", "FH52K", "FH75", "FH64MA", "6817"}:
|
||||
return ["FPC", "FFC"]
|
||||
if brand == "IRISO":
|
||||
return ["BTB", "FPC", "WTB"]
|
||||
return ["Automotive connectors"]
|
||||
|
||||
|
||||
def related_page(item: dict) -> str:
|
||||
brand = item.get("brand", "")
|
||||
series = item.get("series", "")
|
||||
name = item["name"].lower()
|
||||
if brand == "Hirose" and series in {"FH52", "FH52K"}:
|
||||
return "projects/bjd/references/hirose-fh52-fh52k-public-sources-20260605.md"
|
||||
if brand == "Hirose" and series == "FH75":
|
||||
return "projects/bjd/references/hirose-fh75-fh75m-automotive-fpc-connector.md"
|
||||
if brand in {"Hirose", "Kyocera"} and series in {"FH64MA", "6817"}:
|
||||
return "projects/bjd/references/hirose-fh64ma-kyocera-6817-fpc-connectors.md"
|
||||
if brand == "TE" or "73pos" in name:
|
||||
return "projects/bjd/references/te-2188698-73pin-hybrid-connector-public-sources-20260605.md"
|
||||
if brand == "Molex":
|
||||
return "projects/bjd/references/molex-mini50-connector-system.md"
|
||||
if brand == "IRISO":
|
||||
return "projects/bjd/references/iriso-public-competitor-sources-20260605.md"
|
||||
if brand == "Rosenberger":
|
||||
return "projects/bjd/references/bjd-mini-fakra-product-and-test-evidence.md"
|
||||
return "projects/bjd/references/competitor-public-source-archive-20260605.md"
|
||||
|
||||
|
||||
def summary_for(item: dict) -> str:
|
||||
brand = item.get("brand", "Competitor")
|
||||
series = item.get("series", "")
|
||||
dtype = doc_type(item).replace("_", " ")
|
||||
if series:
|
||||
return f"{brand} {series} official public {dtype} archived for BJD competitor lookup and replacement evaluation."
|
||||
return f"{brand} official public {dtype} archived for BJD competitor lookup and replacement evaluation."
|
||||
|
||||
|
||||
def registry_records(items: list[dict]) -> list[dict]:
|
||||
records = []
|
||||
for item in items:
|
||||
if not item.get("ok"):
|
||||
continue
|
||||
brand = item.get("brand", "unknown")
|
||||
stype = source_type(item)
|
||||
rec = {
|
||||
"record_type": "source_archive",
|
||||
"source_id": f"sha256:{item['sha256']}",
|
||||
"archived_at": NOW,
|
||||
"archive_path": item["archive_path"],
|
||||
"original_path": item["url"],
|
||||
"original_name": item["name"],
|
||||
"source_type": stype,
|
||||
"doc_type": doc_type(item),
|
||||
"business_categories": [f"competitors/{brand.lower()}", "products/automotive-connectors"],
|
||||
"product_lines": product_line(item),
|
||||
"customers": [],
|
||||
"competitors": [brand],
|
||||
"standards": [],
|
||||
"market_tags": ["market/automotive"],
|
||||
"title": item["name"].rsplit(".", 1)[0],
|
||||
"summary": summary_for(item),
|
||||
"visibility": "internal",
|
||||
"data_classification": "public-web",
|
||||
"allowed_bots": ["bjd-general-bot"],
|
||||
"classification_confidence": 0.84,
|
||||
"classification_evidence": ["official vendor public source", f"downloaded from {item['url']}"],
|
||||
"catalog_paths": [f"_sources/catalog/bjd/competitors/{brand.lower()}.md"],
|
||||
"project": "bjd",
|
||||
"size_bytes": item.get("size_bytes"),
|
||||
"modified_at": None,
|
||||
"related_pages": [related_page(item)],
|
||||
"bjd_part_numbers": [],
|
||||
"competitor_part_numbers": [item.get("series")] if item.get("series") else [],
|
||||
"series": [item.get("series")] if item.get("series") else [],
|
||||
"pin_count": 73 if "73pos" in item["name"].lower() else None,
|
||||
"needs_review": 0,
|
||||
}
|
||||
records.append(rec)
|
||||
return records
|
||||
|
||||
|
||||
def append_registry(records: list[dict]) -> int:
|
||||
path = VAULT / "_meta/source-registry.jsonl"
|
||||
existing = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
existing_ids = set()
|
||||
for line in existing.splitlines():
|
||||
try:
|
||||
existing_ids.add(json.loads(line).get("source_id"))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
new = [r for r in records if r["source_id"] not in existing_ids]
|
||||
if new:
|
||||
with path.open("a", encoding="utf-8") as f:
|
||||
for rec in new:
|
||||
f.write(json.dumps(rec, ensure_ascii=False, sort_keys=False) + "\n")
|
||||
return len(new)
|
||||
|
||||
|
||||
def md_link(item: dict) -> str:
|
||||
path = item.get("archive_path")
|
||||
if path:
|
||||
return f"[{item['name']}]({path})"
|
||||
return f"[{item['name']}]({item['url']})"
|
||||
|
||||
|
||||
def registry_competitor_items() -> list[dict]:
|
||||
path = VAULT / "_meta/source-registry.jsonl"
|
||||
if not path.exists():
|
||||
return []
|
||||
items = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
categories = rec.get("business_categories") or []
|
||||
competitors = rec.get("competitors") or []
|
||||
if not any(str(c).startswith("competitors/") for c in categories) and not competitors:
|
||||
continue
|
||||
vendor = competitors[0] if competitors else None
|
||||
if not vendor:
|
||||
for c in categories:
|
||||
if str(c).startswith("competitors/"):
|
||||
vendor = str(c).split("/", 1)[1].upper()
|
||||
break
|
||||
if not vendor:
|
||||
continue
|
||||
item = {
|
||||
"ok": True,
|
||||
"name": rec.get("original_name") or rec.get("title") or rec.get("source_id"),
|
||||
"type": rec.get("source_type") or "web",
|
||||
"brand": vendor,
|
||||
"series": (rec.get("series") or rec.get("competitor_part_numbers") or [""])[0],
|
||||
"archive_path": rec.get("archive_path"),
|
||||
"size_bytes": rec.get("size_bytes") or "",
|
||||
"url": rec.get("original_path") or "",
|
||||
"_registry_doc_type": rec.get("doc_type") or "product_parameter",
|
||||
}
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def vendor_catalog(vendor: str, items: list[dict]) -> str:
|
||||
good = [i for i in items if i.get("ok") and i.get("brand") == vendor]
|
||||
failed = [i for i in items if not i.get("ok") and i.get("brand") == vendor]
|
||||
slug = vendor.lower()
|
||||
title = f"{vendor} 竞品源文件目录"
|
||||
rows = []
|
||||
for item in sorted(good, key=lambda x: (x.get("series") or "", x["name"])):
|
||||
rows.append(
|
||||
f"| {md_link(item)} | {item.get('_registry_doc_type') or doc_type(item)} | {item.get('series','')} | {item.get('size_bytes','')} | [source]({item['url']}) |"
|
||||
)
|
||||
failed_lines = ""
|
||||
if failed:
|
||||
failed_lines = "\n\n## 待补链接\n\n" + "\n".join(
|
||||
f"- {i['name']} - 下载失败，需从官方页面重新确认直链：{i['url']}" for i in failed
|
||||
)
|
||||
return f"""---
|
||||
title: {title}
|
||||
category: meta
|
||||
tags: [source-catalog, bjd, competitors, {slug}, visibility/internal]
|
||||
sources:
|
||||
- _meta/source-registry.jsonl
|
||||
created: 2026-06-05
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# {title}
|
||||
|
||||
本页记录已归档到母库的 {vendor} 官方公开竞品资料。资料可用于竞品识别、替代评估、图纸/规格书检索；不要把“有公开资料”自动解释为 BJD 已完成 drop-in 替代。
|
||||
|
||||
| 归档文件 | 类型 | 系列 | 大小 bytes | 原始官方链接 |
|
||||
|---|---|---:|---:|---|
|
||||
{chr(10).join(rows) if rows else '| 暂无 | | | | |'}
|
||||
{failed_lines}
|
||||
|
||||
## 相关页面
|
||||
|
||||
- [[projects/bjd/references/competitor-public-source-archive-20260605]]
|
||||
- [[projects/bjd/references/competitor-cross-reference-map]]
|
||||
"""
|
||||
|
||||
|
||||
def index_catalog(items: list[dict]) -> str:
|
||||
vendors = sorted({i.get("brand", "unknown") for i in items if i.get("brand")})
|
||||
lines = []
|
||||
for vendor in vendors:
|
||||
good = sum(1 for i in items if i.get("brand") == vendor and i.get("ok"))
|
||||
failed = sum(1 for i in items if i.get("brand") == vendor and not i.get("ok"))
|
||||
note = f"{good} 个已归档"
|
||||
if failed:
|
||||
note += f"，{failed} 个待补"
|
||||
lines.append(f"- [[_sources/catalog/bjd/competitors/{vendor.lower()}]] - {vendor}，{note}。")
|
||||
return f"""---
|
||||
title: BJD 竞品资料源文件目录
|
||||
category: meta
|
||||
tags: [source-catalog, bjd, competitors, visibility/internal]
|
||||
sources:
|
||||
- _meta/source-registry.jsonl
|
||||
created: 2026-06-03
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# BJD 竞品资料源文件目录
|
||||
|
||||
该目录记录 BJD wiki 已归档的竞品公开资料，包括官方产品页、目录、规格书、2D 图纸、CAD/STEP/DFX 压缩包和操作说明。归档资料用于后续检索和替代评估。
|
||||
|
||||
{chr(10).join(lines)}
|
||||
|
||||
## 总索引
|
||||
|
||||
- [[projects/bjd/references/competitor-public-source-archive-20260605]] - 2026-06-05 竞品公开资料归档总览。
|
||||
"""
|
||||
|
||||
|
||||
def master_page(items: list[dict]) -> str:
|
||||
by_vendor: dict[str, list[dict]] = {}
|
||||
for item in items:
|
||||
if item.get("ok"):
|
||||
by_vendor.setdefault(item.get("brand", "unknown"), []).append(item)
|
||||
sections = []
|
||||
for vendor, rows in sorted(by_vendor.items()):
|
||||
lines = [
|
||||
f"| {i.get('series','')} | {md_link(i)} | {doc_type(i)} | [official]({i['url']}) |"
|
||||
for i in sorted(rows, key=lambda x: (x.get("series") or "", x["name"]))
|
||||
]
|
||||
sections.append(f"## {vendor}\n\n| 系列 | 归档文件 | 类型 | 原始链接 |\n|---|---|---|---|\n" + "\n".join(lines))
|
||||
failed = [i for i in items if not i.get("ok")]
|
||||
failed_text = "\n".join(f"- {i.get('brand')} {i.get('series')}: {i['name']} - {i['url']}" for i in failed)
|
||||
return f"""---
|
||||
title: 2026-06-05 竞品公开资料归档总览
|
||||
category: references
|
||||
tags: [bjd, competitors, public-sources, drawings, catalogs, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [products, sales, engineering]
|
||||
data_classification: public-web
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/source-registry.jsonl
|
||||
summary: 2026-06-05 联网补充并归档 Hirose、TE、IRISO、Kyocera、Rosenberger 等竞品官方公开资料，覆盖产品页、目录、规格书、2D 图纸和 CAD/STEP/DFX 文件。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/references/competitor-cross-reference-map]]"
|
||||
type: supports
|
||||
- target: "[[projects/bjd/references/bjd-fpc-cross-reference-20260422]]"
|
||||
type: supports
|
||||
- target: "[[projects/bjd/references/bjd-io-cross-reference-20260422]]"
|
||||
type: supports
|
||||
- target: "[[projects/bjd/references/bjd-wtb-hs-cross-reference-20260422]]"
|
||||
type: supports
|
||||
provenance:
|
||||
extracted: 0.88
|
||||
inferred: 0.09
|
||||
ambiguous: 0.03
|
||||
base_confidence: 0.87
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: core
|
||||
created: 2026-06-05T00:00:00Z
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# 2026-06-05 竞品公开资料归档总览
|
||||
|
||||
本页是竞品官方公开资料的检索入口。已归档文件保存在 `_sources/archive/bjd/2026-06/competitor/`，并写入 `_meta/source-registry.jsonl`。这些资料用于识别竞品结构、pin 数、规格书、图纸、land pattern、CAD/STEP/DFX 文件和替代评估边界；不能单独作为 BJD 已经完成完全替代或 drop-in 的证据。
|
||||
|
||||
{chr(10).join(sections)}
|
||||
|
||||
## 待补链接
|
||||
|
||||
以下官方直链下载失败，已记录为待补，不计入已归档文件：
|
||||
|
||||
{failed_text if failed_text else "- 无。"}
|
||||
|
||||
## 使用口径
|
||||
|
||||
- 客户只给竞品料号时，先查本页或对应厂牌 catalog，确认官方图纸/规格书是否已归档。
|
||||
- 有 2D 图纸、STEP/DXF 时，仍需工程复核 FPC/FFC 厚度、tab、land pattern、焊盘、插入方向、锁扣、温度等级和可靠性要求。
|
||||
- 公开竞品资料只用于技术对标；对外报价、具体客户价格、交货数量仍按 [[projects/bjd/references/bjd-general-bot-answer-policy]] 的敏感信息边界处理。
|
||||
|
||||
## 相关页面
|
||||
|
||||
- [[projects/bjd/references/competitor-cross-reference-map]]
|
||||
- [[projects/bjd/references/hirose-fh52-fh52k-public-sources-20260605]]
|
||||
- [[projects/bjd/references/te-2188698-73pin-hybrid-connector-public-sources-20260605]]
|
||||
- [[projects/bjd/references/iriso-public-competitor-sources-20260605]]
|
||||
- [[projects/bjd/references/hirose-fh75-fh75m-automotive-fpc-connector]]
|
||||
- [[projects/bjd/references/hirose-fh64ma-kyocera-6817-fpc-connectors]]
|
||||
- [[projects/bjd/references/molex-mini50-connector-system]]
|
||||
"""
|
||||
|
||||
|
||||
def fh52_page(items: list[dict]) -> str:
|
||||
rows = [i for i in items if i.get("ok") and i.get("brand") == "Hirose" and i.get("series") in {"FH52", "FH52K"}]
|
||||
lines = "\n".join(f"| {i.get('series')} | {md_link(i)} | {doc_type(i)} | [official]({i['url']}) |" for i in rows)
|
||||
return f"""---
|
||||
title: Hirose FH52/FH52K 官方公开资料归档
|
||||
category: references
|
||||
tags: [hirose, fh52, fh52k, fpc, ffc, cross-reference, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [products, sales, engineering]
|
||||
data_classification: public-web
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/source-registry.jsonl
|
||||
summary: Hirose FH52/FH52K 官方产品页、目录、2D 图纸、规格书和 STEP 资料归档，用于 FH52-30S-0.5SH 等料号的替代评估。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/references/bjd-fpc-cross-reference-20260422]]"
|
||||
type: supports
|
||||
- target: "[[projects/bjd/references/hirose-fh75-fh75m-automotive-fpc-connector]]"
|
||||
type: related_to
|
||||
provenance:
|
||||
extracted: 0.86
|
||||
inferred: 0.10
|
||||
ambiguous: 0.04
|
||||
base_confidence: 0.86
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: core
|
||||
created: 2026-06-05T00:00:00Z
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# Hirose FH52/FH52K 官方公开资料归档
|
||||
|
||||
客户问 `FH52-30S-0.5SH`、`FH52K` 或 FH52/FH75 相邻替代时，优先查本页归档的官方产品页、目录、2D 图纸、规格书和 STEP 文件。BJD 已有 FPC `0.5-18` / `101049-X0XXX0` 对 HRS `FH52/FH28` 的对照线索；具体 30Pin 候选如 `101049-203050` 仍需按客户图纸复核 FPC 厚度、tab、land pattern、下接点/上接点、锁扣方向、连接器高度和包装。
|
||||
|
||||
| 系列 | 归档文件 | 类型 | 原始链接 |
|
||||
|---|---|---|---|
|
||||
{lines}
|
||||
|
||||
## 回答边界
|
||||
|
||||
- 可以说：BJD 有 FH52/FH28 对照链路，可从 FPC 0.5-18 系列评估。
|
||||
- 不要直接承诺：完全 drop-in 或无需改板，除非工程已比对 Hirose 2D 图纸和客户 PCB/FPC。
|
||||
- 网上能找到的 Hirose 图纸、规格书和 STEP，已尽量归档；后续客户只给 Hirose 料号时，先检索本页，不必第一时间麻烦客户。
|
||||
|
||||
## 相关页面
|
||||
|
||||
- [[projects/bjd/references/bjd-fpc-cross-reference-20260422]]
|
||||
- [[projects/bjd/references/hirose-fh75-fh75m-automotive-fpc-connector]]
|
||||
- [[projects/bjd/references/competitor-public-source-archive-20260605]]
|
||||
"""
|
||||
|
||||
|
||||
def te_page(items: list[dict]) -> str:
|
||||
rows = [i for i in items if i.get("ok") and i.get("brand") == "TE"]
|
||||
lines = "\n".join(f"| {i.get('series')} | {md_link(i)} | {doc_type(i)} | [official]({i['url']}) |" for i in rows)
|
||||
return f"""---
|
||||
title: TE 2188698 / 73Pin 混合端子官方公开资料归档
|
||||
category: references
|
||||
tags: [te, 2188698, 73pin, hybrid-connector, io, cross-reference, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [products, sales, engineering]
|
||||
data_classification: public-web
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/source-registry.jsonl
|
||||
summary: TE 2188698 相关 customer drawing、73pos instruction sheet 和 2D DXF 文件归档，用于 BJD 73Pin 混合端子产品对标。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/references/bjd-io-cross-reference-20260422]]"
|
||||
type: supports
|
||||
- target: "[[projects/bjd/references/competitor-cross-reference-map]]"
|
||||
type: related_to
|
||||
provenance:
|
||||
extracted: 0.84
|
||||
inferred: 0.10
|
||||
ambiguous: 0.06
|
||||
base_confidence: 0.84
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: core
|
||||
created: 2026-06-05T00:00:00Z
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# TE 2188698 / 73Pin 混合端子官方公开资料归档
|
||||
|
||||
本页保存 TE `2188698` 相关公开图纸和 73pos instruction sheet 资料，用于评估 BJD `P2.0-2-73PIN` / `140084-417320` 这类 73Pin 混合端子产品。既有 wiki 记录显示该产品用于对标 TE `2420419-1` 方向，并与 TE `2188698` / `2188701` 相关配套项有关；具体替代需工程按 customer drawing、DXF、端子/壳体配合和客户应用确认。
|
||||
|
||||
| 系列 | 归档文件 | 类型 | 原始链接 |
|
||||
|---|---|---|---|
|
||||
{lines}
|
||||
|
||||
## 关键使用口径
|
||||
|
||||
- 可以把本页作为 TE 图纸和 2D CAD 检索入口。
|
||||
- 不应仅凭 73Pin 和系列名承诺完全替代；混合端子需要复核机械界面、端子排布、锁止结构、线端/板端配套、材料和测试要求。
|
||||
- TE 产品页 HTML 本次下载失败，已在厂牌 catalog 标为待补；PDF/DXF 资料已归档。
|
||||
|
||||
## 相关页面
|
||||
|
||||
- [[projects/bjd/references/bjd-io-cross-reference-20260422]]
|
||||
- [[projects/bjd/references/competitor-cross-reference-map]]
|
||||
- [[projects/bjd/references/competitor-public-source-archive-20260605]]
|
||||
"""
|
||||
|
||||
|
||||
def iriso_page(items: list[dict]) -> str:
|
||||
rows = [i for i in items if i.get("ok") and i.get("brand") == "IRISO"]
|
||||
lines = "\n".join(f"| {i.get('series')} | {md_link(i)} | {doc_type(i)} | [official]({i['url']}) |" for i in rows)
|
||||
return f"""---
|
||||
title: IRISO 公开竞品资料归档
|
||||
category: references
|
||||
tags: [iriso, btb, fpc, wtb, cross-reference, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [products, sales, engineering]
|
||||
data_classification: public-web
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/source-registry.jsonl
|
||||
summary: IRISO 13065、11507/11607、9827/9828/9860 和产品导览公开资料归档，用于 BJD WTB/BTB/FPC 竞品替代检索。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/references/bjd-btb-cross-reference-20260422]]"
|
||||
type: supports
|
||||
- target: "[[projects/bjd/references/bjd-wtb-hs-cross-reference-20260422]]"
|
||||
type: supports
|
||||
provenance:
|
||||
extracted: 0.84
|
||||
inferred: 0.12
|
||||
ambiguous: 0.04
|
||||
base_confidence: 0.84
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: core
|
||||
created: 2026-06-05T00:00:00Z
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# IRISO 公开竞品资料归档
|
||||
|
||||
本页汇总 IRISO 官方公开资料。既有 BJD cross-reference 中 IRISO 涉及 FPC、浮动 BTB 和 WTB 多条线，尤其 `9827/9828` 浮动 BTB、`13065` WTB 以及 `11507/11607` 新闻资料。使用时应回到具体 BJD 系列和 pin 数做工程比对。
|
||||
|
||||
| 系列 | 归档文件 | 类型 | 原始链接 |
|
||||
|---|---|---|---|
|
||||
{lines}
|
||||
|
||||
## 检索建议
|
||||
|
||||
- 问 `IRISO 9827/9828/9860`：优先查浮动 BTB 归档 PDF，再回到 BJD BTB 对照页。
|
||||
- 问 `IRISO 13065`：先查 13065 series page 和白皮书，再回到 BJD WTB/HS 对照页。
|
||||
- 问 `IRISO 11507/11607`：本次归档的是官方新闻页，若后续找到单独 drawing/spec PDF，应继续补档。
|
||||
|
||||
## 相关页面
|
||||
|
||||
- [[projects/bjd/references/bjd-btb-cross-reference-20260422]]
|
||||
- [[projects/bjd/references/bjd-wtb-hs-cross-reference-20260422]]
|
||||
- [[projects/bjd/references/competitor-public-source-archive-20260605]]
|
||||
"""
|
||||
|
||||
|
||||
def write_pages(items: list[dict]) -> list[str]:
|
||||
written = []
|
||||
catalog_dir = VAULT / "_sources/catalog/bjd/competitors"
|
||||
catalog_items = registry_competitor_items()
|
||||
seen = {(i.get("brand"), i.get("name"), i.get("archive_path"), i.get("url")) for i in catalog_items}
|
||||
for item in items:
|
||||
key = (item.get("brand"), item.get("name"), item.get("archive_path"), item.get("url"))
|
||||
if key not in seen:
|
||||
catalog_items.append(item)
|
||||
seen.add(key)
|
||||
for vendor in sorted({i.get("brand") for i in catalog_items if i.get("brand")}):
|
||||
path = catalog_dir / f"{vendor.lower()}.md"
|
||||
path.write_text(vendor_catalog(vendor, catalog_items), encoding="utf-8")
|
||||
written.append(str(path))
|
||||
(catalog_dir / "index.md").write_text(index_catalog(catalog_items), encoding="utf-8")
|
||||
written.append(str(catalog_dir / "index.md"))
|
||||
pages = {
|
||||
"projects/bjd/references/competitor-public-source-archive-20260605.md": master_page(items),
|
||||
"projects/bjd/references/hirose-fh52-fh52k-public-sources-20260605.md": fh52_page(items),
|
||||
"projects/bjd/references/te-2188698-73pin-hybrid-connector-public-sources-20260605.md": te_page(items),
|
||||
"projects/bjd/references/iriso-public-competitor-sources-20260605.md": iriso_page(items),
|
||||
}
|
||||
for rel, body in pages.items():
|
||||
path = VAULT / rel
|
||||
path.write_text(body, encoding="utf-8")
|
||||
written.append(str(path))
|
||||
return written
|
||||
|
||||
|
||||
def main() -> None:
|
||||
items = load_results()
|
||||
records = registry_records(items)
|
||||
added = append_registry(records)
|
||||
written = write_pages(items)
|
||||
print(json.dumps({
|
||||
"download_ok": sum(1 for i in items if i.get("ok")),
|
||||
"download_fail": sum(1 for i in items if not i.get("ok")),
|
||||
"registry_added": added,
|
||||
"pages_written": written,
|
||||
}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
import collections, hashlib, html, json, re, zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
VAULT=Path('/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault')
|
||||
SOURCE=Path('/home/nac/repo/emails_export/parsed/llm_input.jsonl')
|
||||
ATTACH_BASE=Path('/home/nac/repo/emails_export')
|
||||
RUN_DATE='20260611'
|
||||
NOW=datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00','Z')
|
||||
TODAY=NOW[:10]
|
||||
META=VAULT/'_meta'
|
||||
INV=META/f'bjd-email-attachment-inventory-{RUN_DATE}.jsonl'
|
||||
SNIP=META/f'bjd-email-attachment-extracted-snippets-{RUN_DATE}.jsonl'
|
||||
SUMMARY=META/f'bjd-email-attachment-ingest-{RUN_DATE}.json'
|
||||
KW_CLASSES={
|
||||
'drawing':['图纸','drawing','2d','3d','dwg','step','stp','结构图','线规','工程图'],
|
||||
'spec':['规格书','spec','datasheet','承认书','acknowledge','approval','承認書','产品资料'],
|
||||
'quote':['报价','quote','rfq','询价','quotation','price','价格','单价'],
|
||||
'order':['订单','po','purchase order','cgdd','合同','采购单','订单合同','order'],
|
||||
'forecast':['forecast','fcst','预测','mrp','排期','交期','delivery schedule','需求计划','提货计划'],
|
||||
'quality':['8d','不良','客诉','quality','complaint','退货','改善','异常','索赔','报废','返工'],
|
||||
'compliance':['rohs','reach','halogen','无卤','ppap','iatf','audit','审核','camds','msds','sgs','vda','环保','gp'],
|
||||
}
|
||||
BUSINESS_TERMS=['BJD','北极大','Beijida','RFQ','报价','询价','订单','交期','样品','图纸','规格书','承认书','PPAP','RoHS','REACH','8D','客诉','不良','forecast','MRP','SOP','VAVE','替代','Hirose','Molex','TE','JAE','Kyocera','IRISO','零跑','理想','巨一','Marelli','Valeo','Visteon','BYD','比亚迪']
|
||||
PARSE_EXT={'.xlsx','.xlsm','.csv','.txt','.htm','.html','.xml','.docx','.pptx','.eml'}
|
||||
BINARY_TEXT_EXT={'.pdf','.xls','.doc'}
|
||||
|
||||
def clean(s:str, limit:int=1200)->str:
|
||||
s=html.unescape(str(s or '')).replace('\x00',' ')
|
||||
s=re.sub(r'\s+',' ',s).strip()
|
||||
return s[:limit]
|
||||
|
||||
def cls_for(fn:str, subject:str='')->str:
|
||||
low=(fn+' '+subject).lower()
|
||||
for k,vs in KW_CLASSES.items():
|
||||
if any(v.lower() in low for v in vs): return k
|
||||
return 'other'
|
||||
|
||||
def domain_of(person:Any)->str|None:
|
||||
if not isinstance(person,dict): return None
|
||||
e=(person.get('email') or '').lower()
|
||||
return e.split('@')[-1] if '@' in e else None
|
||||
|
||||
def safe_decode(data:bytes, limit:int=2_000_000)->str:
|
||||
data=data[:limit]
|
||||
for enc in ('utf-8','gb18030','big5','latin1'):
|
||||
try: return data.decode(enc, errors='ignore')
|
||||
except Exception: pass
|
||||
return data.decode('utf-8', errors='ignore')
|
||||
|
||||
def extract_xml_text_from_zip(path:Path, suffixes:tuple[str,...], limit:int=12000)->str:
|
||||
vals=[]
|
||||
try:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
names=[n for n in z.namelist() if n.endswith(suffixes)]
|
||||
for n in names[:80]:
|
||||
try:
|
||||
root=ET.fromstring(z.read(n))
|
||||
for elem in root.iter():
|
||||
if elem.text and elem.tag.rsplit('}',1)[-1] in {'t','v'}:
|
||||
vals.append(elem.text)
|
||||
if sum(map(len,vals))>limit: return clean(' '.join(vals), limit)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
return ''
|
||||
return clean(' '.join(vals), limit)
|
||||
|
||||
def extract_xlsx(path:Path, limit:int=12000)->str:
|
||||
vals=[]; shared=[]
|
||||
try:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
if 'xl/sharedStrings.xml' in z.namelist():
|
||||
try:
|
||||
root=ET.fromstring(z.read('xl/sharedStrings.xml'))
|
||||
for si in root.iter():
|
||||
if si.tag.rsplit('}',1)[-1]=='si':
|
||||
shared.append(''.join(t.text or '' for t in si.iter() if t.tag.rsplit('}',1)[-1]=='t'))
|
||||
except Exception: pass
|
||||
sheets=[n for n in z.namelist() if n.startswith('xl/worksheets/sheet') and n.endswith('.xml')]
|
||||
for n in sheets[:20]:
|
||||
try:
|
||||
root=ET.fromstring(z.read(n))
|
||||
for c in root.iter():
|
||||
if c.tag.rsplit('}',1)[-1] != 'c': continue
|
||||
t=c.attrib.get('t')
|
||||
v=None
|
||||
for ch in c:
|
||||
if ch.tag.rsplit('}',1)[-1]=='v': v=ch.text; break
|
||||
if v is None: continue
|
||||
if t=='s':
|
||||
try: vals.append(shared[int(v)])
|
||||
except Exception: vals.append(v)
|
||||
else: vals.append(v)
|
||||
if sum(map(len,vals))>limit: return clean(' '.join(vals), limit)
|
||||
except Exception: continue
|
||||
except Exception:
|
||||
return ''
|
||||
return clean(' '.join(vals), limit)
|
||||
|
||||
def extract_text_file(path:Path, limit:int=12000)->str:
|
||||
try: text=safe_decode(path.read_bytes(), 1_500_000)
|
||||
except Exception: return ''
|
||||
return clean(text, limit)
|
||||
|
||||
def extract_html(path:Path, limit:int=12000)->str:
|
||||
try: text=safe_decode(path.read_bytes(), 1_500_000)
|
||||
except Exception: return ''
|
||||
text=re.sub(r'<(script|style)[^>]*>.*?</\1>',' ',text,flags=re.I|re.S)
|
||||
text=re.sub(r'<[^>]+>',' ',text)
|
||||
return clean(text, limit)
|
||||
|
||||
def extract_binary_strings(path:Path, limit:int=8000)->str:
|
||||
try: data=path.read_bytes()[:3_000_000]
|
||||
except Exception: return ''
|
||||
txt=safe_decode(data, 3_000_000)
|
||||
runs=re.findall(r'[\u4e00-\u9fffA-Za-z0-9_@./:：;；,，()（）\-\s]{8,}', txt)
|
||||
return clean(' '.join(runs[:300]), limit)
|
||||
|
||||
def extract(path:Path, ext:str)->tuple[str,str]:
|
||||
try:
|
||||
if ext in {'.xlsx','.xlsm'}: return extract_xlsx(path), 'xlsx-xml'
|
||||
if ext == '.docx': return extract_xml_text_from_zip(path, ('document.xml',), 12000), 'docx-xml'
|
||||
if ext == '.pptx': return extract_xml_text_from_zip(path, ('.xml',), 12000), 'pptx-xml'
|
||||
if ext in {'.htm','.html'}: return extract_html(path), 'html'
|
||||
if ext in {'.txt','.csv','.xml','.eml'}: return extract_text_file(path), 'text'
|
||||
if ext in BINARY_TEXT_EXT: return extract_binary_strings(path), 'binary-strings'
|
||||
except Exception as e:
|
||||
return '', 'error:'+type(e).__name__
|
||||
return '', 'metadata-only'
|
||||
|
||||
def should_extract(ext:str, klass:str, size:int, filename:str, subject:str)->bool:
|
||||
low=(filename+' '+subject).lower()
|
||||
important = klass != 'other' or any(t.lower() in low for t in BUSINESS_TERMS)
|
||||
if ext in PARSE_EXT and size <= 4_000_000 and important: return True
|
||||
if ext in BINARY_TEXT_EXT and size <= 3_000_000 and klass in {'quote','order','forecast','quality','compliance','spec','drawing'}: return True
|
||||
return False
|
||||
|
||||
def frontmatter(title,tags,summary):
|
||||
return """---
|
||||
title: {title}
|
||||
category: references
|
||||
project: bjd
|
||||
tags: {tags}
|
||||
visibility: internal
|
||||
data_classification: commercial-confidential
|
||||
allowed_bots: []
|
||||
excluded_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/bjd-email-attachment-ingest-{run}.json
|
||||
- _meta/bjd-email-attachment-inventory-{run}.jsonl
|
||||
- _meta/bjd-email-attachment-extracted-snippets-{run}.jsonl
|
||||
summary: >-
|
||||
{summary}
|
||||
provenance:
|
||||
extracted: 0.76
|
||||
inferred: 0.16
|
||||
ambiguous: 0.08
|
||||
base_confidence: 0.70
|
||||
lifecycle: draft
|
||||
lifecycle_changed: {today}
|
||||
created: {now}
|
||||
updated: {now}
|
||||
---
|
||||
|
||||
""".format(title=title,tags=tags,summary=summary,run=RUN_DATE,today=TODAY,now=NOW)
|
||||
|
||||
def esc(x): return str(x or '').replace('|',' / ').replace('\n',' ')
|
||||
def md(items, empty='未提取'): return '、'.join(items) if items else empty
|
||||
|
||||
def main():
|
||||
META.mkdir(parents=True, exist_ok=True)
|
||||
by_sha={}; refs=0
|
||||
with SOURCE.open(encoding='utf-8', errors='replace') as f:
|
||||
for line in f:
|
||||
o=json.loads(line); refs+=len(o.get('attachments') or [])
|
||||
tid=o.get('thread_id'); subject=clean(o.get('suggested_title_from_subject',''),220); end=o.get('date_end')
|
||||
domains=collections.Counter()
|
||||
for m in o.get('messages') or []:
|
||||
if not isinstance(m,dict): continue
|
||||
for p in [m.get('from',{})]+(m.get('to') or [])+(m.get('cc') or []):
|
||||
d=domain_of(p)
|
||||
if d and d not in {'beijida.com','bjdffc.com'}: domains[d]+=1
|
||||
for a in o.get('attachments') or []:
|
||||
sha=a.get('content_sha1') or a.get('attachment_id')
|
||||
fn=a.get('filename') or ''
|
||||
ext=Path(fn).suffix.lower() or '[none]'
|
||||
path=ATTACH_BASE/(a.get('local_path') or '')
|
||||
rec=by_sha.get(sha)
|
||||
if not rec:
|
||||
klass=cls_for(fn, subject)
|
||||
rec={'content_sha1':sha,'attachment_id':a.get('attachment_id'),'filename':fn,'ext':ext,'content_type':a.get('content_type'),'size_bytes':a.get('size_bytes') or (path.stat().st_size if path.exists() else None),'local_path':a.get('local_path'),'absolute_path':str(path),'exists':path.exists(),'class':klass,'thread_count':0,'ref_count':0,'threads':[],'subjects':[],'domains':collections.Counter()}
|
||||
by_sha[sha]=rec
|
||||
rec['ref_count']+=1
|
||||
if tid not in rec['threads']:
|
||||
rec['thread_count']+=1
|
||||
if len(rec['threads'])<8: rec['threads'].append(tid)
|
||||
if subject and subject not in rec['subjects'] and len(rec['subjects'])<5: rec['subjects'].append(subject)
|
||||
rec['domains'].update(domains)
|
||||
if end and not rec.get('date_end'): rec['date_end']=end
|
||||
stats={'refs':refs,'unique':len(by_sha),'exists':0,'missing':0,'bytes':0}
|
||||
ext_counts=collections.Counter(); class_counts=collections.Counter(); parser_counts=collections.Counter(); extracted_counts=collections.Counter()
|
||||
rows=[]; snippets=[]
|
||||
with INV.open('w', encoding='utf-8') as inv, SNIP.open('w', encoding='utf-8') as snip:
|
||||
for rec in by_sha.values():
|
||||
p=Path(rec['absolute_path']); size=rec.get('size_bytes') or 0
|
||||
if rec['exists']: stats['exists']+=1; stats['bytes']+=int(size or 0)
|
||||
else: stats['missing']+=1
|
||||
ext_counts[rec['ext']]+=1; class_counts[rec['class']]+=1
|
||||
rec['top_domains']=[d for d,_ in rec['domains'].most_common(8)]
|
||||
rec.pop('domains', None)
|
||||
inv.write(json.dumps(rec, ensure_ascii=False)+'\n')
|
||||
if should_extract(rec['ext'], rec['class'], int(size or 0), rec['filename'], ' '.join(rec['subjects'])):
|
||||
text, parser=extract(p, rec['ext']) if rec['exists'] else ('','missing')
|
||||
parser_counts[parser]+=1
|
||||
if text and len(text)>=20:
|
||||
key_terms=[term for term in BUSINESS_TERMS if term.lower() in text.lower()]
|
||||
srec={k:rec.get(k) for k in ['content_sha1','filename','ext','size_bytes','absolute_path','class','threads','subjects','top_domains','date_end']}
|
||||
srec.update({'parser':parser,'text_excerpt':text[:2500],'key_terms':key_terms[:20]})
|
||||
snip.write(json.dumps(srec, ensure_ascii=False)+'\n')
|
||||
snippets.append(srec); extracted_counts[rec['class']]+=1
|
||||
rows.append(rec)
|
||||
def score(r):
|
||||
return {'order':50,'quote':48,'forecast':44,'quality':45,'compliance':40,'drawing':38,'spec':38,'other':0}.get(r['class'],0)+{'.xlsx':12,'.xlsm':12,'.xls':9,'.pdf':8,'.docx':7,'.doc':6,'.csv':6,'.htm':5,'.html':5,'.txt':4,'.stp':3,'.step':3,'.dwg':3}.get(r['ext'],0)+min(r.get('thread_count',0),20)+min(r.get('ref_count',0)//2,20)
|
||||
rows.sort(key=score, reverse=True)
|
||||
snippets.sort(key=lambda r: ({'order':50,'quote':48,'forecast':44,'quality':45,'compliance':40,'drawing':38,'spec':38,'other':0}.get(r['class'],0), r.get('size_bytes') or 0), reverse=True)
|
||||
summary={'generated_at':NOW,'source':str(SOURCE),'attachment_base':str(ATTACH_BASE/'attachments'),'stats':stats,'ext_counts':ext_counts.most_common(50),'class_counts':class_counts.most_common(),'parser_counts':parser_counts.most_common(),'extracted_counts':extracted_counts.most_common(),'inventory':str(INV.relative_to(VAULT)),'snippets':str(SNIP.relative_to(VAULT))}
|
||||
SUMMARY.write_text(json.dumps(summary, ensure_ascii=False, indent=2, default=list)+'\n', encoding='utf-8')
|
||||
pages={}
|
||||
overview=frontmatter(f'BJD 邮件附件 ingest {RUN_DATE}','[bjd, emails, attachments, internal, evidence]','邮件附件全量 inventory 与高价值附件内容摘录索引，覆盖图纸、规格书、订单、报价、预测、质量和合规资料。')
|
||||
overview += f"""# BJD 邮件附件 ingest {RUN_DATE}
|
||||
|
||||
本次针对 `/home/nac/repo/emails_export/parsed/llm_input.jsonl` 中引用的附件做单独 ingest。附件真实根目录为 `/home/nac/repo/emails_export/attachments`。^[extracted]
|
||||
|
||||
## 规模
|
||||
|
||||
- 附件引用数：{stats['refs']:,}
|
||||
- 去重后附件数：{stats['unique']:,}
|
||||
- 本地存在附件：{stats['exists']:,}
|
||||
- 本地附件总大小：{stats['bytes']/1024**3:.2f} GB
|
||||
- 全量 inventory：`{summary['inventory']}`
|
||||
- 内容摘录索引：`{summary['snippets']}`
|
||||
|
||||
## 内容类型
|
||||
|
||||
| 类别 | 去重附件数 |
|
||||
|---|---:|
|
||||
"""
|
||||
for k,v in class_counts.most_common(): overview+=f'| {esc(k)} | {v:,} |\n'
|
||||
overview+='\n## 文件类型 Top 25\n\n| 扩展名 | 去重附件数 |\n|---|---:|\n'
|
||||
for k,v in ext_counts.most_common(25): overview+=f'| `{esc(k)}` | {v:,} |\n'
|
||||
overview+='\n## 解析覆盖\n\n| 解析器 | 附件数 |\n|---|---:|\n'
|
||||
for k,v in parser_counts.most_common(): overview+=f'| {esc(k)} | {v:,} |\n'
|
||||
overview+='\n## 内部页面\n\n- [[projects/bjd/references/bjd-email-attachment-high-value-index-20260611]]\n- [[projects/bjd/references/bjd-email-attachment-extracted-content-index-20260611]]\n- [[projects/bjd/references/bjd-email-attachment-parser-coverage-20260611]]\n\n这些页面均为母库 internal-only，不进入 `bjd-general-bot`。^[extracted]\n'
|
||||
pages[f'projects/bjd/references/bjd-email-attachment-ingest-{RUN_DATE}.md']=overview
|
||||
high=frontmatter(f'BJD 邮件高价值附件索引 {RUN_DATE}','[bjd, emails, attachments, orders, rfq, quality, internal]','按业务类别、文件名、客户域名、thread 和本地路径索引高价值邮件附件。')
|
||||
high += f'# BJD 邮件高价值附件索引 {RUN_DATE}\n\n| 类别 | 扩展名 | 文件名 | 大小 | 引用/线程 | 客户域名 | 代表主题 | 本地路径 |\n|---|---|---|---:|---:|---|---|---|\n'
|
||||
for r in rows[:350]: high += f"| {esc(r['class'])} | `{esc(r['ext'])}` | {esc(r['filename'])} | {int(r.get('size_bytes') or 0):,} | {r.get('ref_count',0)}/{r.get('thread_count',0)} | {esc(md(r.get('top_domains',[])[:5]))} | {esc('；'.join(r.get('subjects',[])[:2]))} | `{esc(r.get('absolute_path'))}` |\n"
|
||||
pages[f'projects/bjd/references/bjd-email-attachment-high-value-index-{RUN_DATE}.md']=high
|
||||
cont=frontmatter(f'BJD 邮件附件内容摘录索引 {RUN_DATE}','[bjd, emails, attachments, extracted-content, internal]','可解析附件的文本摘录索引，用于检索报价、订单、预测、质量和合规附件内容。')
|
||||
cont += f'# BJD 邮件附件内容摘录索引 {RUN_DATE}\n\n| 类别 | 文件名 | 解析器 | 客户域名 | 代表主题 | 关键词 | 内容摘录 |\n|---|---|---|---|---|---|---|\n'
|
||||
for r in snippets[:260]: cont += f"| {esc(r['class'])} | {esc(r['filename'])} | {esc(r['parser'])} | {esc(md(r.get('top_domains',[])[:5]))} | {esc('；'.join(r.get('subjects',[])[:2]))} | {esc(md(r.get('key_terms',[])[:8]))} | {esc(r.get('text_excerpt','')[:550])} |\n"
|
||||
pages[f'projects/bjd/references/bjd-email-attachment-extracted-content-index-{RUN_DATE}.md']=cont
|
||||
cov=frontmatter(f'BJD 邮件附件解析覆盖 {RUN_DATE}','[bjd, emails, attachments, parser-coverage, internal]','说明附件 ingest 的覆盖范围、未解析类型和后续 OCR/PDF/CAD 处理建议。')
|
||||
cov += f"""# BJD 邮件附件解析覆盖 {RUN_DATE}
|
||||
|
||||
## 已完成
|
||||
|
||||
- 全量附件 inventory：所有 {stats['unique']:,} 个去重附件均已登记路径、sha1、文件名、大小、类型、关联 thread、主题和客户域名。^[extracted]
|
||||
- 可用标准库解析的高价值附件已写入内容摘录索引，包括 `.xlsx/.xlsm/.csv/.txt/.html/.docx/.pptx/.eml`，以及部分 `.pdf/.xls/.doc` 的二进制字符串摘录。^[extracted]
|
||||
|
||||
## 需要后续增强
|
||||
|
||||
- PDF：当前没有 `pdftotext` / PyPDF / OCR，部分 PDF 只能建索引或抽少量未压缩字符串；规格书、图纸 PDF 要高质量 ingest 需要安装 PDF 解析/OCR。^[inferred]
|
||||
- 图片：`.png/.jpg/.tif` 当前只建索引，没有 OCR；质量照片、截图、标签图片要另跑 OCR/视觉模型。^[inferred]
|
||||
- CAD/3D：`.stp/.step/.dwg` 当前只建索引；几何内容和图纸参数需要 CAD 专用解析或人工/视觉流程。^[inferred]
|
||||
- 压缩包：`.zip/.rar/.7z` 当前只建外层索引；需要后续安全解包再逐项 ingest。^[inferred]
|
||||
|
||||
## 解析器统计
|
||||
|
||||
| 解析器 | 附件数 |
|
||||
|---|---:|
|
||||
"""
|
||||
for k,v in parser_counts.most_common(): cov+=f'| {esc(k)} | {v:,} |\n'
|
||||
cov+='\n## 按类别的内容摘录数\n\n| 类别 | 已摘录附件数 |\n|---|---:|\n'
|
||||
for k,v in extracted_counts.most_common(): cov+=f'| {esc(k)} | {v:,} |\n'
|
||||
pages[f'projects/bjd/references/bjd-email-attachment-parser-coverage-{RUN_DATE}.md']=cov
|
||||
for rel,text in pages.items():
|
||||
path=VAULT/rel; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(text, encoding='utf-8')
|
||||
idx=VAULT/'index.md'; text=idx.read_text(encoding='utf-8'); text=re.sub(r'最后更新：[^*]+', f'最后更新：{NOW}', text)
|
||||
marker='## 综合分析\n'
|
||||
if marker not in text: text+='\n## 综合分析\n'
|
||||
ins=text.index(marker)+len(marker)
|
||||
entries=[f'- [[projects/bjd/references/bjd-email-attachment-ingest-{RUN_DATE}]] - 邮件附件全量 inventory 和高价值附件内容摘录总览。 ( #bjd #emails #attachments )',f'- [[projects/bjd/references/bjd-email-attachment-high-value-index-{RUN_DATE}]] - 高价值附件索引，关联客户、主题、thread 和本地路径。 ( #bjd #attachments #internal )',f'- [[projects/bjd/references/bjd-email-attachment-extracted-content-index-{RUN_DATE}]] - 可解析附件内容摘录索引。 ( #bjd #attachments #extracted-content )',f'- [[projects/bjd/references/bjd-email-attachment-parser-coverage-{RUN_DATE}]] - 附件解析覆盖和后续 OCR/PDF/CAD 处理建议。 ( #bjd #attachments #parser )']
|
||||
block=''.join(e+'\n' for e in entries if e not in text); idx.write_text(text[:ins]+block+text[ins:], encoding='utf-8')
|
||||
hot=VAULT/'hot.md'; h=hot.read_text(encoding='utf-8'); h=re.sub(r'updated: [^\n]+', f'updated: {NOW}', h, count=1)
|
||||
item=f"- [{NOW}] DATA_INGEST - 新增邮件附件 ingest：{stats['refs']:,} 个引用、{stats['unique']:,} 个去重附件、{stats['bytes']/1024**3:.2f}GB；已生成全量 inventory 和高价值附件内容摘录，internal-only。\n"
|
||||
if item not in h: h=h.replace('## 近期活动\n\n','## 近期活动\n\n'+item,1)
|
||||
hot.write_text(h, encoding='utf-8')
|
||||
with (VAULT/'log.md').open('a', encoding='utf-8') as f: f.write(f'\n- [{NOW}] DATA_INGEST source="{SOURCE}" format=email_attachments refs={stats["refs"]} unique={stats["unique"]} pages_created={len(pages)} inventory="_meta/{INV.name}" snippets="_meta/{SNIP.name}" bot_visible=false\n')
|
||||
man=VAULT/'.manifest.json'; data=json.loads(man.read_text(encoding='utf-8'))
|
||||
key=f'{SOURCE}::email-attachments::{RUN_DATE}'
|
||||
data.setdefault('sources',{})[key]={'ingested_at':NOW,'original_path':str(SOURCE),'source_type':'email_attachment_corpus','doc_type':'followup_email_attachments','project':'bjd','content_hash':'sha256:'+hashlib.sha256(SOURCE.read_bytes()).hexdigest(),'attachment_refs':stats['refs'],'unique_attachments':stats['unique'],'existing_attachments':stats['exists'],'attachment_total_bytes':stats['bytes'],'attachment_base':str(ATTACH_BASE/'attachments'),'inventory_path':str(INV.relative_to(VAULT)),'snippet_path':str(SNIP.relative_to(VAULT)),'pages_created':sorted(pages),'pages_updated':['index.md','hot.md','log.md','.manifest.json'],'bot_visible':False}
|
||||
data['updated']=NOW; data.setdefault('stats',{})['total_sources_ingested']=len(data.get('sources',{})); man.write_text(json.dumps(data, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
|
||||
print(json.dumps({'stats':stats,'pages':sorted(pages),'inventory':str(INV),'snippets':str(SNIP),'parser_counts':parser_counts.most_common(10),'generated_at':NOW}, ensure_ascii=False, indent=2))
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,489 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create internal-only enhanced BJD email knowledge pages.
|
||||
|
||||
Unlike the bot-safe derived ingest, this keeps internal commercial evidence
|
||||
such as order refs, quantity refs and price refs in mother vault pages. It still
|
||||
summarizes/extracts; it does not copy full raw email bodies or attachments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from email.header import decode_header
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
VAULT = Path("/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault")
|
||||
SOURCE = Path("/home/nac/repo/emails_export/parsed/llm_input.jsonl")
|
||||
RUN_DATE = "20260611"
|
||||
NOW = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
TODAY = NOW[:10]
|
||||
META_DIR = VAULT / "_meta"
|
||||
INTERNAL_META = META_DIR / f"bjd-email-internal-enhanced-knowledge-{RUN_DATE}.json"
|
||||
INTERNAL_REGISTRY = META_DIR / f"bjd-email-internal-thread-evidence-registry-{RUN_DATE}.jsonl"
|
||||
|
||||
EMAIL_RE = re.compile(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}")
|
||||
PHONE_RE = re.compile(r"(?:\+?\d[\d\s\-()]{6,}\d)|(?:0\d{2,4}-?\d{6,8})")
|
||||
ORDER_RE = re.compile(r"\b(?:XSDD|PO|PRD|SO|CGDD|SMCG|AECJ|RFQ|AL|MAR|BJD|LH|PH|LP|N|P|EP|DG|J|T)\d[-_A-Z0-9]*\b", re.I)
|
||||
QTY_RE = re.compile(r"\b\d+(?:\.\d+)?\s*(?:pcs|parts|nos|ea|套|件|只|个|kg|kgs|cbm|cartons?|箱|kpcs|千只|万只)\b", re.I)
|
||||
PRICE_RE = re.compile(r"(?:[$￥¥]\s*\d+(?:\.\d+)?|\b\d+(?:\.\d+)?\s*(?:RMB|CNY|USD|EUR|元/pcs|元/pc|元/只|美金|美元|人民币)\b|(?:价格|报价|单价|price|quote)[:：]?\s*\d+(?:\.\d+)?)", re.I)
|
||||
DATE_RE = re.compile(r"\b(?:20\d{2}[-/.]\d{1,2}[-/.]\d{1,2}|\d{1,2}/\d{1,2}/20\d{2}|Wk\s*\d{1,2}|WK\s*\d{1,2})\b")
|
||||
PART_RE = re.compile(r"\b(?:\d{6}-\d{4,6}(?:-HF)?|\d{3,}-\d{3,}(?:-HF)?|FH\d+[A-Z0-9-]*|C-\d{7}|\d{3,}-\d{3,}-\d{2,}(?:-HF)?|[A-Z]{1,6}\d{2,}[-A-Z0-9]*)\b", re.I)
|
||||
|
||||
KEYWORDS = [
|
||||
"报价", "询价", "RFQ", "Quote", "样品", "打样", "送样", "图纸", "承认书", "规格书",
|
||||
"替代", "替换", "竞品", "VAVE", "Hirose", "Molex", "TE", "泰科", "JAE", "Kyocera",
|
||||
"IRISO", "JST", "KEL", "Omron", "订单", "交期", "交货", "发货", "备料", "forecast",
|
||||
"SOP", "PPAP", "QMS", "CAMDS", "RoHS", "REACH", "无卤", "IATF", "VDA", "审核",
|
||||
"品质", "客诉", "投诉", "不良", "8D", "退货", "索赔", "对账", "付款", "发票", "合同",
|
||||
"开模", "开发", "封样", "零跑", "领跑", "维胜", "恒润", "安悦", "先锋",
|
||||
]
|
||||
OEM_TERMS = ["零跑", "Leapmotor", "BYD", "比亚迪", "Toyota", "上汽大众", "SVW", "Mazda", "VW", "Skoda", "Stellantis", "Ford", "Honda", "Hyundai", "Kia", "Marelli", "Valeo", "Visteon", "Mercedes-Benz", "BMW", "Volvo", "Porsche", "GM", "HARMAN", "AUMOVIO"]
|
||||
APP_TERMS = ["吸顶屏", "显示", "显示屏", "CID", "FID", "Cluster", "仪表", "氛围灯", "Ambient Lighting", "储能", "BMS", "车灯", "lighting", "摄像头", "Camera", "排线", "FPC", "FFC", "Fakra", "FAKRA", "Mini-Fakra", "Socket Header", "Pin Header", "SMB", "BTB", "WTB", "Wafer", "ZIF", "HSAL"]
|
||||
INTERNAL_DOMAINS = {"beijida.com", "bjdffc.com"}
|
||||
|
||||
DOMAIN_LABELS = {
|
||||
"visteon.com": "Visteon", "byd.com": "BYD/比亚迪", "leapmotor.com": "零跑/Leapmotor",
|
||||
"bicv.com.cn": "BICV/北斗智联", "marelli.com": "Marelli", "valeo.com": "Valeo",
|
||||
"tpk-auto.com": "TPK Auto", "dsbj.com": "深南电路/DSBJ", "luxshare-ict.com": "立讯/Luxshare",
|
||||
"wintec.com": "Wintec", "tospo.com": "得邦照明/Tospo", "skyworth.com": "创维/Skyworth",
|
||||
"harman.com": "HARMAN", "aumovio.com": "AUMOVIO", "continental-corporation.com": "Continental/AUMOVIO",
|
||||
}
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return "sha256:" + h.hexdigest()
|
||||
|
||||
|
||||
def decode_text(value: Any) -> str:
|
||||
s = str(value or "")
|
||||
if "=?" in s:
|
||||
try:
|
||||
parts = []
|
||||
for part, enc in decode_header(s):
|
||||
if isinstance(part, bytes):
|
||||
parts.append(part.decode(enc or "utf-8", errors="replace"))
|
||||
else:
|
||||
parts.append(part)
|
||||
s = "".join(parts)
|
||||
except Exception:
|
||||
pass
|
||||
return re.sub(r"\s+", " ", s.replace("\r", " ").replace("\n", " ")).strip()
|
||||
|
||||
|
||||
def clean_light(value: Any, limit: int = 220) -> str:
|
||||
# Mother-vault internal extraction keeps numeric evidence intact.
|
||||
# Only collapse whitespace; do not phone/order/quantity-redact.
|
||||
s = decode_text(value)
|
||||
return s[:limit]
|
||||
|
||||
|
||||
def table_escape(value: Any) -> str:
|
||||
return str(value or "").replace("|", " / ").replace("\n", " ")
|
||||
|
||||
|
||||
def contains_term(text: str, term: str) -> bool:
|
||||
if any("\u4e00" <= ch <= "\u9fff" for ch in term):
|
||||
return term.lower() in text.lower()
|
||||
return re.search(rf"(?<![A-Za-z0-9]){re.escape(term)}(?![A-Za-z0-9])", text, re.I) is not None
|
||||
|
||||
|
||||
def domain_of(person: Any) -> str | None:
|
||||
if not isinstance(person, dict):
|
||||
return None
|
||||
email = (person.get("email") or "").lower()
|
||||
return email.split("@")[-1] if "@" in email else None
|
||||
|
||||
|
||||
def domain_label(domain: str) -> str:
|
||||
d = (domain or "").lower()
|
||||
for key, label in DOMAIN_LABELS.items():
|
||||
if key in d:
|
||||
return label
|
||||
return d
|
||||
|
||||
|
||||
def valid_part(part: str) -> bool:
|
||||
p = str(part or "").upper().strip()
|
||||
if len(p) < 5 or "[" in p or "]" in p:
|
||||
return False
|
||||
if PHONE_RE.fullmatch(p):
|
||||
return False
|
||||
if re.fullmatch(r"[A-F0-9]{24,}", p):
|
||||
return False
|
||||
if p.startswith(("IMAGE", "SHA", "AES", "DATE", "GB2312", "FEDEX", "DHL", "UPS")):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def uniq(items: list[str], limit: int = 12) -> list[str]:
|
||||
out, seen = [], set()
|
||||
for item in items:
|
||||
s = decode_text(item).strip()
|
||||
if not s or s in seen:
|
||||
continue
|
||||
out.append(s)
|
||||
seen.add(s)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def snippets(text: str, patterns: list[str], limit: int = 3) -> list[str]:
|
||||
hits = []
|
||||
lowered = text.lower()
|
||||
for pat in patterns:
|
||||
idx = lowered.find(pat.lower())
|
||||
if idx >= 0:
|
||||
start = max(0, idx - 70)
|
||||
end = min(len(text), idx + len(pat) + 110)
|
||||
hits.append(clean_light(text[start:end], 220))
|
||||
if len(hits) >= limit:
|
||||
break
|
||||
return uniq(hits, limit)
|
||||
|
||||
|
||||
def stage_for(subject: str, text: str) -> str:
|
||||
s = f"{subject} {text[:3000]}".lower()
|
||||
if re.search(r"8d|quality|cleanliness|不良|客诉|索赔|退货|改善|particle|异常|风险批次", s):
|
||||
return "质量/风险闭环"
|
||||
if re.search(r"ppap|audit|qms|camds|调查表|体系|审核|vda|iatf|rohs|reach|无卤|合规", s):
|
||||
return "审核/PPAP/合规资料"
|
||||
if re.search(r"rfq|quote|报价|询价|寻源|sourcing|new development|替代|vave|zif|inquiry", s):
|
||||
return "RFQ/报价/技术评估"
|
||||
if re.search(r"样品|sample|打样|图纸|drawing|spec|承认书|封样", s):
|
||||
return "样品/图纸/开发"
|
||||
if re.search(r"订单|order|交期|delivery|发货|备料|forecast|sop|shipment|交货|mrp|fcst|提货", s):
|
||||
return "订单/履约/备料"
|
||||
return "项目/应用机会"
|
||||
|
||||
|
||||
def family_from(products: list[str], apps: list[str], text: str) -> list[str]:
|
||||
low = text.lower()
|
||||
fams = []
|
||||
def add(x: str):
|
||||
if x not in fams:
|
||||
fams.append(x)
|
||||
if any("fakra" in x.lower() for x in apps) or "fakra" in low or "smb" in low:
|
||||
add("高频/FAKRA/SMB")
|
||||
if "pin header" in low:
|
||||
add("Pin Header")
|
||||
if "btb" in low:
|
||||
add("BTB")
|
||||
if "wtb" in low or "wafer" in low:
|
||||
add("WTB/Wafer/线对板")
|
||||
if "ffc" in low or "排线" in low:
|
||||
add("FFC排线/线束")
|
||||
if "fpc" in low or any(p.startswith("101") for p in products):
|
||||
add("FPC连接器")
|
||||
if not fams:
|
||||
add("待分类连接器机会")
|
||||
return fams[:5]
|
||||
|
||||
|
||||
def md_list(items: list[str], empty: str = "未提取") -> str:
|
||||
return "、".join(items) if items else empty
|
||||
|
||||
|
||||
def build() -> dict[str, Any]:
|
||||
stats = {"threads": 0, "messages": 0, "date_start": None, "date_end": None}
|
||||
customer_agg: dict[str, dict[str, Any]] = {}
|
||||
commercial_rows: list[dict[str, Any]] = []
|
||||
project_rows: list[dict[str, Any]] = []
|
||||
quality_rows: list[dict[str, Any]] = []
|
||||
high_rows: list[dict[str, Any]] = []
|
||||
signal_counts = collections.Counter()
|
||||
price_count = qty_count = order_count = 0
|
||||
|
||||
META_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with SOURCE.open(encoding="utf-8", errors="replace") as fh, INTERNAL_REGISTRY.open("w", encoding="utf-8") as reg:
|
||||
for line in fh:
|
||||
obj = json.loads(line)
|
||||
stats["threads"] += 1
|
||||
start, end = obj.get("date_start"), obj.get("date_end")
|
||||
if start and (stats["date_start"] is None or start < stats["date_start"]):
|
||||
stats["date_start"] = start
|
||||
if end and (stats["date_end"] is None or end > stats["date_end"]):
|
||||
stats["date_end"] = end
|
||||
messages = obj.get("messages", []) or []
|
||||
stats["messages"] += len(messages)
|
||||
subject = decode_text(obj.get("suggested_title_from_subject", ""))
|
||||
text = " ".join([subject] + [decode_text((m.get("subject") or "") + " " + (m.get("body_text") or "")) for m in messages if isinstance(m, dict)])
|
||||
thread_domains = collections.Counter()
|
||||
for msg in messages:
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
for person in [msg.get("from", {})] + (msg.get("to", []) or []) + (msg.get("cc", []) or []):
|
||||
d = domain_of(person)
|
||||
if d:
|
||||
thread_domains[d] += 1
|
||||
external_domains = [d for d, _ in thread_domains.most_common(8) if d not in INTERNAL_DOMAINS]
|
||||
customers = uniq([domain_label(d) for d in external_domains], 8)
|
||||
keywords = [k for k in KEYWORDS if contains_term(text, k)]
|
||||
oems = [x for x in OEM_TERMS if contains_term(text, x)]
|
||||
apps = [x for x in APP_TERMS if contains_term(text, x)]
|
||||
products = uniq([p.upper() for p in PART_RE.findall(text) if valid_part(p)], 20)
|
||||
order_refs = uniq(ORDER_RE.findall(text), 18)
|
||||
qty_refs = uniq(QTY_RE.findall(text), 12)
|
||||
# QTY_RE with groups returns tuples if groups exist; normalize defensively.
|
||||
qty_refs = uniq([q[0] if isinstance(q, tuple) else q for q in qty_refs], 12)
|
||||
price_refs = uniq(PRICE_RE.findall(text), 12)
|
||||
price_refs = uniq([p[0] if isinstance(p, tuple) else p for p in price_refs], 12)
|
||||
date_refs = uniq(DATE_RE.findall(text), 10)
|
||||
stage = stage_for(subject, text)
|
||||
families = family_from(products, apps, text)
|
||||
evidence = snippets(text, keywords + oems + apps + products[:5] + order_refs[:5], 3)
|
||||
score = len(keywords) * 3 + len(oems) * 6 + len(apps) * 4 + len(products) * 2 + len(order_refs) * 2 + len(qty_refs) * 2 + len(price_refs) * 4 + len(messages) // 5
|
||||
if price_refs:
|
||||
price_count += 1
|
||||
signal_counts["price"] += 1
|
||||
if qty_refs:
|
||||
qty_count += 1
|
||||
signal_counts["quantity"] += 1
|
||||
if order_refs:
|
||||
order_count += 1
|
||||
signal_counts["order_ref"] += 1
|
||||
signal_counts[stage] += 1
|
||||
row = {
|
||||
"thread_id": obj.get("thread_id"),
|
||||
"thread_content_hash": obj.get("thread_content_hash"),
|
||||
"date_end": end,
|
||||
"subject": clean_light(subject, 260),
|
||||
"customers": customers,
|
||||
"domains": external_domains[:8],
|
||||
"oems": uniq(oems, 8),
|
||||
"applications": uniq(apps, 10),
|
||||
"products": products[:15],
|
||||
"families": families,
|
||||
"stage": stage,
|
||||
"order_refs": order_refs[:12],
|
||||
"qty_refs": qty_refs[:10],
|
||||
"price_refs": price_refs[:10],
|
||||
"date_refs": date_refs[:8],
|
||||
"evidence": evidence,
|
||||
"message_count": len(messages),
|
||||
"attachment_count": len(obj.get("attachments", []) or []),
|
||||
"score": score,
|
||||
}
|
||||
reg.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
if score >= 20 or price_refs or qty_refs or order_refs or oems or apps:
|
||||
high_rows.append(row)
|
||||
if stage in {"RFQ/报价/技术评估", "样品/图纸/开发", "项目/应用机会"} and (oems or apps or products):
|
||||
project_rows.append(row)
|
||||
if stage in {"订单/履约/备料", "RFQ/报价/技术评估"} and (order_refs or qty_refs or price_refs):
|
||||
commercial_rows.append(row)
|
||||
if stage == "质量/风险闭环":
|
||||
quality_rows.append(row)
|
||||
for d in external_domains[:5]:
|
||||
label = domain_label(d)
|
||||
rec = customer_agg.setdefault(label, {"customer": label, "domain": d, "threads": 0, "stages": collections.Counter(), "oems": collections.Counter(), "apps": collections.Counter(), "families": collections.Counter(), "products": collections.Counter(), "order_threads": 0, "qty_threads": 0, "price_threads": 0, "subjects": []})
|
||||
rec["threads"] += 1
|
||||
rec["stages"].update([stage])
|
||||
rec["oems"].update(oems)
|
||||
rec["apps"].update(apps)
|
||||
rec["families"].update(families)
|
||||
rec["products"].update(products[:10])
|
||||
rec["order_threads"] += bool(order_refs)
|
||||
rec["qty_threads"] += bool(qty_refs)
|
||||
rec["price_threads"] += bool(price_refs)
|
||||
if len(rec["subjects"]) < 4:
|
||||
rec["subjects"].append(clean_light(subject, 160))
|
||||
|
||||
def pack_counter(c: collections.Counter, n=8):
|
||||
return [k for k, _ in c.most_common(n)]
|
||||
|
||||
customer_rows = []
|
||||
for rec in sorted(customer_agg.values(), key=lambda x: x["threads"], reverse=True)[:120]:
|
||||
customer_rows.append({
|
||||
"customer": rec["customer"], "domain": rec["domain"], "threads": rec["threads"],
|
||||
"stages": pack_counter(rec["stages"]), "oems": pack_counter(rec["oems"]), "apps": pack_counter(rec["apps"]),
|
||||
"families": pack_counter(rec["families"]), "products": pack_counter(rec["products"], 12),
|
||||
"order_threads": rec["order_threads"], "qty_threads": rec["qty_threads"], "price_threads": rec["price_threads"],
|
||||
"subjects": rec["subjects"],
|
||||
})
|
||||
|
||||
key = lambda r: (r["score"], r.get("date_end") or "")
|
||||
return {
|
||||
"source": str(SOURCE), "generated_at": NOW, "content_hash": sha256(SOURCE), "stats": stats,
|
||||
"thread_registry": str(INTERNAL_REGISTRY.relative_to(VAULT)),
|
||||
"signal_counts": dict(signal_counts), "price_thread_count": price_count, "quantity_thread_count": qty_count, "order_ref_thread_count": order_count,
|
||||
"customer_rows": customer_rows,
|
||||
"commercial_rows": sorted(commercial_rows, key=key, reverse=True)[:500],
|
||||
"project_rows": sorted(project_rows, key=key, reverse=True)[:500],
|
||||
"quality_rows": sorted(quality_rows, key=key, reverse=True)[:250],
|
||||
"high_rows": sorted(high_rows, key=key, reverse=True)[:700],
|
||||
}
|
||||
|
||||
|
||||
def frontmatter(title: str, tags: str, summary: str) -> str:
|
||||
return f"""---
|
||||
title: {title}
|
||||
category: references
|
||||
project: bjd
|
||||
tags: {tags}
|
||||
visibility: internal
|
||||
data_classification: commercial-confidential
|
||||
allowed_bots: []
|
||||
excluded_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/bjd-email-internal-enhanced-knowledge-{RUN_DATE}.json
|
||||
- _meta/bjd-email-internal-thread-evidence-registry-{RUN_DATE}.jsonl
|
||||
summary: >-
|
||||
{summary}
|
||||
provenance:
|
||||
extracted: 0.72
|
||||
inferred: 0.20
|
||||
ambiguous: 0.08
|
||||
base_confidence: 0.70
|
||||
lifecycle: draft
|
||||
lifecycle_changed: {TODAY}
|
||||
created: {NOW}
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def write(rel: str, text: str):
|
||||
path = VAULT / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def build_pages(meta: dict[str, Any]) -> dict[str, str]:
|
||||
pages = {}
|
||||
overview = frontmatter(f"BJD 邮件母库内部增强 ingest {RUN_DATE}", "[bjd, emails, internal, commercial-evidence, visibility/internal]", "母库内部增强版邮件 ingest，保留订单、数量、价格、交付和项目证据字段；不进入通用机器人。")
|
||||
overview += f"""# BJD 邮件母库内部增强 ingest {RUN_DATE}
|
||||
|
||||
这是一版母库内部增强索引，区别于安全视图：保留邮件中可用于内部分析的精确商业证据字段，例如订单/PO/RFQ 引用、数量/重量/体积表达、价格/报价表达、交付日期表达、产品料号和项目线索。^[extracted]
|
||||
|
||||
该版本仍然不是原始邮件归档：没有复制完整邮件正文，也没有复制附件原文；但证据字段比 bot-safe 页更完整。所有页面 `excluded_bots: [bjd-general-bot]`。^[extracted]
|
||||
|
||||
## 规模
|
||||
|
||||
- 线程数：{meta['stats']['threads']:,}
|
||||
- 邮件消息数：{meta['stats']['messages']:,}
|
||||
- 时间范围：{meta['stats']['date_start'][:10]} 至 {meta['stats']['date_end'][:10]}
|
||||
- 内部 evidence registry：`{meta['thread_registry']}`
|
||||
- 含订单/RFQ/PO 等引用的线程：{meta['order_ref_thread_count']:,}
|
||||
- 含数量/重量/体积表达的线程：{meta['quantity_thread_count']:,}
|
||||
- 含价格/报价表达的线程：{meta['price_thread_count']:,}
|
||||
|
||||
## 内部增强页
|
||||
|
||||
- [[projects/bjd/references/bjd-email-internal-commercial-evidence-{RUN_DATE}]]
|
||||
- [[projects/bjd/references/bjd-email-internal-customer-ledger-{RUN_DATE}]]
|
||||
- [[projects/bjd/references/bjd-email-internal-project-evidence-index-{RUN_DATE}]]
|
||||
- [[projects/bjd/references/bjd-email-internal-quality-evidence-index-{RUN_DATE}]]
|
||||
"""
|
||||
pages[f"projects/bjd/references/bjd-email-internal-enhanced-ingest-{RUN_DATE}.md"] = overview
|
||||
|
||||
commercial = frontmatter(f"BJD 邮件内部商业证据索引 {RUN_DATE}", "[bjd, emails, orders, pricing, fulfillment, internal]", "从邮件中提取订单、数量、价格、交付和产品证据字段，用于母库内部商业分析。")
|
||||
commercial += f"# BJD 邮件内部商业证据索引 {RUN_DATE}\n\n| 日期 | 客户/渠道 | 阶段 | 产品/料号 | 订单/RFQ/PO引用 | 数量/重量/体积 | 价格/报价表达 | 日期/交付引用 | 主题 | 证据片段 |\n|---|---|---|---|---|---|---|---|---|---|\n"
|
||||
for r in meta["commercial_rows"][:220]:
|
||||
commercial += f"| {(r.get('date_end') or '')[:10]} | {table_escape(md_list(r['customers']))} | {table_escape(r['stage'])} | {table_escape(md_list(r['products'][:8]))} | {table_escape(md_list(r['order_refs'][:8]))} | {table_escape(md_list(r['qty_refs'][:6]))} | {table_escape(md_list(r['price_refs'][:5]))} | {table_escape(md_list(r['date_refs'][:5]))} | {table_escape(r['subject'])} | {table_escape('；'.join(r['evidence'][:2]))} |\n"
|
||||
pages[f"projects/bjd/references/bjd-email-internal-commercial-evidence-{RUN_DATE}.md"] = commercial
|
||||
|
||||
ledger = frontmatter(f"BJD 邮件内部客户线索台账 {RUN_DATE}", "[bjd, emails, customers, internal, ledger]", "按客户/域名聚合内部邮件线索，保留订单、数量、价格线程计数和代表性产品/应用。")
|
||||
ledger += f"# BJD 邮件内部客户线索台账 {RUN_DATE}\n\n| 客户/域名 | 线程数 | 订单线索 | 数量线索 | 价格线索 | 阶段 | OEM | 应用 | 产品族 | 代表产品 | 样例主题 |\n|---|---:|---:|---:|---:|---|---|---|---|---|---|\n"
|
||||
for r in meta["customer_rows"][:100]:
|
||||
ledger += f"| {table_escape(r['customer'])} / `{table_escape(r['domain'])}` | {r['threads']} | {r['order_threads']} | {r['qty_threads']} | {r['price_threads']} | {table_escape(md_list(r['stages']))} | {table_escape(md_list(r['oems']))} | {table_escape(md_list(r['apps']))} | {table_escape(md_list(r['families']))} | {table_escape(md_list(r['products'][:8]))} | {table_escape('；'.join(r['subjects']))} |\n"
|
||||
pages[f"projects/bjd/references/bjd-email-internal-customer-ledger-{RUN_DATE}.md"] = ledger
|
||||
|
||||
project = frontmatter(f"BJD 邮件内部项目证据索引 {RUN_DATE}", "[bjd, emails, projects, applications, internal]", "母库内部项目/OEM/应用证据表，保留更完整的料号、订单、数量、价格和证据片段。")
|
||||
project += f"# BJD 邮件内部项目证据索引 {RUN_DATE}\n\n邮件共现不等于已中标/已量产；需要订单台账、项目台账、图纸承认或销售确认升级判断。^[inferred]\n\n| 日期 | 客户/渠道 | OEM | 应用 | 产品方向 | 阶段 | 产品/料号 | 订单/数量/价格证据 | 主题 | 证据片段 |\n|---|---|---|---|---|---|---|---|---|---|\n"
|
||||
for r in meta["project_rows"][:240]:
|
||||
evidence_bits = []
|
||||
if r['order_refs']:
|
||||
evidence_bits.append('订单/RFQ: ' + md_list(r['order_refs'][:5]))
|
||||
if r['qty_refs']:
|
||||
evidence_bits.append('数量: ' + md_list(r['qty_refs'][:4]))
|
||||
if r['price_refs']:
|
||||
evidence_bits.append('价格: ' + md_list(r['price_refs'][:3]))
|
||||
project += f"| {(r.get('date_end') or '')[:10]} | {table_escape(md_list(r['customers']))} | {table_escape(md_list(r['oems']))} | {table_escape(md_list(r['applications']))} | {table_escape(md_list(r['families']))} | {table_escape(r['stage'])} | {table_escape(md_list(r['products'][:10]))} | {table_escape('；'.join(evidence_bits) or '未提取')} | {table_escape(r['subject'])} | {table_escape('；'.join(r['evidence'][:2]))} |\n"
|
||||
pages[f"projects/bjd/references/bjd-email-internal-project-evidence-index-{RUN_DATE}.md"] = project
|
||||
|
||||
quality = frontmatter(f"BJD 邮件内部质量证据索引 {RUN_DATE}", "[bjd, emails, quality, complaints, internal]", "母库内部质量/客诉/退货/8D 证据索引，保留产品、客户、主题和证据片段。")
|
||||
quality += f"# BJD 邮件内部质量证据索引 {RUN_DATE}\n\n| 日期 | 客户/渠道 | OEM | 应用 | 产品/料号 | 订单/数量证据 | 主题 | 证据片段 |\n|---|---|---|---|---|---|---|---|\n"
|
||||
for r in meta["quality_rows"][:180]:
|
||||
quality += f"| {(r.get('date_end') or '')[:10]} | {table_escape(md_list(r['customers']))} | {table_escape(md_list(r['oems']))} | {table_escape(md_list(r['applications']))} | {table_escape(md_list(r['products'][:10]))} | {table_escape(md_list((r['order_refs'] + r['qty_refs'])[:8]))} | {table_escape(r['subject'])} | {table_escape('；'.join(r['evidence'][:2]))} |\n"
|
||||
pages[f"projects/bjd/references/bjd-email-internal-quality-evidence-index-{RUN_DATE}.md"] = quality
|
||||
return pages
|
||||
|
||||
|
||||
def update_special(meta: dict[str, Any], pages: dict[str, str]):
|
||||
idx = VAULT / "index.md"
|
||||
text = idx.read_text(encoding="utf-8")
|
||||
text = re.sub(r"最后更新：[^*]+", f"最后更新：{NOW}", text)
|
||||
marker = "## 综合分析\n"
|
||||
if marker not in text:
|
||||
text += "\n## 综合分析\n"
|
||||
entries = [
|
||||
f"- [[projects/bjd/references/bjd-email-internal-enhanced-ingest-{RUN_DATE}]] - 母库内部增强版邮件 ingest，总览订单/数量/价格/项目证据字段。 ( #bjd #emails #internal )",
|
||||
f"- [[projects/bjd/references/bjd-email-internal-commercial-evidence-{RUN_DATE}]] - 内部商业证据索引，保留订单/RFQ/数量/价格/交付表达。 ( #bjd #emails #orders )",
|
||||
f"- [[projects/bjd/references/bjd-email-internal-customer-ledger-{RUN_DATE}]] - 内部客户线索台账，按域名聚合订单/数量/价格线程和产品应用。 ( #bjd #emails #customers )",
|
||||
f"- [[projects/bjd/references/bjd-email-internal-project-evidence-index-{RUN_DATE}]] - 内部项目证据索引，保留更完整料号和证据片段。 ( #bjd #emails #projects )",
|
||||
f"- [[projects/bjd/references/bjd-email-internal-quality-evidence-index-{RUN_DATE}]] - 内部质量证据索引。 ( #bjd #emails #quality )",
|
||||
]
|
||||
ins = text.index(marker) + len(marker)
|
||||
block = "".join(e + "\n" for e in entries if e not in text)
|
||||
idx.write_text(text[:ins] + block + text[ins:], encoding="utf-8")
|
||||
|
||||
hot = VAULT / "hot.md"
|
||||
h = hot.read_text(encoding="utf-8")
|
||||
h = re.sub(r"updated: [^\n]+", f"updated: {NOW}", h, count=1)
|
||||
item = f"- [{NOW}] DATA_INGEST - 补充母库内部增强版邮件索引：保留订单/数量/价格/交付/项目证据字段；新增 5 个 internal-only 页面，仍排除 bjd-general-bot。\n"
|
||||
if item not in h:
|
||||
h = h.replace("## 近期活动\n\n", "## 近期活动\n\n" + item, 1)
|
||||
hot.write_text(h, encoding="utf-8")
|
||||
|
||||
with (VAULT / "log.md").open("a", encoding="utf-8") as fh:
|
||||
fh.write(f'\n- [{NOW}] DATA_INGEST source="{SOURCE}" format=jsonl mode="email-internal-enhanced" pages_created={len(pages)} registry="_meta/{INTERNAL_REGISTRY.name}" bot_visible=false\n')
|
||||
|
||||
man = VAULT / ".manifest.json"
|
||||
data = json.loads(man.read_text(encoding="utf-8"))
|
||||
st = SOURCE.stat()
|
||||
key = f"{SOURCE}::email-internal-enhanced::{RUN_DATE}"
|
||||
data.setdefault("sources", {})[key] = {
|
||||
"ingested_at": NOW,
|
||||
"size_bytes": st.st_size,
|
||||
"modified_at": datetime.fromtimestamp(st.st_mtime, timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
"content_hash": meta["content_hash"],
|
||||
"source_id": meta["content_hash"],
|
||||
"original_path": str(SOURCE),
|
||||
"source_type": "email_jsonl",
|
||||
"doc_type": "followup_email_archive_internal_enhanced",
|
||||
"project": "bjd",
|
||||
"business_categories": ["sales/follow-up", "orders/internal-evidence", "pricing/internal-evidence", "customers/communications", "quality/customer-feedback"],
|
||||
"email_thread_count": meta["stats"]["threads"],
|
||||
"email_message_count": meta["stats"]["messages"],
|
||||
"thread_registry_path": str(INTERNAL_REGISTRY.relative_to(VAULT)),
|
||||
"pages_created": sorted(pages),
|
||||
"pages_updated": ["index.md", "hot.md", "log.md", ".manifest.json"],
|
||||
"bot_visible": False,
|
||||
"note": "Mother-vault internal enhanced extraction. Preserves structured order/quantity/price evidence fields; does not copy full raw email bodies or attachments.",
|
||||
}
|
||||
data["updated"] = NOW
|
||||
data.setdefault("stats", {})["total_sources_ingested"] = len(data.get("sources", {}))
|
||||
man.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
meta = build()
|
||||
INTERNAL_META.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
pages = build_pages(meta)
|
||||
for rel, text in pages.items():
|
||||
write(rel, text)
|
||||
update_special(meta, pages)
|
||||
print(json.dumps({"pages": sorted(pages), "registry": str(INTERNAL_REGISTRY), "threads": meta["stats"]["threads"], "orders": meta["order_ref_thread_count"], "qty": meta["quantity_thread_count"], "prices": meta["price_thread_count"], "generated_at": NOW}, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,541 @@
|
||||
#!/usr/bin/env python3
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
VAULT = Path("/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault")
|
||||
SOURCE = Path("/home/nac/repo/emails_export/parsed/llm_input.jsonl")
|
||||
NOW = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
PART_RE = re.compile(
|
||||
r"\b(?:\d{6}-\d{4,6}(?:-HF)?|\d{3,}-\d{3,}(?:-HF)?|FH\d+[A-Z0-9-]*|C-\d{7})\b",
|
||||
re.I,
|
||||
)
|
||||
KEYWORDS = [
|
||||
"报价",
|
||||
"询价",
|
||||
"样品",
|
||||
"打样",
|
||||
"图纸",
|
||||
"承认书",
|
||||
"规格书",
|
||||
"替代",
|
||||
"替换",
|
||||
"Hirose",
|
||||
"Molex",
|
||||
"TE",
|
||||
"泰科",
|
||||
"JAE",
|
||||
"Kyocera",
|
||||
"IRISO",
|
||||
"订单",
|
||||
"交期",
|
||||
"交货",
|
||||
"品质",
|
||||
"客诉",
|
||||
"投诉",
|
||||
"不良",
|
||||
"8D",
|
||||
"退货",
|
||||
"索赔",
|
||||
"对账",
|
||||
"付款",
|
||||
"发票",
|
||||
"合同",
|
||||
"送样",
|
||||
"开模",
|
||||
"开发",
|
||||
"零跑",
|
||||
"维胜",
|
||||
"恒润",
|
||||
"安悦",
|
||||
"先锋",
|
||||
]
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return "sha256:" + h.hexdigest()
|
||||
|
||||
|
||||
def parse_source():
|
||||
stats = {
|
||||
"threads": 0,
|
||||
"messages": 0,
|
||||
"attachments": 0,
|
||||
"truncated_threads": 0,
|
||||
"date_start": None,
|
||||
"date_end": None,
|
||||
}
|
||||
keywords = collections.Counter()
|
||||
parts = collections.Counter()
|
||||
domains = collections.Counter()
|
||||
months = collections.Counter()
|
||||
attachment_ext = collections.Counter()
|
||||
high_signal = []
|
||||
with SOURCE.open(encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
obj = json.loads(line)
|
||||
stats["threads"] += 1
|
||||
stats["truncated_threads"] += bool(obj.get("body_truncated"))
|
||||
start = obj.get("date_start")
|
||||
end = obj.get("date_end")
|
||||
if start and (stats["date_start"] is None or start < stats["date_start"]):
|
||||
stats["date_start"] = start
|
||||
if end and (stats["date_end"] is None or end > stats["date_end"]):
|
||||
stats["date_end"] = end
|
||||
messages = obj.get("messages", [])
|
||||
text = " ".join(
|
||||
[obj.get("suggested_title_from_subject", "")]
|
||||
+ [
|
||||
(m.get("subject") or "") + " " + (m.get("body_text") or "")
|
||||
for m in messages
|
||||
]
|
||||
)
|
||||
lowered = text.lower()
|
||||
for key in KEYWORDS:
|
||||
if key.lower() in lowered:
|
||||
keywords[key] += 1
|
||||
found_parts = [p.upper() for p in PART_RE.findall(text)]
|
||||
for part in found_parts:
|
||||
parts[part] += 1
|
||||
for msg in messages:
|
||||
stats["messages"] += 1
|
||||
month = (msg.get("date") or "")[:7]
|
||||
if len(month) == 7:
|
||||
months[month] += 1
|
||||
for person in [msg.get("from", {})] + msg.get("to", []) + msg.get("cc", []):
|
||||
if isinstance(person, dict):
|
||||
email = (person.get("email") or "").lower()
|
||||
if "@" in email:
|
||||
domains[email.split("@")[-1]] += 1
|
||||
for item in msg.get("attachments", []) or []:
|
||||
stats["attachments"] += 1
|
||||
if isinstance(item, dict):
|
||||
filename = item.get("filename") or item.get("name") or ""
|
||||
else:
|
||||
filename = str(item)
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "[no_ext]"
|
||||
attachment_ext[ext] += 1
|
||||
score = (
|
||||
sum(lowered.count(key.lower()) for key in KEYWORDS)
|
||||
+ len(found_parts) * 2
|
||||
+ len(messages) // 3
|
||||
+ len(obj.get("attachments", [])) * 2
|
||||
)
|
||||
if score > 120:
|
||||
high_signal.append(
|
||||
{
|
||||
"thread_id": obj.get("thread_id"),
|
||||
"subject": obj.get("suggested_title_from_subject", "")[:160],
|
||||
"date_end": obj.get("date_end"),
|
||||
"message_count": len(messages),
|
||||
"attachment_count": len(obj.get("attachments", [])),
|
||||
"keywords": [k for k in KEYWORDS if k.lower() in lowered][:10],
|
||||
"parts": found_parts[:10],
|
||||
}
|
||||
)
|
||||
high_signal.sort(key=lambda x: (x["date_end"] or "", x["message_count"]), reverse=True)
|
||||
return {
|
||||
"source": str(SOURCE),
|
||||
"generated_at": NOW,
|
||||
"content_hash": sha256(SOURCE),
|
||||
"stats": stats,
|
||||
"keywords_top": keywords.most_common(40),
|
||||
"parts_top": parts.most_common(80),
|
||||
"domains_top": domains.most_common(50),
|
||||
"months_top": months.most_common(18),
|
||||
"attachment_extensions": attachment_ext.most_common(30),
|
||||
"high_signal_threads": high_signal[:120],
|
||||
"redaction_note": "This metadata file contains derived counts and thread-level labels only; it does not store email bodies, prices, order lines, personal contacts, or raw attachments.",
|
||||
}
|
||||
|
||||
|
||||
def write(path: str, text: str):
|
||||
full = VAULT / path
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
full.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def update_index(entries):
|
||||
p = VAULT / "index.md"
|
||||
text = p.read_text(encoding="utf-8")
|
||||
text = re.sub(r"最后更新：[^*]+", f"最后更新：{NOW}", text)
|
||||
for section, lines in entries.items():
|
||||
marker = f"## {section}\n"
|
||||
if marker not in text:
|
||||
text += f"\n{marker}"
|
||||
insert_at = text.index(marker) + len(marker)
|
||||
block = "".join(line + "\n" for line in lines if line not in text)
|
||||
text = text[:insert_at] + block + text[insert_at:]
|
||||
p.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def update_hot():
|
||||
p = VAULT / "hot.md"
|
||||
text = p.read_text(encoding="utf-8")
|
||||
text = re.sub(r"updated: .*", f"updated: {NOW}", text, count=1)
|
||||
item = (
|
||||
f"- [{NOW}] DATA_INGEST - 新增 [[projects/bjd/references/bjd-follow-up-email-corpus-2025-2026]]、"
|
||||
"[[projects/bjd/skills/bjd-customer-email-playbook]] 和 "
|
||||
"[[projects/bjd/concepts/bjd-order-fulfillment-risk-patterns]]；从 2025-2026 跟单邮件 JSONL 中提取产品/RFQ、交期、品质、包装变更和客户话术知识，未录入邮件正文、个人联系方式、客户具体价格或订单数量。\n"
|
||||
)
|
||||
text = text.replace("## 近期活动\n\n", "## 近期活动\n\n" + item, 1)
|
||||
lines = text.splitlines()
|
||||
out = []
|
||||
count = 0
|
||||
in_recent = False
|
||||
for line in lines:
|
||||
if line == "## 近期活动":
|
||||
in_recent = True
|
||||
out.append(line)
|
||||
continue
|
||||
if in_recent and line.startswith("- ["):
|
||||
count += 1
|
||||
if count <= 8:
|
||||
out.append(line)
|
||||
continue
|
||||
out.append(line)
|
||||
p.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def append_log():
|
||||
p = VAULT / "log.md"
|
||||
with p.open("a", encoding="utf-8") as fh:
|
||||
fh.write(
|
||||
f'- [{NOW}] DATA_INGEST source="{SOURCE}" format=jsonl project="bjd" '
|
||||
'pages_created=3 pages_updated=5 raw_email_bodies_archived=false '
|
||||
'sensitive_fields_redacted=true\n'
|
||||
)
|
||||
|
||||
|
||||
def update_manifest(meta):
|
||||
p = VAULT / ".manifest.json"
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
st = SOURCE.stat()
|
||||
key = str(SOURCE)
|
||||
data.setdefault("sources", {})[key] = {
|
||||
"ingested_at": NOW,
|
||||
"size_bytes": st.st_size,
|
||||
"modified_at": datetime.fromtimestamp(st.st_mtime, timezone.utc)
|
||||
.replace(microsecond=0)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
"content_hash": meta["content_hash"],
|
||||
"source_id": meta["content_hash"],
|
||||
"original_path": key,
|
||||
"source_type": "email_jsonl",
|
||||
"doc_type": "followup_email_archive",
|
||||
"business_categories": [
|
||||
"sales/follow-up",
|
||||
"customers/communications",
|
||||
"products/rfq",
|
||||
"quality/customer-feedback",
|
||||
],
|
||||
"project": "bjd",
|
||||
"email_thread_count": meta["stats"]["threads"],
|
||||
"email_message_count": meta["stats"]["messages"],
|
||||
"attachment_count": meta["stats"]["attachments"],
|
||||
"archive_note": "Raw email JSONL and attachments were not copied into the vault because this is sensitive correspondence. Only derived metadata and distilled wiki pages were written.",
|
||||
"raw_email_bodies_archived": False,
|
||||
"source_inventory_path": "_meta/bjd-follow-up-email-corpus-20260605.json",
|
||||
"pages_created": [
|
||||
"projects/bjd/references/bjd-follow-up-email-corpus-2025-2026.md",
|
||||
"projects/bjd/skills/bjd-customer-email-playbook.md",
|
||||
"projects/bjd/concepts/bjd-order-fulfillment-risk-patterns.md",
|
||||
],
|
||||
"pages_updated": [
|
||||
"index.md",
|
||||
"hot.md",
|
||||
"log.md",
|
||||
".manifest.json",
|
||||
"projects/bjd/references/bjd-general-bot-answer-policy.md",
|
||||
],
|
||||
}
|
||||
data["updated"] = NOW
|
||||
data.setdefault("stats", {})["total_sources_ingested"] = len(data.get("sources", {}))
|
||||
data["stats"]["total_pages"] = data.get("stats", {}).get("total_pages", 0) + 3
|
||||
p.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def update_policy():
|
||||
p = VAULT / "projects/bjd/references/bjd-general-bot-answer-policy.md"
|
||||
text = p.read_text(encoding="utf-8")
|
||||
text = re.sub(r"updated: .*", f"updated: {NOW}", text, count=1)
|
||||
if "邮件语料只可用于抽象业务知识" not in text:
|
||||
text = text.replace(
|
||||
"## Allowed\n\n",
|
||||
"## Allowed\n\n"
|
||||
"- 邮件语料只可用于抽象业务知识、流程话术、产品/RFQ模式和风险摘要；不得复述邮件正文、个人联系方式、客户具体价格、具体数量或订单行。\n",
|
||||
1,
|
||||
)
|
||||
text = text.replace(
|
||||
"- 禁止提供原始订单 Excel、客户级订单明细、客诉原始证据、质量图片、邮件内容、索赔单、检验控制文件和个人联系方式。\n",
|
||||
"- 禁止提供原始订单 Excel、客户级订单明细、客诉原始证据、质量图片、邮件正文、附件内容、索赔单、检验控制文件和个人联系方式。\n",
|
||||
1,
|
||||
)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
meta = parse_source()
|
||||
meta_path = VAULT / "_meta/bjd-follow-up-email-corpus-20260605.json"
|
||||
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
overview = f"""---
|
||||
title: BJD 2025-2026 跟单邮件语料知识提取
|
||||
category: references
|
||||
project: bjd
|
||||
tags: [bjd, emails, sales-followup, customers, rfq, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [sales, customers, products, quality, delivery]
|
||||
data_classification: business-confidential
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/bjd-follow-up-email-corpus-20260605.json
|
||||
summary: 从一名跟单员 2025-2026 邮件 JSONL 中蒸馏出的业务知识索引，覆盖 RFQ、样品、交期、品质、包装变更和客户沟通模式，不包含邮件正文或客户级价格数量。
|
||||
provenance:
|
||||
extracted: 0.62
|
||||
inferred: 0.31
|
||||
ambiguous: 0.07
|
||||
base_confidence: 0.68
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: supporting
|
||||
created: {NOW}
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# BJD 2025-2026 跟单邮件语料知识提取
|
||||
|
||||
该页面来自 `/home/nac/repo/emails_export/parsed/llm_input.jsonl` 的衍生统计和抽样蒸馏。源文件包含 {meta["stats"]["threads"]} 个线程、{meta["stats"]["messages"]} 封邮件消息和 {meta["stats"]["attachments"]} 个附件引用，时间跨度约为 {meta["stats"]["date_start"][:10]} 至 {meta["stats"]["date_end"][:10]}。原始邮件正文、个人联系方式、客户具体价格、具体数量、订单行和附件内容没有写入 wiki。^[extracted]
|
||||
|
||||
## 高频业务主题
|
||||
|
||||
- `订单`、`交期`、`交货` 是最高频的履约主题，说明跟单邮箱主要承担订单确认、交期追踪、急料协调和客户反馈闭环。^[extracted]
|
||||
- `报价`、`询价`、`样品`、`打样`、`图纸`、`规格书` 反复出现，说明 RFQ 到样品/图纸/报价输出是核心工作流。^[extracted]
|
||||
- `品质`、`不良`、`客诉`、`8D`、`退货`、`索赔` 频繁出现，邮件中存在较多跨部门质量闭环和责任界定沟通。^[extracted]
|
||||
- `替代`、`Hirose`、`Molex`、`IRISO`、`Kyocera` 等关键词显示，邮件中有不少围绕竞品替代和 VAVE 的机会线索。^[extracted]
|
||||
|
||||
## 客户和市场线索
|
||||
|
||||
- 邮件域名统计显示，高频外部往来覆盖 BICV/北斗智联、青松星瀚、创维、高发/广安、Regal、深南电路、Wintec、零跑、Visteon、BYD、阳光电源、立讯、TCL、Marelli、Yazaki 等客户或生态伙伴。^[extracted]
|
||||
- 印度客户/渠道邮件中，BJD 被用于推介 Hirose FH28K/FH52K、Molex Pin Header、IRISO/Kyocera/Omron 等替代机会，并强调汽车级、耐温、振动测试、图纸/规格书和按现有 MPN+Pin 数做替代表。^[extracted]
|
||||
- BICV/零跑吸顶屏相关询价显示，BJD 既提供连接器，也能围绕客户项目需求协同 FFC 排线开发；关键澄清点包括连接器型号、FFC 是否屏蔽、同面/异面、插接到位丝印线、图纸输出和系统报价。^[extracted]
|
||||
- 深南电路类 RFQ 要求填充阶梯价、年用量、SPQ/MOQ、Lead Time、Pkg Qty 等字段；客户会追问 SPQ/MOQ 是否与包装倍数一致。^[extracted]
|
||||
|
||||
## 产品线和机会
|
||||
|
||||
- FPC/FFC：`101049` 系列、`101110-200250-HF` 等在邮件统计中较高频，主要关联 FPC/FFC 连接器报价、交期、样品和竞品替代。^[extracted]
|
||||
- I/O 与 WTB：`140030-403620`、`141111`、`141114`、`141115`、`241041`、`241111` 等多次出现，关联 36Pin、WTB/Wafer、端子和汽车线束类需求。^[extracted]
|
||||
- 高频/车载：SMB、Fakra、Socket Header 等在海外 RFQ 和 Visteon/VAVE 邮件中出现，需要与 [[projects/bjd/concepts/high-frequency-automotive-connectors]] 联动。^[extracted]
|
||||
- 客户开发话术中经常把 BJD 定位为“连接器+FFC Cable 一体化供应方”，可把 FPC 线缆替换为成本更优的 FFC 方案，但需要客户提供现有线缆规格、应用条件和年用量。^[inferred]
|
||||
|
||||
## 安全边界
|
||||
|
||||
- 该邮件语料只适合沉淀为业务知识、流程话术和风险模式，不适合被机器人直接检索原文。^[inferred]
|
||||
- 客户具体成交价、报价明细、年用量/SOP 数量、订单编号、邮件正文、附件、个人邮箱和手机号不得出现在 `bjd-general-bot` 回答中；如果需要商业分析，只能用聚合或脱敏口径。^[extracted]
|
||||
|
||||
## Related
|
||||
|
||||
- [[projects/bjd/skills/bjd-customer-email-playbook]]
|
||||
- [[projects/bjd/concepts/bjd-order-fulfillment-risk-patterns]]
|
||||
- [[projects/bjd/skills/bjd-sales-engineering-workflow]]
|
||||
- [[projects/bjd/references/bjd-general-bot-answer-policy]]
|
||||
"""
|
||||
write("projects/bjd/references/bjd-follow-up-email-corpus-2025-2026.md", overview)
|
||||
|
||||
playbook = f"""---
|
||||
title: BJD 客户邮件跟进话术手册
|
||||
category: skills
|
||||
project: bjd
|
||||
tags: [bjd, sales, email-playbook, rfq, delivery, quality, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [sales, customers, products, delivery, quality]
|
||||
data_classification: business-confidential
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/bjd-follow-up-email-corpus-20260605.json
|
||||
summary: 从跟单邮件中提炼的客户沟通话术和流程，覆盖 RFQ、样品图纸、交期、包装变更、品质客诉和海外替代提案。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/references/bjd-follow-up-email-corpus-2025-2026]]"
|
||||
type: derived_from
|
||||
- target: "[[projects/bjd/skills/bjd-sales-engineering-workflow]]"
|
||||
type: extends
|
||||
provenance:
|
||||
extracted: 0.46
|
||||
inferred: 0.47
|
||||
ambiguous: 0.07
|
||||
base_confidence: 0.62
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: supporting
|
||||
created: {NOW}
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# BJD 客户邮件跟进话术手册
|
||||
|
||||
该手册从 2025-2026 跟单邮件中抽象沟通模式，供内部销售/跟单/工程协同使用。话术应按实际客户、项目阶段和授权范围调整，不能在对外邮件中泄露其他客户价格、数量、订单或客诉细节。^[inferred]
|
||||
|
||||
## RFQ 和报价
|
||||
|
||||
- 收到客户 RFQ 后，先确认应用、终端/OEM、项目名称、预计 SOP、年用量区间、目标交付地、是否汽车级、是否需要 PPAP/IMDS/CAMDS、现有 MPN、Pin 数、接点方向、FPC/FFC 厚度、锁扣方式和包装要求。^[extracted]
|
||||
- 报价输出建议同步给出 `MOQ/SPQ/Pkg Qty/Lead Time/币种/贸易条款/图纸或规格书版本`，并检查 SPQ/MOQ 是否为包装倍数，避免客户回问。^[extracted]
|
||||
- 客户要求阶梯价时，只在授权报价表中维护客户具体价格；进入机器人或 wiki 安全视图时只保留“需要阶梯价、MOQ、SPQ、Lead Time”的流程知识。^[inferred]
|
||||
|
||||
可用话术：
|
||||
|
||||
> 我们可以评估，请先提供现用料号、Pin 数、应用位置、目标年用量区间和所需认证/提交资料；收到后我们会同步确认 BJD 对应料号、图纸、MOQ/SPQ、Lead Time 和报价有效期。
|
||||
|
||||
## 样品、图纸和规格书
|
||||
|
||||
- 邮件中常见顺序是：客户给应用和连接条件 -> 工程确认连接器/排线方向 -> 输出图纸 -> 业务评估报价 -> 系统建料/报价。^[extracted]
|
||||
- 对 FFC/FPC 类项目，必须确认普通/屏蔽、同面/异面、补强板、金手指方向、插接到位丝印线、长度、公差、冲扣、外加工和匹配座子型号。^[extracted]
|
||||
- 客户只给“替代某竞品”时，不应直接承诺 drop-in；应要求客户提供图纸或现用 MPN，并声明需要确认高度、焊盘、FPC 厚度、接点方向、锁扣和包装。^[inferred]
|
||||
|
||||
可用话术：
|
||||
|
||||
> 现有信息可以先做初步选型，但是否完全替代需要按图纸确认焊盘、外形高度、接点方向、FPC 厚度和锁扣结构。请提供现用规格书或样品，我们会给出 BJD 候选料号和差异点。
|
||||
|
||||
## 海外替代提案
|
||||
|
||||
- 海外客户邮件中有效结构是：感谢会议 -> 总结 BJD 可替代系列 -> 请求现用 MPN/Pin count/SOP qty -> 承诺给出图纸、规格书、候选料号和报价。^[extracted]
|
||||
- 对 Hirose FH28K/FH52K、Molex、IRISO、Kyocera 等替代机会，可以先讲 BJD 有汽车级 FPC/FFC、Pin Header、FFC Cable 方案，再进入逐料号匹配。^[extracted]
|
||||
- 对 FPC cable 成本压力，可提出 FFC cable 替代评估，但必须要求客户提供线缆规格、屏蔽要求、长度、公差、补强、金手指、电镀和测试要求。^[inferred]
|
||||
|
||||
可用英文话术：
|
||||
|
||||
> Please share the existing MPN, pin count, application, annual SOP volume range and drawing/specification. We will check the closest BJD alternative and return the proposal with drawing, specification, MOQ/SPQ, lead time and commercial terms.
|
||||
|
||||
## 交期和急料
|
||||
|
||||
- 交期沟通要区分“客户需求日期、工厂承诺日期、当前库存/在制/外购物料状态、风险点和下一次更新时间”。^[inferred]
|
||||
- 客户急料不能只回复“已催”；需要明确内部责任方、下一步动作和可确认的时间节点。^[inferred]
|
||||
- 供应无法持续满足客户交期时，建议同时给客户阶段性方案：可交数量/可交批次/替代包装或部分交付方案，但不得擅自变更客户已确认包装。^[inferred]
|
||||
|
||||
可用话术：
|
||||
|
||||
> 当前我们已同步生产/仓库确认，今天先反馈可交批次和风险点；如需变更包装或分批交付，会先取得贵司确认后执行。
|
||||
|
||||
## 品质和客诉
|
||||
|
||||
- 客诉邮件中常见风险包括贴胶/不贴胶备注缺失、OK/NG 物料或客户标签混淆、样品包装误用、包装变更未提前沟通、吸塑盘防呆不足和仓库按客户/物料区分不清。^[extracted]
|
||||
- 对客户投诉应先承认收到并启动内部闭环，避免在事实未清前对外归责；内部复盘应拆分订单备注、工程资料、包装规范、仓库标识、出货检验和客户确认。^[inferred]
|
||||
- 涉及 8D、索赔、退货、图片证据时，只能在授权质量流程内处理；机器人只能回答问题类型和改善方向。^[extracted]
|
||||
|
||||
可用话术：
|
||||
|
||||
> 我们已收到反馈并启动内部排查，会从订单要求、工程/包装规范、生产包装、仓库标识和出货检验几个环节确认原因。初步结果和临时遏制措施会在约定时间内反馈。
|
||||
|
||||
## 包装变更
|
||||
|
||||
- 工厂生产包装必须按工程资料或客户已确认口径执行，不能随意变更最小包装、卷装数量、吸塑盘或样品包装。^[extracted]
|
||||
- 若因实际生产需要变更包装，应先提出工程变更/客户变更申请，经业务端和客户确认后再执行。^[extracted]
|
||||
- 不同客户对同一物料的包装接受程度可能不同；一个客户同意变更不代表另一个客户也同意。^[extracted]
|
||||
|
||||
可用话术：
|
||||
|
||||
> 包装方式涉及客户收货、上线和库存管理，未经客户确认我们不会直接切换。若确需变更，我们会提供变更原因、变更前后差异、影响范围和切换时间，请贵司确认后执行。
|
||||
"""
|
||||
write("projects/bjd/skills/bjd-customer-email-playbook.md", playbook)
|
||||
|
||||
risks = f"""---
|
||||
title: BJD 订单履约与品质风险模式
|
||||
category: concepts
|
||||
project: bjd
|
||||
tags: [bjd, fulfillment, delivery, quality, packaging, risk, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [operations, sales, quality, warehouse, engineering]
|
||||
data_classification: business-confidential
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/bjd-follow-up-email-corpus-20260605.json
|
||||
summary: 从跟单邮件中抽象出的履约和质量风险模式，重点是包装变更、订单备注、贴胶、标签、仓库区分、样品管理和客户确认。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/concepts/customer-complaint-recurrence-patterns]]"
|
||||
type: extends
|
||||
- target: "[[projects/bjd/skills/bjd-customer-email-playbook]]"
|
||||
type: mitigated_by
|
||||
provenance:
|
||||
extracted: 0.52
|
||||
inferred: 0.41
|
||||
ambiguous: 0.07
|
||||
base_confidence: 0.64
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: supporting
|
||||
created: {NOW}
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# BJD 订单履约与品质风险模式
|
||||
|
||||
跟单邮件显示，BJD 的客户问题不只来自产品本体，也来自“订单要求 -> 工程资料 -> 生产包装 -> 仓库标识 -> 出货检验 -> 客户沟通”的链路断点。^[inferred]
|
||||
|
||||
## 包装口径不一致
|
||||
|
||||
- 工程资料、工厂实物包装、业务报价/沟通口径、客户已确认包装之间可能不一致。^[extracted]
|
||||
- 最小包装、每卷数量、吸塑盘、卷装/盘装和样品包装如果随意变更，会造成客户拒收、急料无法交付、尾单无法互通或产品挤压变形。^[extracted]
|
||||
- 对策：包装属于客户交付条件，必须有工程资料、业务确认和必要的客户变更确认；品管按确认版检验。^[inferred]
|
||||
|
||||
## 订单备注缺失
|
||||
|
||||
- 贴胶/不贴胶、丝印线、特殊包装、客户标签、样品特殊要求如果只存在历史口头经验或邮件中，没有进入销售订单/工程资料/出货检验点，就容易在交付时失效。^[extracted]
|
||||
- 对策：跟单创建订单时应把客户关键要求结构化写入订单备注或系统字段，并在出货前由仓库/品管按备注确认。^[inferred]
|
||||
|
||||
## 物料和客户标签混淆
|
||||
|
||||
- 邮件中出现 OK/NG 物料错出、客户标签贴错、仓库空间不足导致按客户和物料区分困难的风险。^[extracted]
|
||||
- 对策：高风险客户和相似料号应有客户维度库位/区域/标签防呆；异常盘数或系统差异不能只按“入库差异”解释，需要追溯。^[inferred]
|
||||
|
||||
## 样品订单管理
|
||||
|
||||
- 样品订单容易因为“少量、临时、客户急”而使用非正式包装或非对应吸塑盘，导致产品变形或客户投诉。^[extracted]
|
||||
- 对策：样品订单也要按料号包装规范执行；如果样品确需例外包装，需工程/品质/业务确认并告知客户。^[inferred]
|
||||
|
||||
## 竞品替代和 VAVE 风险
|
||||
|
||||
- 海外 VAVE/RFQ 邮件常以竞品 MPN 开始，但客户后续会要求图纸、规格、价格、MOQ、SOP 数量和交期。^[extracted]
|
||||
- 风险在于把“候选替代”说成“完全替代”；对 Hirose、Molex、IRISO、Kyocera、TE 等竞品必须复核焊盘、结构高度、锁扣、FPC 厚度、接点方向、电镀和测试要求。^[inferred]
|
||||
- 对策：对外统一说“candidate / closest alternative / needs drawing verification”，内部再用 [[projects/bjd/references/competitor-cross-reference-map]] 查候选。^[inferred]
|
||||
|
||||
## 对机器人回答的影响
|
||||
|
||||
- 可以回答这些风险模式和改善建议。
|
||||
- 不能复述具体邮件、处罚内容、索赔金额、订单号、客户具体价格/数量、个人姓名联系方式或附件内容。
|
||||
"""
|
||||
write("projects/bjd/concepts/bjd-order-fulfillment-risk-patterns.md", risks)
|
||||
|
||||
update_policy()
|
||||
update_index(
|
||||
{
|
||||
"参考": [
|
||||
"- [[projects/bjd/references/bjd-follow-up-email-corpus-2025-2026]] - 从 2025-2026 跟单邮件 JSONL 中提取 RFQ、样品、交期、品质、包装变更和客户沟通知识的脱敏索引。 ( #bjd #emails #sales-followup #customers )",
|
||||
],
|
||||
"技能": [
|
||||
"- [[projects/bjd/skills/bjd-customer-email-playbook]] - BJD 客户邮件跟进话术，覆盖 RFQ、样品图纸、交期、包装变更、品质客诉和海外替代提案。 ( #bjd #sales #email-playbook #rfq )",
|
||||
],
|
||||
"概念": [
|
||||
"- [[projects/bjd/concepts/bjd-order-fulfillment-risk-patterns]] - 从跟单邮件中抽象出的履约和品质风险模式，覆盖包装变更、订单备注、贴胶、标签和样品管理。 ( #bjd #fulfillment #quality #risk )",
|
||||
],
|
||||
}
|
||||
)
|
||||
update_hot()
|
||||
append_log()
|
||||
update_manifest(meta)
|
||||
print("wrote pages and metadata")
|
||||
print(json.dumps({"meta": str(meta_path), "now": NOW}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path("/home/nac/repo/obsidian-wiki-linux/kb/obsidian-wiki")
|
||||
VAULT = Path("/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault")
|
||||
SOURCE = Path("/home/nac/repo/emails_export/parsed/llm_input.jsonl")
|
||||
NOW = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def run_signal_extract():
|
||||
raw = subprocess.check_output(
|
||||
[
|
||||
"python3",
|
||||
str(ROOT / "scripts/extract_email_project_signals.py"),
|
||||
str(SOURCE),
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
data = json.loads(raw)
|
||||
data["source"] = str(SOURCE)
|
||||
data["generated_at"] = NOW
|
||||
data["interpretation_note"] = (
|
||||
"Signals are derived from email subjects, bodies, domains and extracted keywords. "
|
||||
"They indicate opportunities, discussions, RFQs, sample/development or order-related activity; "
|
||||
"they do not prove BJD won the project unless separately confirmed by order/project records."
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def write(path, text):
|
||||
p = VAULT / path
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def replace_or_append(path, marker, block):
|
||||
p = VAULT / path
|
||||
text = p.read_text(encoding="utf-8")
|
||||
if marker in text:
|
||||
return
|
||||
p.write_text(text.rstrip() + "\n\n" + block.strip() + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def update_index():
|
||||
p = VAULT / "index.md"
|
||||
text = p.read_text(encoding="utf-8")
|
||||
text = text.replace(
|
||||
text.split("*本索引由 wiki 自动维护。最后更新：", 1)[1].split("*", 1)[0],
|
||||
NOW,
|
||||
1,
|
||||
)
|
||||
line = "- [[projects/bjd/references/bjd-email-project-application-signals-2025-2026]] - 从跟单邮件中整理的项目/应用/OEM线索矩阵，区分机会线索、RFQ、样品开发、订单/履约和未知是否拿到。 ( #bjd #emails #projects #applications )\n"
|
||||
marker = "## 参考\n"
|
||||
if line not in text:
|
||||
text = text.replace(marker, marker + line, 1)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def update_hot():
|
||||
p = VAULT / "hot.md"
|
||||
text = p.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith("updated: "):
|
||||
lines[i] = f"updated: {NOW}"
|
||||
break
|
||||
text = "\n".join(lines) + "\n"
|
||||
item = (
|
||||
f"- [{NOW}] DATA_INGEST - 新增 [[projects/bjd/references/bjd-email-project-application-signals-2025-2026]]；"
|
||||
"从跟单邮件中整理项目/应用/OEM线索，明确这些信息代表机会、RFQ、样品开发或订单履约活动，不等同于已拿到项目。"
|
||||
"后续机器人回答必须保留阶段和置信度。\n"
|
||||
)
|
||||
text = text.replace("## 近期活动\n\n", "## 近期活动\n\n" + item, 1)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def append_log():
|
||||
p = VAULT / "log.md"
|
||||
with p.open("a", encoding="utf-8") as fh:
|
||||
fh.write(
|
||||
f'- [{NOW}] DATA_INGEST source="{SOURCE}" format=jsonl '
|
||||
'page="projects/bjd/references/bjd-email-project-application-signals-2025-2026.md" '
|
||||
'mode=project_application_signal_mining confidence_mixed=true\n'
|
||||
)
|
||||
|
||||
|
||||
def update_manifest():
|
||||
p = VAULT / ".manifest.json"
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
key = "derived:/home/nac/repo/emails_export/parsed/llm_input.jsonl::project-application-signals-20260605"
|
||||
data.setdefault("sources", {})[key] = {
|
||||
"ingested_at": NOW,
|
||||
"source_type": "derived_email_signal_index",
|
||||
"project": "bjd",
|
||||
"source_id": key,
|
||||
"original_path": str(SOURCE),
|
||||
"doc_type": "project_application_signal_index",
|
||||
"business_categories": [
|
||||
"sales/projects",
|
||||
"customers/applications",
|
||||
"products/rfq",
|
||||
"market/oem-signals",
|
||||
],
|
||||
"note": "Derived project/application/OEM signals from the follow-up email JSONL. Signals do not imply project win unless separately confirmed.",
|
||||
"source_inventory_path": "_meta/bjd-email-project-application-signals-20260605.json",
|
||||
"pages_created": [
|
||||
"projects/bjd/references/bjd-email-project-application-signals-2025-2026.md"
|
||||
],
|
||||
"pages_updated": [
|
||||
"projects/bjd/references/bjd-follow-up-email-corpus-2025-2026.md",
|
||||
"index.md",
|
||||
"hot.md",
|
||||
"log.md",
|
||||
],
|
||||
}
|
||||
data["updated"] = NOW
|
||||
data.setdefault("stats", {})["total_sources_ingested"] = len(data.get("sources", {}))
|
||||
p.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
data = run_signal_extract()
|
||||
meta_path = VAULT / "_meta/bjd-email-project-application-signals-20260605.json"
|
||||
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
meta_path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
page = f"""---
|
||||
title: BJD 邮件项目应用线索 2025-2026
|
||||
category: references
|
||||
project: bjd
|
||||
tags: [bjd, emails, projects, applications, oem, rfq, visibility/internal]
|
||||
visibility: internal
|
||||
business_domains: [sales, customers, products, market]
|
||||
data_classification: commercial-confidential
|
||||
allowed_bots: [bjd-general-bot]
|
||||
sources:
|
||||
- _meta/bjd-email-project-application-signals-20260605.json
|
||||
- _meta/bjd-follow-up-email-corpus-20260605.json
|
||||
summary: 从 2025-2026 跟单邮件中整理项目、应用、终端车厂和产品机会线索；这些线索代表邮件讨论或商务机会，不自动等同于已中标或已量产。
|
||||
relationships:
|
||||
- target: "[[projects/bjd/references/bjd-follow-up-email-corpus-2025-2026]]"
|
||||
type: derived_from
|
||||
- target: "[[projects/bjd/skills/bjd-customer-email-playbook]]"
|
||||
type: informs
|
||||
- target: "[[projects/bjd/concepts/bjd-product-portfolio]]"
|
||||
type: related_to
|
||||
provenance:
|
||||
extracted: 0.55
|
||||
inferred: 0.35
|
||||
ambiguous: 0.10
|
||||
base_confidence: 0.60
|
||||
lifecycle: draft
|
||||
lifecycle_changed: 2026-06-05
|
||||
tier: supporting
|
||||
created: {NOW}
|
||||
updated: {NOW}
|
||||
---
|
||||
|
||||
# BJD 邮件项目应用线索 2025-2026
|
||||
|
||||
该页从跟单邮件 JSONL 中提取“客户/渠道 - 终端车厂 - 项目/应用 - BJD 产品方向”的线索。它的价值在于发现真实客户场景和产品机会，但必须注意：邮件出现不等于 BJD 已经拿到项目；可能只是 RFQ、样品、技术交流、报价、交期沟通、PPAP 或质量闭环。^[inferred]
|
||||
|
||||
## 使用规则
|
||||
|
||||
- 回答“我们产品用在哪些应用/车厂”时，必须带阶段和置信度：`已供货/订单线索`、`RFQ/报价`、`样品/开发`、`质量/履约`、`仅机会线索`、`未知是否拿到`。^[inferred]
|
||||
- 不输出邮件正文、个人联系方式、客户具体价格、客户具体数量、订单编号、附件内容或索赔细节。^[extracted]
|
||||
- 若要判断是否真实量产或已供货，应回查订单聚合、项目台账、图纸/承认书、客户定点或出货记录。邮件线索本身只能作为入口。^[inferred]
|
||||
|
||||
## 应用和产品方向
|
||||
|
||||
- FFC/FPC 是最高频应用方向，覆盖显示、吸顶屏、仪表/Cluster、CID/FID、排线、样品打样和竞品替代。^[extracted]
|
||||
- WTB、Wafer、BTB、Pin Header 在 BYD、Visteon、Marelli、VW/Skoda 等线索中反复出现。^[extracted]
|
||||
- FAKRA/SMB/高频连接器出现于车载显示、摄像头、VAVE 或高频方案讨论中，需要与 [[projects/bjd/concepts/high-frequency-automotive-connectors]] 交叉确认。^[inferred]
|
||||
|
||||
## 重点项目/应用线索
|
||||
|
||||
| 客户/渠道 | 终端/OEM | 项目或应用 | BJD 产品方向 | 阶段判断 | 置信度 | 说明 |
|
||||
|---|---|---|---|---|---|---|
|
||||
| BICV/北斗智联 | 零跑/Leapmotor | 吸顶屏、霍尔小板连接 | FFC 排线 + BJD FPC 座 | RFQ/开发线索 | 高 | 邮件明确提到零跑项目、普通 FFC、异面、插接到位丝印线、图纸输出和报价流程。 |
|
||||
| Visteon | Mercedes-Benz MB.EA-L Front Display | Front Display / CID | FFC、Pin Header、WTB、FH75 相关候选 | RFQ/技术讨论 | 中 | 邮件主题和线索指向前显示项目；是否定点或量产需另查订单/项目记录。 |
|
||||
| Visteon | Scout B-RUV / C-PICKUP | CID / FID | 显示类连接器、候选替代 | RFQ/报价线索 | 中 | 邮件主题含 RFQ、CID/FID、Chihuahua Plant；属于机会线索。 |
|
||||
| Visteon | Toyota X-Carline | Full Digital Cluster | FPC/FFC、WTB、FH63 相关线索 | RFQ/样品或开发线索 | 中 | 与仪表/Cluster 相关，需图纸和订单记录确认具体产品。 |
|
||||
| Marelli | BMW projects | CID/FID/lighting | Pin Header | 新开发机会 | 中 | 邮件主题指向 BMW 项目 pin header new development。 |
|
||||
| BYD/比亚迪 | BYD | WTB、FPC/FFC、BTB、车灯、储能 | WTB、BTB、FPC/FFC、端子 | 混合：订单/RFQ/质量/商务 | 中 | BYD 高频出现，但类型混合，需按单个线程区分订单确认、招标、QMS、对账或质量。 |
|
||||
| BYD/比亚迪 | BYD E 项目 | 20Pin 浮动 BTB | BTB floating connector | 寻源/技术交流 | 高 | 邮件主题明确为“E项目 20pin浮动btb连接器寻源”。 |
|
||||
| BYD/比亚迪 | 第十一事业部 | 0.63/1.2 规格免焊端子 | 端子开发 | 招标/技术交流 | 高 | 邮件主题明确指向端子开发招标/供应商意愿确认。 |
|
||||
| VW/Skoda | Skoda/VW | AP SVM Camera / PPAP | WTB/Wafer | PPAP/项目资料 | 中 | 邮件主题为 PPAP pending，说明可能进入资料提交或项目支持阶段。 |
|
||||
| Valeo | Valeo BLADE | ZIF connectors | FPC ZIF | 询价/技术咨询 | 中 | 邮件主题指向 ZIF connectors，具体 BJD 料号需二次确认。 |
|
||||
| MZD 项目 | MZD | FPC/FFC 软排线 | 101049 系列方向/排线 | 打样/开发 | 中 | 邮件中有 MZD 项目软排线打样和样品报价线索。 |
|
||||
| GF/高发或广安相关 | 储能 | 储能备货 | WTB/端子类 | 备货/交期 | 中 | 邮件中出现储能备货和 WTB/端子类物料，需订单记录确认客户和项目。 |
|
||||
|
||||
## 高频 OEM/终端线索
|
||||
|
||||
- BYD/比亚迪、零跑、Visteon、Marelli、VW/SVW/Skoda、Valeo、Toyota、Yazaki、Kia、Stellantis/STLA 都在邮件线索中出现。^[extracted]
|
||||
- BYD/比亚迪线索最多，但混合了订单确认、寻源、QMS、对账和质量流程，不应直接归为全部已供货项目。^[inferred]
|
||||
- Visteon 线索集中在 CID/FID、Cluster、WTB、Pin Header、FPC/FFC 和部分高频连接器/VAVE。^[extracted]
|
||||
- 零跑线索较集中，尤其是 BICV/北斗智联吸顶屏 FFC 开发。^[extracted]
|
||||
|
||||
## 后续挖掘方向
|
||||
|
||||
- 对每个高价值项目建立二级页：`客户/OEM + 项目 + 应用 + 产品族 + 竞品 + 阶段 + 证据来源`。^[inferred]
|
||||
- 用订单聚合表交叉验证邮件项目线索，将“RFQ/机会”升级或降级为“已报价、已送样、已供货、无结果”。^[inferred]
|
||||
- 对海外 VAVE 线索单独建立 `Hirose/Molex/IRISO/Kyocera 替代机会池`，避免散落在邮件线程里。^[inferred]
|
||||
|
||||
## Related
|
||||
|
||||
- [[projects/bjd/references/bjd-follow-up-email-corpus-2025-2026]]
|
||||
- [[projects/bjd/skills/bjd-customer-email-playbook]]
|
||||
- [[projects/bjd/references/bjd-fast-product-lookup]]
|
||||
- [[projects/bjd/references/competitor-cross-reference-map]]
|
||||
"""
|
||||
write("projects/bjd/references/bjd-email-project-application-signals-2025-2026.md", page)
|
||||
|
||||
replace_or_append(
|
||||
"projects/bjd/references/bjd-follow-up-email-corpus-2025-2026.md",
|
||||
"[[projects/bjd/references/bjd-email-project-application-signals-2025-2026]]",
|
||||
"""
|
||||
## Project/Application Signal Index
|
||||
|
||||
- [[projects/bjd/references/bjd-email-project-application-signals-2025-2026]] extracts project, application, OEM and product opportunity signals from the same email corpus. These are opportunity/RFQ/development/order-support signals and do not automatically mean BJD won the project. ^[inferred]
|
||||
""",
|
||||
)
|
||||
|
||||
update_index()
|
||||
update_hot()
|
||||
append_log()
|
||||
update_manifest()
|
||||
print(json.dumps({"page": "projects/bjd/references/bjd-email-project-application-signals-2025-2026.md", "meta": str(meta_path), "now": NOW}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
ROOT = Path('/home/nac/smb/品质部/品质管理资料夹2009.11.3/PPAP资料')
|
||||
VAULT = Path('/home/nac/repo/obsidian-wiki-linux/kb/bjd-vault')
|
||||
META = VAULT / '_meta'
|
||||
OUT = VAULT / 'projects/bjd/references'
|
||||
RUN_DATE = '20260611'
|
||||
|
||||
CATEGORY_PATTERNS = [
|
||||
('ppap_submission_list', r'PPAP提交清单|PPAP文件清单|文件清单'),
|
||||
('psw_part_submission_warrant', r'零件提交保证书|PSW|Part Submission Warrant'),
|
||||
('dimensional_report', r'全尺寸|寸法|尺寸|样品检验|尺寸测量|全尺寸报告'),
|
||||
('msa_grr', r'MSA|GR&R|GRR|量测系统|测量系统|计数型'),
|
||||
('process_capability', r'CPK|PPK|CMK|过程能力|设备能力|X-R|X-R图'),
|
||||
('reliability_test', r'可靠性|测试报告|测试成绩|低温|高温|耐湿|盐雾|沾锡|可焊|耐焊热|保持力|接触电阻|绝缘耐压|重复插拔|拉拔|DV|PV'),
|
||||
('drawing', r'图纸|产品图|包装图|Model|平面图|胶芯|端子|后盖|垫片|组立品|完成品'),
|
||||
('material_compliance', r'SGS|RoHS|Halogen|卤|材料|C5210|C5191|C2680|LCP|PA9T|GN2330|MG350'),
|
||||
('control_plan', r'控制计划|Control Plan|产控制计划'),
|
||||
('pfmea_dfmea', r'PFMEA|DFMEA|失效模式'),
|
||||
('process_flow', r'流程图|工艺流程|过程流程'),
|
||||
('apqp_plan_review', r'APQP|先期策划|工作计划|阶段评审|可行性|项目小组|立项|设计输入|设计输出|设计开发|试产总结|质量策划'),
|
||||
('specification', r'规格书|产品质量标准|标准样品|特殊特性'),
|
||||
('equipment_training', r'设备|量具|试验装置|培训|作业人员|技能'),
|
||||
('lab_manual', r'实验室手册'),
|
||||
]
|
||||
CATEGORY_LABELS = {
|
||||
'ppap_submission_list': 'PPAP提交清单/文件清单',
|
||||
'psw_part_submission_warrant': '零件提交保证书/PSW',
|
||||
'dimensional_report': '全尺寸/样品寸法报告',
|
||||
'msa_grr': 'MSA/GR&R/测量系统',
|
||||
'process_capability': 'CPK/PPK/CMK/过程能力',
|
||||
'reliability_test': '可靠性/DV/PV/功能测试',
|
||||
'drawing': '图纸/结构件图',
|
||||
'material_compliance': '材料/SGS/RoHS/有卤无卤',
|
||||
'control_plan': '控制计划',
|
||||
'pfmea_dfmea': 'PFMEA/DFMEA',
|
||||
'process_flow': '过程流程/工艺流程',
|
||||
'apqp_plan_review': 'APQP计划/评审/可行性',
|
||||
'specification': '规格书/标准样品/特殊特性',
|
||||
'equipment_training': '设备/量具/培训',
|
||||
'lab_manual': '实验室手册',
|
||||
'other': '其他',
|
||||
}
|
||||
|
||||
CUSTOMER_PATTERNS = [
|
||||
('Valeo/法雷奥', r'法雷奥|Valeo'),
|
||||
('Jicheng/集诚', r'集诚|集成'),
|
||||
('Ford', r'18B966|Ford'),
|
||||
('Ineos', r'Ineos'),
|
||||
]
|
||||
|
||||
FAMILY_PATTERNS = [
|
||||
('FPC 0.3-3', r'0\.3-3'),
|
||||
('FPC 0.5系列', r'0\.5-\d+'),
|
||||
('FPC 0.8系列', r'0\.8-\d+'),
|
||||
('FPC 1.0系列', r'1\.0-\d+|1\.0PH'),
|
||||
('FPC 1.25-2', r'1\.25-2'),
|
||||
('FFC/FPC排线', r'FFC|FPC'),
|
||||
('BTB 0.5-5', r'BTB0\.5-5|BTB'),
|
||||
('FAKRA', r'FAKRA'),
|
||||
('WTB/Wafer 2.0', r'WAFER|Wafer|2\.0-10|2\.0-12'),
|
||||
('WTB 28+4P', r'28\+4'),
|
||||
('LCD 2.0-2', r'LCD ?2\.0-2'),
|
||||
('ISO-C P5.0', r'ISO-C|P5\.0'),
|
||||
('Pin Header/弯插', r'P1\.8|P2\.2|P2\.2-3|P3\.5|P5\.0|10PIN|40P弯插|36P弯插'),
|
||||
('P13端子', r'P13'),
|
||||
]
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def sha1_path(path: Path) -> str:
|
||||
return hashlib.sha1(str(path).encode('utf-8', 'ignore')).hexdigest()[:12]
|
||||
|
||||
|
||||
def clean(s: str) -> str:
|
||||
return re.sub(r'\s+', ' ', s.replace('\x00', ' ')).strip()
|
||||
|
||||
|
||||
def infer_category(text: str) -> str:
|
||||
for cat, pat in CATEGORY_PATTERNS:
|
||||
if re.search(pat, text, re.I):
|
||||
return cat
|
||||
return 'other'
|
||||
|
||||
|
||||
def infer_all_categories(text: str) -> list[str]:
|
||||
cats = []
|
||||
for cat, pat in CATEGORY_PATTERNS:
|
||||
if re.search(pat, text, re.I):
|
||||
cats.append(cat)
|
||||
return cats or ['other']
|
||||
|
||||
|
||||
def infer_customer(text: str) -> list[str]:
|
||||
out=[]
|
||||
for label, pat in CUSTOMER_PATTERNS:
|
||||
if re.search(pat, text, re.I):
|
||||
out.append(label)
|
||||
return out
|
||||
|
||||
|
||||
def infer_family(text: str) -> list[str]:
|
||||
out=[]
|
||||
for label, pat in FAMILY_PATTERNS:
|
||||
if re.search(pat, text, re.I):
|
||||
out.append(label)
|
||||
# more exact series tokens, useful for lookup
|
||||
for token in sorted(set(re.findall(r'(?<!\d)(?:0\.3|0\.5|0\.8|1\.0|1\.25|1\.8|2\.0|2\.2|3\.5|5\.0)-\d+(?:B)?', text, re.I))):
|
||||
out.append(token)
|
||||
return list(dict.fromkeys(out))
|
||||
|
||||
|
||||
def xlsx_text_sample(path: Path, max_chars=5000) -> str:
|
||||
try:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
names = set(z.namelist())
|
||||
shared=[]
|
||||
if 'xl/sharedStrings.xml' in names:
|
||||
root = ET.fromstring(z.read('xl/sharedStrings.xml'))
|
||||
ns = {'a':'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
||||
for si in root.findall('.//a:si', ns):
|
||||
parts=[t.text or '' for t in si.findall('.//a:t', ns)]
|
||||
if parts:
|
||||
shared.append(''.join(parts))
|
||||
cells=[]
|
||||
for name in sorted(n for n in names if n.startswith('xl/worksheets/sheet') and n.endswith('.xml'))[:3]:
|
||||
root=ET.fromstring(z.read(name))
|
||||
ns={'a':'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
||||
for c in root.findall('.//a:c', ns)[:300]:
|
||||
v=c.find('a:v', ns)
|
||||
if v is None or v.text is None:
|
||||
continue
|
||||
val=v.text
|
||||
if c.get('t') == 's':
|
||||
try:
|
||||
val=shared[int(val)]
|
||||
except Exception:
|
||||
pass
|
||||
cells.append(val)
|
||||
return clean(' '.join(cells))[:max_chars]
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def binary_text_sample(path: Path, max_chars=2000) -> str:
|
||||
# Small, dependency-free fallback for old xls/doc/pdf names/text fragments. It is not a full parser.
|
||||
try:
|
||||
data = path.read_bytes()[:2_000_000]
|
||||
except Exception:
|
||||
return ''
|
||||
words=[]
|
||||
cur=[]
|
||||
for b in data:
|
||||
if 32 <= b <= 126:
|
||||
cur.append(chr(b))
|
||||
else:
|
||||
if len(cur) >= 4:
|
||||
words.append(''.join(cur))
|
||||
cur=[]
|
||||
if len(' '.join(words)) > max_chars:
|
||||
break
|
||||
if len(cur) >= 4:
|
||||
words.append(''.join(cur))
|
||||
return clean(' '.join(words))[:max_chars]
|
||||
|
||||
|
||||
def collect():
|
||||
records=[]
|
||||
for path in ROOT.rglob('*'):
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
st=path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
rel=path.relative_to(ROOT).as_posix()
|
||||
top=rel.split('/')[0]
|
||||
ext=path.suffix.lower().lstrip('.') or 'no_ext'
|
||||
text_context=f'{rel} {path.name}'
|
||||
sample=''
|
||||
if ext == 'xlsx':
|
||||
sample=xlsx_text_sample(path)
|
||||
elif ext in {'txt','log'}:
|
||||
try: sample=path.read_text(encoding='utf-8', errors='ignore')[:5000]
|
||||
except Exception: sample=''
|
||||
elif ext in {'xls','doc','pdf'}:
|
||||
# Only use binary strings for classification hints; full extraction requires external tools/OCR.
|
||||
sample=binary_text_sample(path, max_chars=1200)
|
||||
all_text=f'{text_context} {sample}'
|
||||
cats=infer_all_categories(all_text)
|
||||
records.append({
|
||||
'id': sha1_path(path),
|
||||
'path': str(path),
|
||||
'relative_path': rel,
|
||||
'top_package': top,
|
||||
'filename': path.name,
|
||||
'extension': ext,
|
||||
'size_bytes': st.st_size,
|
||||
'modified_at': datetime.fromtimestamp(st.st_mtime, timezone.utc).replace(microsecond=0).isoformat().replace('+00:00','Z'),
|
||||
'categories': cats,
|
||||
'primary_category': cats[0],
|
||||
'families': infer_family(all_text),
|
||||
'customers': infer_customer(all_text),
|
||||
'text_sample': clean(sample)[:1000],
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def write_jsonl(path, rows):
|
||||
with path.open('w', encoding='utf-8') as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
def write_csv(path, rows):
|
||||
fields=['id','relative_path','top_package','filename','extension','size_bytes','modified_at','primary_category','categories','families','customers','path']
|
||||
with path.open('w', encoding='utf-8-sig', newline='') as f:
|
||||
w=csv.DictWriter(f, fieldnames=fields)
|
||||
w.writeheader()
|
||||
for r in rows:
|
||||
row={k:r.get(k,'') for k in fields}
|
||||
for k in ['categories','families','customers']:
|
||||
row[k]='; '.join(row[k] or [])
|
||||
w.writerow(row)
|
||||
|
||||
|
||||
def package_summary(records):
|
||||
by=defaultdict(list)
|
||||
for r in records:
|
||||
by[r['top_package']].append(r)
|
||||
rows=[]
|
||||
for pkg, rs in by.items():
|
||||
cats=Counter(c for r in rs for c in r['categories'])
|
||||
exts=Counter(r['extension'] for r in rs)
|
||||
fam=Counter(f for r in rs for f in r['families'])
|
||||
cust=Counter(c for r in rs for c in r['customers'])
|
||||
notable=[]
|
||||
wanted=['ppap_submission_list','psw_part_submission_warrant','dimensional_report','msa_grr','process_capability','reliability_test','pfmea_dfmea','control_plan','drawing','material_compliance']
|
||||
for cat in wanted:
|
||||
hit=next((r for r in rs if cat in r['categories']), None)
|
||||
if hit:
|
||||
notable.append(f"{CATEGORY_LABELS[cat]}: {hit['filename']}")
|
||||
rows.append({
|
||||
'package': pkg,
|
||||
'file_count': len(rs),
|
||||
'extensions': dict(exts.most_common()),
|
||||
'categories': dict(cats.most_common()),
|
||||
'families': [x for x,_ in fam.most_common(8)],
|
||||
'customers': [x for x,_ in cust.most_common(5)],
|
||||
'notable_files': notable[:12],
|
||||
})
|
||||
rows.sort(key=lambda x: (-x['file_count'], x['package']))
|
||||
return rows
|
||||
|
||||
|
||||
def esc(s):
|
||||
return str(s).replace('|','\\|').replace('\n',' ')
|
||||
|
||||
|
||||
def table_packages(rows, limit=80):
|
||||
lines=['| 资料包 | 文件数 | 识别产品/客户 | 覆盖资料类型 | 代表文件 |','|---|---:|---|---|---|']
|
||||
for r in rows[:limit]:
|
||||
cats='；'.join(CATEGORY_LABELS.get(k,k) for k in list(r['categories'].keys())[:8])
|
||||
fc='；'.join((r['families']+r['customers'])[:8]) or '未识别'
|
||||
nf='；'.join(r['notable_files'][:4])
|
||||
lines.append(f"| {esc(r['package'])} | {r['file_count']} | {esc(fc)} | {esc(cats)} | {esc(nf)} |")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def category_table(counter):
|
||||
lines=['| 类型 | 文件数 |','|---|---:|']
|
||||
for k,v in counter.most_common():
|
||||
lines.append(f"| {CATEGORY_LABELS.get(k,k)} | {v} |")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def write_pages(records, packages, jsonl, csvp, package_json):
|
||||
ts=now_iso()
|
||||
total=len(records)
|
||||
ext=Counter(r['extension'] for r in records)
|
||||
cats=Counter(c for r in records for c in r['categories'])
|
||||
fam=Counter(f for r in records for f in r['families'])
|
||||
cust=Counter(c for r in records for c in r['customers'])
|
||||
top_packages=packages[:20]
|
||||
|
||||
index_page=OUT / f'bjd-ppap-apqp-archive-index-{RUN_DATE}.md'
|
||||
index_lines=[
|
||||
'---',
|
||||
'title: BJD PPAP/APQP 资料库索引',
|
||||
'category: reference',
|
||||
'project: bjd',
|
||||
'tags: [bjd, quality, ppap, apqp, msa, cpk, reliability, visibility/internal]',
|
||||
'visibility: internal',
|
||||
'business_domains: [quality, engineering, products]',
|
||||
'data_classification: technical-confidential',
|
||||
f'updated: {ts}',
|
||||
'---','',
|
||||
'# BJD PPAP/APQP 资料库索引','',
|
||||
f'本页索引来源目录 `{ROOT}`。本次共登记 `{total}` 个文件、`{len(packages)}` 个顶层资料包。由于环境缺少 LibreOffice/pdftotext/catdoc，旧版 `.xls/.doc/.pdf` 暂以文件名、路径层级和可读片段做结构化索引；`.xlsx` 文件做了轻量文本抽取。','',
|
||||
'## 文件类型分布','',
|
||||
', '.join(f'`.{k}`={v}' for k,v in ext.most_common(20)), '',
|
||||
'## 资料类型覆盖','',
|
||||
category_table(cats), '',
|
||||
'## 识别到的产品族/客户线索','',
|
||||
'- 产品族/系列：' + '；'.join(f'{k}({v})' for k,v in fam.most_common(30)),
|
||||
'- 客户/项目线索：' + ('；'.join(f'{k}({v})' for k,v in cust.most_common(20)) or '未从路径显式识别。'),'',
|
||||
'## 重点资料包','',
|
||||
table_packages(top_packages, limit=20), '',
|
||||
'## 检索口径','',
|
||||
'- 查某个系列 PPAP 完整性：检索系列名加 `PPAP提交清单`、`零件提交保证书`、`全尺寸`、`MSA`、`CPK/PPK/CMK`、`可靠性`。',
|
||||
'- 查客户项目资料：优先检索客户名或项目名，例如 `法雷奥 28+4P APQP`、`集诚 10PIN PPAP`、`Ford FFC PPAP`。',
|
||||
'- 查质量能力证据：检索 `MSA GR&R`、`过程能力`、`可靠性测试`、`DV PV`、`控制计划`、`PFMEA`。',
|
||||
'- 正式对外提交前仍需打开原文件确认版本、签核、日期和客户要求；本页是索引和知识入口，不替代正式 PPAP 文件包。','',
|
||||
'## 数据文件','',
|
||||
f'- 文件清单 JSONL：`_meta/{jsonl.name}`',
|
||||
f'- 文件清单 CSV：`_meta/{csvp.name}`',
|
||||
f'- 资料包汇总 JSON：`_meta/{package_json.name}`',
|
||||
]
|
||||
index_page.write_text('\n'.join(index_lines)+'\n', encoding='utf-8')
|
||||
|
||||
map_page=OUT / f'bjd-ppap-apqp-package-map-{RUN_DATE}.md'
|
||||
map_lines=[
|
||||
'---','title: BJD PPAP/APQP 资料包证据地图','category: reference','project: bjd',
|
||||
'tags: [bjd, quality, ppap, apqp, evidence-map, visibility/internal]',
|
||||
'visibility: internal','business_domains: [quality, engineering, products]','data_classification: technical-confidential',f'updated: {ts}','---','',
|
||||
'# BJD PPAP/APQP 资料包证据地图','',
|
||||
'本页按顶层资料包展示文件覆盖情况，用来快速判断某个系列或客户项目有哪些 PPAP/APQP 证据。','',
|
||||
table_packages(packages, limit=120),
|
||||
]
|
||||
map_page.write_text('\n'.join(map_lines)+'\n', encoding='utf-8')
|
||||
|
||||
knowledge_page=OUT / f'bjd-ppap-apqp-quality-knowledge-{RUN_DATE}.md'
|
||||
knowledge_lines=[
|
||||
'---','title: BJD PPAP/APQP 质量资料知识提炼','category: reference','project: bjd',
|
||||
'tags: [bjd, quality, ppap, apqp, lessons, msa, cpk, reliability, visibility/internal]',
|
||||
'visibility: internal','business_domains: [quality, engineering, products]','data_classification: technical-confidential',f'updated: {ts}','---','',
|
||||
'# BJD PPAP/APQP 质量资料知识提炼','',
|
||||
'## 总体结构','',
|
||||
'BJD 的 PPAP/APQP 资料包通常围绕以下证据链组织：PPAP提交清单、零件提交保证书、产品/零部件图纸、全尺寸或样品寸法检验、MSA/GR&R、过程能力 CPK/PPK/CMK、可靠性/DV/PV 测试、材料/SGS/RoHS/卤素报告、控制计划、PFMEA/DFMEA、过程流程图、标准样品或特殊特性清单。','',
|
||||
'## FPC 系列资料特点','',
|
||||
'0.3-3、0.5-3 到 0.5-26、0.8系列、1.0系列等 FPC 资料包大量复用同一套 PPAP 结构：提交清单、PSW、端子/胶芯/后盖/垫片/组立品全尺寸报告、初始过程能力、MSA GR&R、可靠性测试、标准样品和结构图。0.5-15/0.5-16 等资料中还出现 C2680、C5210、LCP MG350/E130i 等材料合规报告。','',
|
||||
'## 0.5-18 APQP 是较完整的新项目模板','',
|
||||
'0.5-18 APQP 资料包含 APQP 工作计划、测试设备能力调查、FPC产品规格书、DV/PV试验测试工作计划、可靠性测试成绩、MSA计划、过程能力研究计划、PPK报告、CCD MSA/CMK、自动机 CCD 与工具显微镜对比，以及保持力、可焊性、接触电阻、绝缘耐压、耐焊热、重复插拔等样品阶段测试。它适合作为 FPC 新项目 PPAP/APQP 完整资料包的参考样板。','',
|
||||
'## 法雷奥 28+4P 资料包','',
|
||||
'法雷奥28+4P APQP 产品资料覆盖 WTB 28+4P 的 DV/PV 测试报告、测试工作计划、委外测试报告、半自动机设备验收、CCD MSA、电测 MSA、设备能力 CMK、板端/线端胶壳与端子全尺寸、过程能力计划、MSA计划、现场作业人员技能培训等。该资料包是汽车客户项目 PPAP/APQP 证据较完整的案例。','',
|
||||
'## FFC/Ford 相关资料','',
|
||||
'目录中存在 APQP FFC VPKSJF-18B966-AAD、PPAP FFC VPHNMF-18B966-AB、VPDNNMF-18B966-AD、FFC A-0.5-60P105mm 等资料包，说明 FFC/排线项目的 PPAP 证据链也被单独归档，且 Ford 18B966 相关料号在资料包命名中反复出现。','',
|
||||
'## Pin Header/弯插与 WTB/Wafer 资料','',
|
||||
'P2.2-2-40P、P2.2-3-16P、P1.8-2-36P、P3.5-1-10P、P5.0-2-15P、集诚10PIN、WAFER 2.0-12、Wafer 2.0-10-4P 等资料包显示，BJD 对弯插、Pin Header、线对板/Wafer 产品也有 PPAP/APQP 归档。集诚10PIN资料包含规格书、特殊特性清单、尺寸测试、CPK、产品图/包装图、DFMEA、工艺流程、PFMEA、控制计划等。','',
|
||||
'## 使用注意','',
|
||||
'- 这批资料多数是正式质量文件或客户项目证据，默认应按 internal/technical-confidential 处理。',
|
||||
'- 检索结果只能说明资料存在或路径命中；是否可对外提交，要检查原文件中的版本、签核、客户名称、日期、适用料号和保密要求。',
|
||||
'- 旧版 `.xls/.doc/.pdf` 暂未完整表格抽取；如果需要精确数值，比如 CPK/PPK、GR&R百分比、可靠性结论，需要后续安装 LibreOffice 或专用解析器后做二次深抽。',
|
||||
]
|
||||
knowledge_page.write_text('\n'.join(knowledge_lines)+'\n', encoding='utf-8')
|
||||
|
||||
update_meta(ts, [index_page, map_page, knowledge_page], jsonl, csvp, package_json, total, len(packages))
|
||||
return [index_page, map_page, knowledge_page]
|
||||
|
||||
|
||||
def update_meta(ts, pages, jsonl, csvp, package_json, total, package_count):
|
||||
idx=VAULT/'index.md'
|
||||
if idx.exists():
|
||||
text=idx.read_text(encoding='utf-8')
|
||||
text=re.sub(r'最后更新：[^*]+', f'最后更新：{ts}', text)
|
||||
marker='## 综合分析\n'
|
||||
entries=[]
|
||||
for p in pages:
|
||||
rel=p.relative_to(VAULT).with_suffix('').as_posix()
|
||||
entries.append(f'- [[{rel}]] - PPAP/APQP 质量资料目录 ingest：文件清单、资料包地图与知识提炼。 ( #bjd #quality #ppap #apqp )')
|
||||
insert='\n'.join(e for e in entries if e.split(']]')[0][4:] not in text)
|
||||
if insert:
|
||||
text=text.replace(marker, marker+insert+'\n', 1) if marker in text else text+'\n## 综合分析\n'+insert+'\n'
|
||||
idx.write_text(text, encoding='utf-8')
|
||||
with (VAULT/'log.md').open('a', encoding='utf-8') as f:
|
||||
f.write(f'\n- [{ts}] DATA_INGEST source="{ROOT}" project="bjd" mode="ppap-apqp-archive-index" files={total} packages={package_count} pages="{",".join(p.relative_to(VAULT).as_posix() for p in pages)}" inventory="{jsonl.relative_to(VAULT).as_posix()}" csv="{csvp.relative_to(VAULT).as_posix()}" bot_visible=false\n')
|
||||
mp=VAULT/'.manifest.json'
|
||||
data={}
|
||||
if mp.exists():
|
||||
try: data=json.loads(mp.read_text(encoding='utf-8'))
|
||||
except Exception: data={}
|
||||
data.setdefault('sources',{})[f'filesystem:ppap-apqp-archive-{RUN_DATE}']={
|
||||
'ingested_at': ts,
|
||||
'source_path': str(ROOT),
|
||||
'source_type': 'filesystem_directory',
|
||||
'doc_type': 'ppap_apqp_quality_archive',
|
||||
'project': 'bjd',
|
||||
'files': total,
|
||||
'packages': package_count,
|
||||
'pages_created': [p.relative_to(VAULT).as_posix() for p in pages],
|
||||
'data_paths': [jsonl.relative_to(VAULT).as_posix(), csvp.relative_to(VAULT).as_posix(), package_json.relative_to(VAULT).as_posix()],
|
||||
'bot_visible': False,
|
||||
'extraction_note': 'Old xls/doc/pdf files are indexed by filename/path and limited binary strings; xlsx files use lightweight OOXML text extraction.',
|
||||
}
|
||||
mp.write_text(json.dumps(data, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
META.mkdir(parents=True, exist_ok=True)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
records=collect()
|
||||
packages=package_summary(records)
|
||||
jsonl=META/f'bjd-ppap-apqp-file-inventory-{RUN_DATE}.jsonl'
|
||||
csvp=META/f'bjd-ppap-apqp-file-inventory-{RUN_DATE}.csv'
|
||||
package_json=META/f'bjd-ppap-apqp-package-summary-{RUN_DATE}.json'
|
||||
write_jsonl(jsonl, records)
|
||||
write_csv(csvp, records)
|
||||
package_json.write_text(json.dumps(packages, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
|
||||
pages=write_pages(records, packages, jsonl, csvp, package_json)
|
||||
print(json.dumps({'files': len(records), 'packages': len(packages), 'pages': [str(p) for p in pages], 'jsonl': str(jsonl), 'csv': str(csvp), 'package_json': str(package_json)}, ensure_ascii=False))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+357
@@ -0,0 +1,357 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build a portable Linux migration bundle for an obsidian-wiki installation.
|
||||
#
|
||||
# The bundle contains:
|
||||
# - obsidian-wiki/ this repository, excluding local caches and symlink mirrors
|
||||
# - vault/ the configured vault, including _sources/ and _meta/
|
||||
# - deploy-linux-server.sh
|
||||
# - server.env.example, README-LINUX-MIGRATION.md, bundle-info.env, SHA256SUMS
|
||||
#
|
||||
# It does not bundle the QMD cache. QMD is rebuilt on the server.
|
||||
|
||||
set -Eeuo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEFAULT_REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
REPO_DIR="$DEFAULT_REPO_DIR"
|
||||
VAULT_DIR=""
|
||||
OUT_DIR="$DEFAULT_REPO_DIR/dist"
|
||||
BUNDLE_NAME=""
|
||||
REBUILD_FILE_INDEX=1
|
||||
RUN_BOT_AUDIT=1
|
||||
WITHOUT_SOURCES=0
|
||||
KEEP_STAGING=0
|
||||
DRY_RUN=0
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/package-linux-migration.sh [options]
|
||||
|
||||
Options:
|
||||
--vault PATH Vault path to package. Defaults to OBSIDIAN_VAULT_PATH
|
||||
from repo .env, then ~/.obsidian-wiki/config.
|
||||
--repo PATH obsidian-wiki repo path. Defaults to this script's parent.
|
||||
--out DIR Output directory for the .tar.gz bundle. Default: ./dist
|
||||
--name NAME Bundle directory/archive name. Default: obsidian-wiki-linux-<UTC>
|
||||
--skip-file-index Do not rebuild _meta/file-assets.sqlite before packaging.
|
||||
--skip-bot-audit Do not run bot-view audit before packaging.
|
||||
--without-sources Exclude vault/_sources/archive from the bundle. This is
|
||||
only for small test bundles; citations/file lookup suffer.
|
||||
--keep-staging Keep the temporary expanded bundle directory after tar.
|
||||
--dry-run Print resolved paths and checks, but do not copy or tar.
|
||||
-h, --help Show this help.
|
||||
|
||||
Example:
|
||||
scripts/package-linux-migration.sh \
|
||||
--vault "/Users/nactang/repo/knowlege Base/Knowlege Base" \
|
||||
--out /Users/nactang/Desktop/wiki-migration
|
||||
EOF
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
info() {
|
||||
echo "[INFO] $*"
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo "[WARN] $*" >&2
|
||||
}
|
||||
|
||||
clean_config_value() {
|
||||
local value="$1"
|
||||
value="${value#\"}"
|
||||
value="${value%\"}"
|
||||
value="${value#\'}"
|
||||
value="${value%\'}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
expand_path() {
|
||||
local path="$1"
|
||||
case "$path" in
|
||||
"~") printf '%s' "$HOME" ;;
|
||||
"~/"*) printf '%s/%s' "$HOME" "${path#~/}" ;;
|
||||
*) printf '%s' "$path" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
read_config_key() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
[[ -f "$file" ]] || return 1
|
||||
awk -v key="$key" '
|
||||
index($0, key "=") == 1 {
|
||||
sub("^[^=]*=", "", $0)
|
||||
print $0
|
||||
exit
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
|
||||
resolve_vault_dir() {
|
||||
if [[ -n "$VAULT_DIR" ]]; then
|
||||
clean_config_value "$VAULT_DIR"
|
||||
return
|
||||
fi
|
||||
|
||||
local raw=""
|
||||
raw="$(read_config_key "$REPO_DIR/.env" "OBSIDIAN_VAULT_PATH" || true)"
|
||||
if [[ -z "$raw" ]]; then
|
||||
raw="$(read_config_key "$HOME/.obsidian-wiki/config" "OBSIDIAN_VAULT_PATH" || true)"
|
||||
fi
|
||||
clean_config_value "$raw"
|
||||
}
|
||||
|
||||
require_command() {
|
||||
local cmd="$1"
|
||||
command -v "$cmd" >/dev/null 2>&1 || die "missing command: $cmd"
|
||||
}
|
||||
|
||||
human_size() {
|
||||
du -sh "$1" 2>/dev/null | awk '{print $1}'
|
||||
}
|
||||
|
||||
file_count() {
|
||||
find "$1" -type f 2>/dev/null | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
run_obsidian_wiki() {
|
||||
(cd "$REPO_DIR" && PYTHONPATH="$REPO_DIR${PYTHONPATH:+:$PYTHONPATH}" python3 -m obsidian_wiki "$@")
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--vault)
|
||||
[[ $# -ge 2 ]] || die "--vault requires a path"
|
||||
VAULT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--repo)
|
||||
[[ $# -ge 2 ]] || die "--repo requires a path"
|
||||
REPO_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--out)
|
||||
[[ $# -ge 2 ]] || die "--out requires a directory"
|
||||
OUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--name)
|
||||
[[ $# -ge 2 ]] || die "--name requires a value"
|
||||
BUNDLE_NAME="$2"
|
||||
shift 2
|
||||
;;
|
||||
--skip-file-index)
|
||||
REBUILD_FILE_INDEX=0
|
||||
shift
|
||||
;;
|
||||
--skip-bot-audit)
|
||||
RUN_BOT_AUDIT=0
|
||||
shift
|
||||
;;
|
||||
--without-sources)
|
||||
WITHOUT_SOURCES=1
|
||||
shift
|
||||
;;
|
||||
--keep-staging)
|
||||
KEEP_STAGING=1
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
REPO_DIR="$(expand_path "$REPO_DIR")"
|
||||
REPO_DIR="$(cd "$REPO_DIR" && pwd)"
|
||||
VAULT_DIR="$(resolve_vault_dir)"
|
||||
VAULT_DIR="$(expand_path "$VAULT_DIR")"
|
||||
OUT_DIR="$(expand_path "$OUT_DIR")"
|
||||
|
||||
[[ -d "$REPO_DIR/obsidian_wiki" ]] || die "repo path does not look like obsidian-wiki: $REPO_DIR"
|
||||
[[ -n "$VAULT_DIR" ]] || die "vault path is not set; pass --vault PATH"
|
||||
[[ -d "$VAULT_DIR" ]] || die "vault path does not exist: $VAULT_DIR"
|
||||
[[ -f "$VAULT_DIR/index.md" ]] || warn "vault has no index.md: $VAULT_DIR"
|
||||
[[ -d "$VAULT_DIR/_meta" ]] || warn "vault has no _meta directory: $VAULT_DIR"
|
||||
[[ -d "$VAULT_DIR/_sources" ]] || warn "vault has no _sources directory: $VAULT_DIR"
|
||||
|
||||
require_command python3
|
||||
require_command rsync
|
||||
require_command tar
|
||||
require_command shasum
|
||||
|
||||
if [[ -z "$BUNDLE_NAME" ]]; then
|
||||
BUNDLE_NAME="obsidian-wiki-linux-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
OUT_DIR="$(cd "$OUT_DIR" && pwd)"
|
||||
|
||||
info "repo: $REPO_DIR"
|
||||
info "vault: $VAULT_DIR"
|
||||
info "vault size: $(human_size "$VAULT_DIR")"
|
||||
info "vault files: $(file_count "$VAULT_DIR")"
|
||||
info "out: $OUT_DIR"
|
||||
info "bundle: $BUNDLE_NAME.tar.gz"
|
||||
|
||||
if [[ "$WITHOUT_SOURCES" -eq 1 ]]; then
|
||||
warn "--without-sources is enabled; _sources/archive will not be packaged."
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
info "dry run complete; no files copied."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$REBUILD_FILE_INDEX" -eq 1 ]]; then
|
||||
info "rebuilding file asset index before packaging..."
|
||||
run_obsidian_wiki file-index rebuild --vault "$VAULT_DIR"
|
||||
fi
|
||||
|
||||
if [[ "$RUN_BOT_AUDIT" -eq 1 ]]; then
|
||||
info "auditing bot-view ACLs before packaging..."
|
||||
run_obsidian_wiki bot-view audit --vault "$VAULT_DIR"
|
||||
fi
|
||||
|
||||
WORK_DIR="$OUT_DIR/.staging-$BUNDLE_NAME"
|
||||
STAGE_DIR="$WORK_DIR/$BUNDLE_NAME"
|
||||
rm -rf "$WORK_DIR"
|
||||
mkdir -p "$STAGE_DIR"
|
||||
|
||||
info "copying repo..."
|
||||
rsync -a \
|
||||
--exclude '/.git/' \
|
||||
--exclude '/.venv/' \
|
||||
--exclude '/dist/' \
|
||||
--exclude '/build/' \
|
||||
--exclude '/.mypy_cache/' \
|
||||
--exclude '/.pytest_cache/' \
|
||||
--exclude '/__pycache__/' \
|
||||
--exclude '*.pyc' \
|
||||
--exclude '.DS_Store' \
|
||||
--exclude '/.claude/skills/' \
|
||||
--exclude '/.cursor/skills/' \
|
||||
--exclude '/.windsurf/skills/' \
|
||||
--exclude '/.agents/skills/' \
|
||||
--exclude '/.pi/skills/' \
|
||||
--exclude '/.kiro/skills/' \
|
||||
"$REPO_DIR/" "$STAGE_DIR/obsidian-wiki/"
|
||||
|
||||
info "copying vault..."
|
||||
VAULT_RSYNC_EXCLUDES=(
|
||||
--exclude '.DS_Store'
|
||||
--exclude '.trash/'
|
||||
--exclude '.obsidian/workspace*'
|
||||
)
|
||||
if [[ "$WITHOUT_SOURCES" -eq 1 ]]; then
|
||||
VAULT_RSYNC_EXCLUDES+=(--exclude '_sources/archive/')
|
||||
fi
|
||||
rsync -a "${VAULT_RSYNC_EXCLUDES[@]}" "$VAULT_DIR/" "$STAGE_DIR/vault/"
|
||||
|
||||
cp "$REPO_DIR/scripts/deploy-linux-server.sh" "$STAGE_DIR/deploy-linux-server.sh"
|
||||
chmod +x "$STAGE_DIR/deploy-linux-server.sh"
|
||||
|
||||
cat > "$STAGE_DIR/server.env.example" <<'EOF'
|
||||
# Copy values from here into deploy options if you want non-default paths.
|
||||
INSTALL_DIR=/opt/obsidian-wiki
|
||||
VAULT_DIR=/srv/bjd-vault
|
||||
BOT_VIEW_DIR=/srv/bjd-bot-views
|
||||
QMD_WIKI_COLLECTION=bjd-wiki
|
||||
QMD_CLI_SEARCH_MODE=balanced
|
||||
SOURCE_FILE_PROFILES=quality-bot,engineering-bot
|
||||
EOF
|
||||
|
||||
cat > "$STAGE_DIR/README-LINUX-MIGRATION.md" <<'EOF'
|
||||
# obsidian-wiki Linux migration bundle
|
||||
|
||||
This bundle is meant for a headless Linux server. Obsidian Desktop is not
|
||||
required on the server.
|
||||
|
||||
## Deploy
|
||||
|
||||
Unpack the archive on the server, then run:
|
||||
|
||||
```bash
|
||||
cd obsidian-wiki-linux-*
|
||||
sudo bash deploy-linux-server.sh \
|
||||
--install-dir /opt/obsidian-wiki \
|
||||
--vault-dir /srv/bjd-vault \
|
||||
--bot-view-dir /srv/bjd-bot-views
|
||||
```
|
||||
|
||||
For a user-local deployment without sudo:
|
||||
|
||||
```bash
|
||||
bash deploy-linux-server.sh \
|
||||
--install-dir "$HOME/obsidian-wiki" \
|
||||
--vault-dir "$HOME/bjd-vault" \
|
||||
--bot-view-dir "$HOME/bjd-bot-views"
|
||||
```
|
||||
|
||||
The deploy script copies the repo and vault, writes config, rebuilds the SQLite
|
||||
file index, audits bot ACLs, exports bot views, and rebuilds QMD collections if
|
||||
`qmd` is installed.
|
||||
EOF
|
||||
|
||||
cat > "$STAGE_DIR/bundle-info.env" <<EOF
|
||||
BUNDLE_CREATED_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
SOURCE_REPO="$REPO_DIR"
|
||||
SOURCE_VAULT="$VAULT_DIR"
|
||||
VAULT_SIZE_HUMAN="$(human_size "$VAULT_DIR")"
|
||||
VAULT_FILE_COUNT="$(file_count "$VAULT_DIR")"
|
||||
WITHOUT_SOURCES="$WITHOUT_SOURCES"
|
||||
EOF
|
||||
|
||||
info "writing checksums..."
|
||||
(
|
||||
cd "$STAGE_DIR"
|
||||
: > SHA256SUMS
|
||||
find . -type f ! -name SHA256SUMS -print | LC_ALL=C sort | while IFS= read -r file; do
|
||||
shasum -a 256 "$file"
|
||||
done > SHA256SUMS
|
||||
)
|
||||
|
||||
BUNDLE_TAR="$OUT_DIR/$BUNDLE_NAME.tar.gz"
|
||||
rm -f "$BUNDLE_TAR" "$BUNDLE_TAR.sha256"
|
||||
|
||||
info "creating tarball..."
|
||||
(
|
||||
cd "$WORK_DIR"
|
||||
COPYFILE_DISABLE=1 tar -czf "$BUNDLE_TAR" "$BUNDLE_NAME"
|
||||
)
|
||||
shasum -a 256 "$BUNDLE_TAR" > "$BUNDLE_TAR.sha256"
|
||||
|
||||
if [[ "$KEEP_STAGING" -eq 0 ]]; then
|
||||
rm -rf "$WORK_DIR"
|
||||
else
|
||||
info "kept staging dir: $STAGE_DIR"
|
||||
fi
|
||||
|
||||
info "bundle ready: $BUNDLE_TAR"
|
||||
info "checksum: $BUNDLE_TAR.sha256"
|
||||
cat <<EOF
|
||||
|
||||
Next:
|
||||
scp "$BUNDLE_TAR" user@server:/tmp/
|
||||
ssh user@server
|
||||
cd /tmp && tar -xzf "$(basename "$BUNDLE_TAR")"
|
||||
cd "$BUNDLE_NAME"
|
||||
sudo bash deploy-linux-server.sh --install-dir /opt/obsidian-wiki --vault-dir /srv/bjd-vault --bot-view-dir /srv/bjd-bot-views
|
||||
EOF
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# Source this from your shell rc file to get wiki freshness reminders on terminal open.
|
||||
#
|
||||
# Setup (shell-specific):
|
||||
# zsh/bash: source /path/to/obsidian-wiki/scripts/wiki-notify.sh
|
||||
# fish: bass source /path/to/obsidian-wiki/scripts/wiki-notify.sh
|
||||
# (or copy _wiki_notify logic natively using fish syntax)
|
||||
#
|
||||
# State is vault-scoped under ~/.obsidian-wiki/state/<vault-id>/
|
||||
# Multiple vaults are supported — all stale vaults are shown.
|
||||
|
||||
_wiki_notify() {
|
||||
local state_base="$HOME/.obsidian-wiki/state"
|
||||
[[ -d "$state_base" ]] || return
|
||||
|
||||
local now age_s age_h stale last vault_path shown=0
|
||||
|
||||
now=$(date +%s)
|
||||
|
||||
# Iterate over all vault state dirs
|
||||
for state_dir in "$state_base"/*/; do
|
||||
[[ -f "$state_dir/.last_update" ]] || continue
|
||||
|
||||
last=$(cat "$state_dir/.last_update" 2>/dev/null || echo 0)
|
||||
age_s=$(( now - last ))
|
||||
|
||||
# Only show if >20 hours stale
|
||||
(( age_s > 72000 )) || continue
|
||||
|
||||
age_h=$(( age_s / 3600 ))
|
||||
stale=$(cat "$state_dir/.pending_delta" 2>/dev/null || echo 0)
|
||||
vault_path=$(cat "$state_dir/.vault_path" 2>/dev/null || echo "unknown vault")
|
||||
|
||||
echo "┌─ wiki: last synced ${age_h}h ago · ${vault_path##*/}$([ "$stale" -gt 0 ] && echo " · ${stale} source(s) have new content" || echo "")"
|
||||
echo "│ /wiki-history-ingest claude sync Claude sessions"
|
||||
echo "│ /wiki-status see full delta"
|
||||
echo "└─ /memory-bridge diff compare tool memories"
|
||||
shown=$(( shown + 1 ))
|
||||
done
|
||||
}
|
||||
|
||||
_wiki_notify
|
||||
Reference in New Issue
Block a user