473 lines
15 KiB
Python
473 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, date, datetime
|
|
from typing import Any
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from .config import CrmConfig
|
|
from .filters import (
|
|
build_and_filter,
|
|
build_date_range_condition,
|
|
build_eq_condition,
|
|
build_relation_id_condition,
|
|
build_relation_id_in_condition,
|
|
build_search_filter,
|
|
)
|
|
from .nb_client import NbClient, NbCommandError
|
|
|
|
__all__ = [
|
|
"Any",
|
|
"CUSTOMER_STATUS_LABELS",
|
|
"CrmConfig",
|
|
"FOLLOWUP_CONTENT_KEYS",
|
|
"FOLLOWUP_DATE_KEYS",
|
|
"FOLLOWUP_STATUS_LABELS",
|
|
"FOLLOWUP_TYPE_LABELS",
|
|
"INVALID_FOLLOWUP_STATUS",
|
|
"LabelSpec",
|
|
"MAX_PAGE_SIZE",
|
|
"MAX_SCAN_RECORDS",
|
|
"Mapping",
|
|
"NbClient",
|
|
"NbCommandError",
|
|
"PROJECTION_FALLBACK_WARNING",
|
|
"UNKNOWN_NAME_PREFIX",
|
|
"ZoneInfo",
|
|
"_beijing_datetime",
|
|
"_compact_field_list",
|
|
"_data",
|
|
"_decorate_dimension_counts",
|
|
"_decorate_id_counts",
|
|
"_dimension_value",
|
|
"_fields_with_projections",
|
|
"_first_numeric",
|
|
"_first_text",
|
|
"_first_value",
|
|
"_format_user_candidate",
|
|
"_int_or_default",
|
|
"_normalize_pagination",
|
|
"_page_from_offset",
|
|
"_paginated_rows",
|
|
"_pagination_metadata",
|
|
"_parse_datetime",
|
|
"_pick_user_matches",
|
|
"_prune",
|
|
"_ref_from_mapping",
|
|
"_ref_ids",
|
|
"_ref_names",
|
|
"_relation_ids",
|
|
"_relation_refs",
|
|
"_relation_rows",
|
|
"_total",
|
|
"_unique_strings",
|
|
"_unique_values",
|
|
"build_and_filter",
|
|
"build_date_range_condition",
|
|
"build_eq_condition",
|
|
"build_relation_id_condition",
|
|
"build_relation_id_in_condition",
|
|
"build_search_filter",
|
|
"date",
|
|
"datetime",
|
|
]
|
|
|
|
DEFAULT_PAGE_SIZE = 20
|
|
MAX_PAGE_SIZE = 100
|
|
MAX_SCAN_RECORDS = 5000
|
|
UNKNOWN_NAME_PREFIX = "#"
|
|
PROJECTION_FALLBACK_WARNING = "NocoBase rejected relation field projection; check nameFields in the CRM config"
|
|
|
|
CUSTOMER_STATUS_LABELS = {
|
|
"kh_p_not_start": "未开始",
|
|
"kh_p_inprogress": "进行中",
|
|
"kh_p_ordered": "已下单",
|
|
"kh_p_audited": "已审核",
|
|
}
|
|
|
|
FOLLOWUP_CONTENT_KEYS = ("f_19iplxz0nds", "content", "description", "remark")
|
|
FOLLOWUP_DATE_KEYS = ("f_srysmjkwfq1", "followedAt", "createdAt", "updatedAt")
|
|
FOLLOWUP_TYPE_LABELS = {
|
|
"tdwfk5h1ohv": "邮件",
|
|
"s48pnyhutwc": "电话",
|
|
"mwyy8xwk66m": "视频会议",
|
|
"m34aocz3ebo": "拜访/请客",
|
|
"0lmj78zpdnm": "审厂/来访",
|
|
"o7rjlg4guzh": "论坛/大会",
|
|
"2csix8r3h0f": "代理跟进情况代填",
|
|
}
|
|
FOLLOWUP_STATUS_LABELS = {
|
|
"2rowk1bc8vc": "审批中",
|
|
"3ge6geal2u1": "OK",
|
|
"68vqz8raus6": "迟交",
|
|
"b5pf5njqgp6": "无效",
|
|
}
|
|
INVALID_FOLLOWUP_STATUS = "b5pf5njqgp6"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LabelSpec:
|
|
"""How to identify and name one related record.
|
|
|
|
Every relation the API renders (customer, contact, user, project) resolves to
|
|
the same ``{"id": ..., "name": ...}`` shape through this spec, so the same
|
|
entity looks identical in every tool response.
|
|
"""
|
|
|
|
id_field: str = "id"
|
|
name_fields: tuple[str, ...] = ()
|
|
|
|
def projection_fields(self) -> list[str]:
|
|
return _unique_strings([self.id_field, *self.name_fields])
|
|
|
|
|
|
def _page_from_offset(limit: int, offset: int) -> int:
|
|
bounded_limit = max(1, int(limit))
|
|
bounded_offset = max(0, int(offset))
|
|
return bounded_offset // bounded_limit + 1
|
|
|
|
|
|
def _normalize_pagination(limit: Any, offset: Any) -> tuple[int, int]:
|
|
return min(max(1, _int_or_default(limit, DEFAULT_PAGE_SIZE)), MAX_PAGE_SIZE), max(0, _int_or_default(offset, 0))
|
|
|
|
|
|
def _compact_field_list(configured_fields: list[str] | None, candidates: list[str]) -> list[str]:
|
|
available = set(configured_fields or [])
|
|
selected: list[str] = []
|
|
for candidate in _unique_strings(candidates):
|
|
if available and candidate not in available:
|
|
continue
|
|
selected.append(candidate)
|
|
return selected
|
|
|
|
|
|
def _int_or_default(value: Any, default: int) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _data(response: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|
data = response.get("data")
|
|
if isinstance(data, list):
|
|
return [item for item in data if isinstance(item, dict)]
|
|
if isinstance(data, dict):
|
|
for key in ("data", "rows", "results"):
|
|
nested = data.get(key)
|
|
if isinstance(nested, list):
|
|
return [item for item in nested if isinstance(item, dict)]
|
|
return []
|
|
|
|
|
|
def _total(response: Mapping[str, Any], fallback: int) -> int:
|
|
meta = response.get("meta")
|
|
if isinstance(meta, dict):
|
|
for key in ("count", "total", "totalCount"):
|
|
value = meta.get(key)
|
|
if isinstance(value, int):
|
|
return value
|
|
data = response.get("data")
|
|
if isinstance(data, dict):
|
|
for key in ("count", "total", "totalCount"):
|
|
value = data.get(key)
|
|
if isinstance(value, int):
|
|
return value
|
|
return fallback
|
|
|
|
|
|
def _paginated_response(response: Mapping[str, Any], limit: int, offset: int) -> dict[str, Any]:
|
|
rows = _data(response)
|
|
total = _total(response, len(rows))
|
|
return _paginated_rows(rows, total, limit, offset)
|
|
|
|
|
|
def _paginated_rows(rows: list[dict[str, Any]], total: int, limit: int, offset: int) -> dict[str, Any]:
|
|
next_offset = offset + len(rows)
|
|
return {
|
|
"results": rows,
|
|
"total": total,
|
|
"returned": len(rows),
|
|
"limit": limit,
|
|
"offset": offset,
|
|
"has_more": bool(rows) and next_offset < total,
|
|
"next_offset": next_offset if next_offset < total else None,
|
|
}
|
|
|
|
|
|
def _pagination_metadata(result: Mapping[str, Any]) -> dict[str, Any]:
|
|
return {key: result.get(key) for key in ("total", "returned", "limit", "offset", "has_more", "next_offset")}
|
|
|
|
|
|
def _beijing_datetime(value: Any, timezone: str) -> Any:
|
|
if not isinstance(value, str) or not value.strip():
|
|
return value
|
|
normalized = value.strip()
|
|
if len(normalized) == 10:
|
|
try:
|
|
datetime.strptime(normalized, "%Y-%m-%d")
|
|
except ValueError:
|
|
return value
|
|
return normalized
|
|
candidate = normalized[:-1] + "+00:00" if normalized.endswith("Z") else normalized
|
|
try:
|
|
parsed = datetime.fromisoformat(candidate)
|
|
except ValueError:
|
|
return value
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=UTC)
|
|
try:
|
|
return parsed.astimezone(ZoneInfo(timezone)).isoformat()
|
|
except (ValueError, KeyError):
|
|
return parsed.astimezone(ZoneInfo("Asia/Shanghai")).isoformat()
|
|
|
|
|
|
def _parse_datetime(value: Any, timezone: str) -> datetime | None:
|
|
normalized = _beijing_datetime(value, timezone)
|
|
if not isinstance(normalized, str) or len(normalized) == 10:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(normalized)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _first_numeric(rows: list[dict[str, Any]], preferred_key: str) -> int:
|
|
for row in rows:
|
|
value = row.get(preferred_key)
|
|
if isinstance(value, (int, float)):
|
|
return int(value)
|
|
for nested_value in row.values():
|
|
if isinstance(nested_value, (int, float)):
|
|
return int(nested_value)
|
|
return 0
|
|
|
|
|
|
def _decorate_dimension_counts(
|
|
rows: list[dict[str, Any]],
|
|
*,
|
|
preferred_keys: list[str],
|
|
labels: Mapping[Any, str],
|
|
) -> list[dict[str, Any]]:
|
|
decorated: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
value, key = _dimension_value(row, preferred_keys)
|
|
item = dict(row)
|
|
if value is not None:
|
|
label = labels.get(value) or str(value)
|
|
# The raw dimension key is dropped once normalized -- keeping it meant
|
|
# every bucket carried the same value under an internal field code too.
|
|
if key is not None and key not in {"value", "label"}:
|
|
item.pop(key, None)
|
|
item["value"] = value
|
|
item["label"] = label
|
|
decorated.append(item)
|
|
decorated.sort(key=lambda item: (_is_blank(item.get("value")), -_count_value(item), str(item.get("label") or "")))
|
|
return decorated
|
|
|
|
|
|
def _decorate_id_counts(
|
|
rows: list[dict[str, Any]],
|
|
*,
|
|
preferred_keys: list[str],
|
|
output_key: str,
|
|
) -> list[dict[str, Any]]:
|
|
decorated: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
value, key = _dimension_value(row, preferred_keys)
|
|
item = dict(row)
|
|
if value is not None:
|
|
if key is not None and key != output_key:
|
|
item.pop(key, None)
|
|
item[output_key] = value
|
|
decorated.append(item)
|
|
decorated.sort(key=lambda item: (-_count_value(item), str(item.get(output_key) or "")))
|
|
return decorated
|
|
|
|
|
|
def _dimension_value(row: Mapping[str, Any], preferred_keys: list[str]) -> tuple[Any | None, str | None]:
|
|
for key in preferred_keys:
|
|
if key in row:
|
|
return row[key], key
|
|
for key, value in row.items():
|
|
if key == "count":
|
|
continue
|
|
if not isinstance(value, (dict, list)):
|
|
return value, str(key)
|
|
return None, None
|
|
|
|
|
|
def _count_value(row: Mapping[str, Any]) -> int:
|
|
value = row.get("count")
|
|
if isinstance(value, (int, float)):
|
|
return int(value)
|
|
return 0
|
|
|
|
|
|
def _is_blank(value: Any) -> bool:
|
|
return value is None or str(value).strip() == ""
|
|
|
|
|
|
def _first_value(row: Mapping[str, Any], fields: list[str]) -> Any:
|
|
for field in _unique_strings(fields):
|
|
value = row.get(field)
|
|
if value is not None and value != "":
|
|
return value
|
|
return None
|
|
|
|
|
|
def _first_text(row: Mapping[str, Any], fields: list[str]) -> str | None:
|
|
for field in _unique_strings(fields):
|
|
value = row.get(field)
|
|
if value is None or value == "":
|
|
continue
|
|
if isinstance(value, (dict, list)):
|
|
continue
|
|
return str(value)
|
|
return None
|
|
|
|
|
|
def _prune(mapping: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Drop keys whose value carries no information.
|
|
|
|
An absent key means "empty", never "unknown" -- callers reconciling records
|
|
should not have to skip past a screenful of nulls and empty lists. ``id`` is
|
|
always kept so a record stays addressable even when nothing else resolved.
|
|
"""
|
|
pruned: dict[str, Any] = {}
|
|
for key, value in mapping.items():
|
|
if key == "id" or not _is_empty(value):
|
|
pruned[key] = value
|
|
return pruned
|
|
|
|
|
|
def _is_empty(value: Any) -> bool:
|
|
if value is None:
|
|
return True
|
|
if isinstance(value, str):
|
|
return value.strip() == ""
|
|
if isinstance(value, (list, tuple, dict, set)):
|
|
return len(value) == 0
|
|
return False
|
|
|
|
|
|
def _ref_from_mapping(item: Mapping[str, Any], spec: LabelSpec) -> dict[str, Any]:
|
|
"""Render one related record as ``{"id": ..., "name": ...}``.
|
|
|
|
Deliberately never falls back to the raw id as the *name*: a customer whose
|
|
name fields are all empty used to surface as e.g. ``"4249"``, which reads
|
|
like a real name. It becomes ``"#4249"`` instead -- unmistakably an id, and
|
|
a signal that the source record needs its name filled in.
|
|
"""
|
|
ident = item.get(spec.id_field)
|
|
name = _first_text(item, list(spec.name_fields))
|
|
if name is None and ident is not None:
|
|
name = f"{UNKNOWN_NAME_PREFIX}{ident}"
|
|
return _prune({"id": ident, "name": name})
|
|
|
|
|
|
def _relation_refs(row: Mapping[str, Any], fields: list[str], spec: LabelSpec) -> list[dict[str, Any]]:
|
|
refs: list[dict[str, Any]] = []
|
|
seen: set[Any] = set()
|
|
for field in _unique_strings(fields):
|
|
for item in _relation_rows(row.get(field)):
|
|
ref = _ref_from_mapping(item, spec)
|
|
if not ref:
|
|
continue
|
|
key = ref.get("id", ref.get("name"))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
refs.append(ref)
|
|
return refs
|
|
|
|
|
|
def _ref_names(refs: list[dict[str, Any]]) -> list[str]:
|
|
return [str(ref["name"]) for ref in refs if ref.get("name")]
|
|
|
|
|
|
def _ref_ids(refs: list[dict[str, Any]]) -> list[Any]:
|
|
return [ref["id"] for ref in refs if ref.get("id") is not None]
|
|
|
|
|
|
def _fields_with_projections(scalars: list[str], projections: list[str]) -> list[str]:
|
|
"""Combine scalar field names with dotted relation projections.
|
|
|
|
A relation requested bare (``gjjl_client``) next to a dotted projection
|
|
(``gjjl_client.id``) makes NocoBase expand the whole related row again, which
|
|
is exactly the bloat the projections exist to avoid -- so any scalar that is
|
|
also a projection root is dropped.
|
|
"""
|
|
roots = {path.split(".", 1)[0] for path in projections}
|
|
kept = [name for name in _unique_strings(scalars) if name not in roots]
|
|
return kept + _unique_strings(projections)
|
|
|
|
|
|
def _relation_rows(value: Any) -> list[dict[str, Any]]:
|
|
if isinstance(value, Mapping):
|
|
nested = value.get("data")
|
|
if isinstance(nested, list):
|
|
return [dict(item) for item in nested if isinstance(item, Mapping)]
|
|
return [dict(value)]
|
|
if isinstance(value, list):
|
|
return [dict(item) for item in value if isinstance(item, Mapping)]
|
|
return []
|
|
|
|
|
|
def _relation_ids(value: Any, id_field: str = "id") -> list[Any]:
|
|
return [row.get(id_field) for row in _relation_rows(value) if row.get(id_field) is not None]
|
|
|
|
|
|
def _unique_values(values: list[Any]) -> list[Any]:
|
|
unique: list[Any] = []
|
|
seen: set[Any] = set()
|
|
for value in values:
|
|
if value in seen:
|
|
continue
|
|
seen.add(value)
|
|
unique.append(value)
|
|
return unique
|
|
|
|
|
|
def _unique_strings(values: list[str]) -> list[str]:
|
|
unique: list[str] = []
|
|
for value in values:
|
|
normalized = str(value or "").strip()
|
|
if normalized and normalized not in unique:
|
|
unique.append(normalized)
|
|
return unique
|
|
|
|
|
|
def _pick_user_matches(rows: list[dict[str, Any]], query: str, match_fields: list[str]) -> list[dict[str, Any]]:
|
|
normalized = query.casefold()
|
|
exact = [
|
|
row for row in rows if any(str(row.get(field) or "").strip().casefold() == normalized for field in match_fields)
|
|
]
|
|
if exact:
|
|
return exact
|
|
fuzzy = [
|
|
row for row in rows if any(normalized in str(row.get(field) or "").strip().casefold() for field in match_fields)
|
|
]
|
|
return fuzzy
|
|
|
|
|
|
def _user_label(row: dict[str, Any]) -> str:
|
|
for key in ("nickname", "username", "email", "f_yh_english_name", "id"):
|
|
value = row.get(key)
|
|
if value is not None and value != "":
|
|
return str(value)
|
|
return "<unnamed>"
|
|
|
|
|
|
def _format_user_candidate(row: dict[str, Any], id_field: str) -> dict[str, Any]:
|
|
candidate: dict[str, Any] = {
|
|
"id": row.get(id_field),
|
|
"label": _user_label(row),
|
|
}
|
|
for key in ("username", "nickname", "email", "f_yh_english_name"):
|
|
value = row.get(key)
|
|
if value is not None and value != "":
|
|
candidate[key] = value
|
|
return candidate
|