gitea-admin/nocobase-mcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: make CRM detail and stale counts reliable

Browse Source
This commit is contained in:
nac
2026-08-20 06:24:04 +00:00
parent 7cc521938f
commit 6b5b3a413a
2 changed files with 245 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/crm-service.ts
+97 -36
View File
@@ -29,7 +29,6 @@ import {
refFromRow,
refIds,
refNames,
relationIds,
relationRefs,
relationRows,
responseTotal,
@@ -324,18 +323,21 @@ export class CrmService {
projectId: unknown,
options: { includeContacts?: boolean; includeFollowups?: boolean; includeUpdates?: boolean } = {},
): Promise<Record<string, unknown>> {
const includeContacts = options.includeContacts ?? true;
const row = singleRow(
await this.client.getRecord(this.config.project.resource, projectId, { fields: this.projectFields() }),
await this.client.getRecord(this.config.project.resource, projectId, {
fields: this.projectFields(includeContacts),
}),
);
if (!row)
return {
error: { code: "project_not_found", message: `No project matched id ${JSON.stringify(projectId)}` },
};
const result: Record<string, unknown> = {
project: this.compactProject(row, false),
project: this.compactProject(row, false, false),
timezone: this.config.timezone,
};
if (options.includeContacts ?? true) {
if (includeContacts) {
result.contacts = relationRefs(row, [this.config.project.contactField], this.contactSpec());
}
if (options.includeFollowups ?? true) {
@@ -710,6 +712,12 @@ export class CrmService {
if (!/^\d{4}-\d{2}-\d{2}$/.test(asOf) || Number.isNaN(new Date(`${asOf}T00:00:00Z`).getTime())) {
return { error: { code: "invalid_date", message: "as_of must use YYYY-MM-DD" } };
}
const followupFilter = buildAndFilter([
buildRelationIdCondition(this.config.followup.ownerField, ownerId),
buildRelationIdInCondition(this.config.followup.ownerField, teamMemberIds),
{ [this.config.followup.dateField]: { $dateNotAfter: asOf } },
{ [this.config.followup.statusField]: { $ne: INVALID_FOLLOWUP_STATUS } },
]);
const [customers, followups] = await Promise.all([
this.scanRecords(this.config.customer.resource, {
filter: buildAndFilter([
@@ -719,38 +727,19 @@ export class CrmService {
fields: this.customerCompactFields(),
sort: this.config.customer.sort,
}),
this.scanRecords(this.config.followup.resource, {
filter: buildAndFilter([
buildRelationIdCondition(this.config.followup.ownerField, ownerId),
buildRelationIdInCondition(this.config.followup.ownerField, teamMemberIds),
{ [this.config.followup.dateField]: { $dateNotAfter: asOf } },
{ [this.config.followup.statusField]: { $ne: INVALID_FOLLOWUP_STATUS } },
]),
fields: [
this.config.followup.idField,
`${this.config.followup.customerField}.${this.config.customer.idField}`,
this.config.followup.dateField,
this.config.followup.statusField,
],
sort: this.config.followup.sort,
}),
this.latestFollowupsByCustomer(followupFilter),
]);
const lastFollowup = new Map<unknown, Date>();
const lastFollowup = new Map<string, Date>();
for (const followup of followups.rows) {
const date = parseDate(firstText(followup, [this.config.followup.dateField, ...FOLLOWUP_DATE_KEYS]));
if (!date) continue;
for (const customerId of relationIds(
followup[this.config.followup.customerField],
this.config.customer.idField,
)) {
const previous = lastFollowup.get(customerId);
if (!previous || date > previous) lastFollowup.set(customerId, date);
}
const customerId = followup.customer_id;
const date = parseDate(firstText(followup, ["last_followup_at"]));
if (customerId === undefined || customerId === null || !date) continue;
lastFollowup.set(String(customerId), date);
}
const asOfDate = new Date(`${asOf}T00:00:00Z`);
const stale = customers.rows
.map((customer) => {
const last = lastFollowup.get(customer[this.config.customer.idField]);
const last = lastFollowup.get(String(customer[this.config.customer.idField]));
const days = last
? Math.floor((asOfDate.getTime() - dateOnlyUtc(last, this.config.timezone).getTime()) / 86_400_000)
: null;
@@ -773,7 +762,21 @@ export class CrmService {
);
});
const result = paginatedRows(stale.slice(offset, offset + limit), stale.length, limit, offset);
Object.assign(result, { as_of: asOf, inactive_days: inactiveDays, timezone: this.config.timezone });
const totalIsExact = !customers.truncation && !followups.truncation;
Object.assign(result, {
as_of: asOf,
inactive_days: inactiveDays,
timezone: this.config.timezone,
total_is_exact: totalIsExact,
});
if (!totalIsExact) {
const scannedStaleCount = result.total;
delete result.total;
Object.assign(result, {
scanned_stale_count: scannedStaleCount,
pagination_scope: "scanned_records_only",
});
}
const warnings: string[] = [];
const truncation: Record<string, unknown> = {};
if (customers.truncation) {
@@ -781,7 +784,7 @@ export class CrmService {
truncation.customers = customers.truncation;
}
if (followups.truncation) {
warnings.push("follow-up scan was truncated");
warnings.push("follow-up customer aggregation was truncated");
truncation.followups = followups.truncation;
}
if (warnings.length > 0) Object.assign(result, { data_quality_warnings: warnings, truncation });
@@ -864,6 +867,60 @@ export class CrmService {
}
}
private async latestFollowupsByCustomer(filter: Filter): Promise<ScanResult> {
const followup = this.config.followup;
const rows: RecordRow[] = [];
let offset = 0;
const query = async (limit: number, queryOffset: number): Promise<RecordRow[]> =>
dataRows(
await this.client.queryRecords(followup.resource, {
dimensions: [
{
field: [followup.customerField, this.config.customer.idField],
alias: "customer_id",
},
],
measures: [
{
field: [followup.dateField],
aggregation: "max",
alias: "last_followup_at",
},
],
orders: [
{
field: [followup.customerField, this.config.customer.idField],
alias: "customer_id",
order: "asc",
},
],
filter,
limit,
offset: queryOffset,
timezone: this.config.timezone,
}),
);
while (rows.length < MAX_SCAN_RECORDS) {
const batchLimit = Math.min(MAX_PAGE_SIZE, MAX_SCAN_RECORDS - rows.length);
const batch = await query(batchLimit, offset);
rows.push(...batch);
if (batch.length < batchLimit) return { rows };
offset += batch.length;
}
const overflow = await query(1, offset);
if (overflow.length === 0) return { rows };
return {
rows,
truncation: {
minimum_total: rows.length + 1,
returned: rows.length,
limit: MAX_SCAN_RECORDS,
next_offset: offset,
},
};
}
private decorateStatusCounts(rows: RecordRow[]): RecordRow[] {
return decorateDimensionCounts(
rows,
@@ -1228,7 +1285,7 @@ export class CrmService {
}
}
private projectFields(): string[] {
private projectFields(includeContacts = true): string[] {
const project = this.config.project;
return fieldsWithProjections(
[
@@ -1247,12 +1304,16 @@ export class CrmService {
],
[
...this.projection(project.customerRelationField, this.customerSpec()),
...this.projection(project.contactField, this.contactSpec()),
...(includeContacts ? this.projection(project.contactField, this.contactSpec()) : []),
],
);
}
private compactProject(row: RecordRow, includeRaw = false): Record<string, unknown> {
private compactProject(
row: RecordRow,
includeRaw = false,
includeContacts = true,
): Record<string, unknown> {
const project = this.config.project;
const progress = row[project.progressField];
const progressLabels = Object.fromEntries(
@@ -1272,7 +1333,7 @@ export class CrmService {
part_numbers: row[project.partNumbersField],
period_usage: row[project.periodUsageField],
details: row[project.detailsField],
contacts: relationRefs(row, [project.contactField], this.contactSpec()),
...(includeContacts ? { contacts: relationRefs(row, [project.contactField], this.contactSpec()) } : {}),
createdAt: toZonedIso(row.createdAt, this.config.timezone),
updatedAt: toZonedIso(row.updatedAt, this.config.timezone),
});
test/crm-service.test.ts
+148
View File
@@ -143,6 +143,154 @@ test("health never returns the Authorize Key", async () => {
assert.doesNotMatch(JSON.stringify(result), /token|secret|authorizeKey/i);
});
test("getProject returns contacts only at the top level when requested", async () => {
const config = structuredClone(DEFAULT_CONFIG);
const requests: Array<Record<string, unknown>> = [];
const fake = {
getRecord: async (_resource: string, _projectId: unknown, options: Record<string, unknown>) => {
requests.push(options);
return {
data: {
[config.project.idField]: 30,
[config.project.nameField]: "Project",
[config.project.contactField]: [{ [config.contact.idField]: 2, name: "Contact" }],
},
};
},
} as unknown as NocoBaseClient;
const service = new CrmService(config, fake);
const withoutContacts = await service.getProject(30, {
includeContacts: false,
includeFollowups: false,
includeUpdates: false,
});
assert.equal("contacts" in withoutContacts, false);
assert.equal("contacts" in (withoutContacts.project as Record<string, unknown>), false);
assert.equal(
((requests[0]?.fields ?? []) as string[]).some(
(field) => field === config.project.contactField || field.startsWith(`${config.project.contactField}.`),
),
false,
);
const withContacts = await service.getProject(30, {
includeContacts: true,
includeFollowups: false,
includeUpdates: false,
});
assert.deepEqual(withContacts.contacts, [{ id: 2, name: "Contact" }]);
assert.equal("contacts" in (withContacts.project as Record<string, unknown>), false);
assert.equal(
((requests[1]?.fields ?? []) as string[]).some((field) =>
field.startsWith(`${config.project.contactField}.`),
),
true,
);
});
test("listStaleCustomers uses paginated max aggregation and returns an exact total", async () => {
const config = structuredClone(DEFAULT_CONFIG);
const customers = Array.from({ length: 101 }, (_, index) => ({
[config.customer.idField]: index + 1,
name: `Customer ${index + 1}`,
}));
const latest = customers.map((customer) => ({
customer_id: customer[config.customer.idField],
last_followup_at: "2026-01-01T00:00:00Z",
}));
const queryRequests: Array<Record<string, unknown>> = [];
const fake = {
listRecords: async (resource: string, options: Record<string, unknown>) => {
assert.equal(resource, config.customer.resource);
const pageSize = options.pageSize as number;
const start = ((options.page as number) - 1) * pageSize;
return { data: customers.slice(start, start + pageSize), meta: { count: customers.length } };
},
queryRecords: async (resource: string, options: Record<string, unknown>) => {
assert.equal(resource, config.followup.resource);
queryRequests.push(options);
const offset = options.offset as number;
const limit = options.limit as number;
return { data: latest.slice(offset, offset + limit) };
},
} as unknown as NocoBaseClient;
const result = await new CrmService(config, fake).listStaleCustomers({
asOf: "2026-08-20",
inactiveDays: 30,
limit: 20,
});
assert.equal(result.total, 101);
assert.equal(result.total_is_exact, true);
assert.deepEqual(
queryRequests.map((request) => request.offset),
[0, 100],
);
assert.deepEqual(queryRequests[0]?.dimensions, [
{
field: [config.followup.customerField, config.customer.idField],
alias: "customer_id",
},
]);
assert.deepEqual(queryRequests[0]?.measures, [
{ field: [config.followup.dateField], aggregation: "max", alias: "last_followup_at" },
]);
assert.equal(queryRequests[0]?.timezone, config.timezone);
});
test("listStaleCustomers never presents a truncated scan count as an exact total", async () => {
const config = structuredClone(DEFAULT_CONFIG);
const fake = {
listRecords: async () => ({
data: [{ [config.customer.idField]: 1, name: "Customer" }],
meta: { count: 2 },
}),
queryRecords: async () => ({ data: [] }),
} as unknown as NocoBaseClient;
const result = await new CrmService(config, fake).listStaleCustomers({
asOf: "2026-08-20",
inactiveDays: 30,
});
assert.equal("total" in result, false);
assert.equal(result.total_is_exact, false);
assert.equal(result.scanned_stale_count, 1);
assert.equal(result.pagination_scope, "scanned_records_only");
assert.deepEqual(result.data_quality_warnings, ["customer scan was truncated"]);
});
test("listStaleCustomers marks an overflowing follow-up aggregation as inexact", async () => {
const config = structuredClone(DEFAULT_CONFIG);
const latest = Array.from({ length: 5_001 }, (_, index) => ({
customer_id: index + 1,
last_followup_at: "2026-01-01T00:00:00Z",
}));
const fake = {
listRecords: async () => ({
data: [{ [config.customer.idField]: 1, name: "Customer" }],
meta: { count: 1 },
}),
queryRecords: async (_resource: string, options: Record<string, unknown>) => {
const offset = options.offset as number;
const limit = options.limit as number;
return { data: latest.slice(offset, offset + limit) };
},
} as unknown as NocoBaseClient;
const result = await new CrmService(config, fake).listStaleCustomers({
asOf: "2026-08-20",
inactiveDays: 30,
});
assert.equal("total" in result, false);
assert.equal(result.total_is_exact, false);
assert.equal(result.scanned_stale_count, 1);
assert.deepEqual(result.data_quality_warnings, ["follow-up customer aggregation was truncated"]);
assert.deepEqual(result.truncation, {
followups: { minimum_total: 5_001, returned: 5_000, limit: 5_000, next_offset: 5_000 },
});
});
test("all CRM service surfaces execute against the direct HTTP adapter contract", async () => {
const config = structuredClone(DEFAULT_CONFIG);
const rows: Record<string, Record<string, unknown>> = {