diff --git a/src/crm-service.ts b/src/crm-service.ts index 5ec2b0f..387bcb0 100644 --- a/src/crm-service.ts +++ b/src/crm-service.ts @@ -29,7 +29,6 @@ import { refFromRow, refIds, refNames, - relationIds, relationRefs, relationRows, responseTotal, @@ -324,18 +323,21 @@ export class CrmService { projectId: unknown, options: { includeContacts?: boolean; includeFollowups?: boolean; includeUpdates?: boolean } = {}, ): Promise> { + const includeContacts = options.includeContacts ?? true; const row = singleRow( - await this.client.getRecord(this.config.project.resource, projectId, { fields: this.projectFields() }), + await this.client.getRecord(this.config.project.resource, projectId, { + fields: this.projectFields(includeContacts), + }), ); if (!row) return { error: { code: "project_not_found", message: `No project matched id ${JSON.stringify(projectId)}` }, }; const result: Record = { - project: this.compactProject(row, false), + project: this.compactProject(row, false, false), timezone: this.config.timezone, }; - if (options.includeContacts ?? true) { + if (includeContacts) { result.contacts = relationRefs(row, [this.config.project.contactField], this.contactSpec()); } if (options.includeFollowups ?? true) { @@ -710,6 +712,12 @@ export class CrmService { if (!/^\d{4}-\d{2}-\d{2}$/.test(asOf) || Number.isNaN(new Date(`${asOf}T00:00:00Z`).getTime())) { return { error: { code: "invalid_date", message: "as_of must use YYYY-MM-DD" } }; } + const followupFilter = buildAndFilter([ + buildRelationIdCondition(this.config.followup.ownerField, ownerId), + buildRelationIdInCondition(this.config.followup.ownerField, teamMemberIds), + { [this.config.followup.dateField]: { $dateNotAfter: asOf } }, + { [this.config.followup.statusField]: { $ne: INVALID_FOLLOWUP_STATUS } }, + ]); const [customers, followups] = await Promise.all([ this.scanRecords(this.config.customer.resource, { filter: buildAndFilter([ @@ -719,38 +727,19 @@ export class CrmService { fields: this.customerCompactFields(), sort: this.config.customer.sort, }), - this.scanRecords(this.config.followup.resource, { - filter: buildAndFilter([ - buildRelationIdCondition(this.config.followup.ownerField, ownerId), - buildRelationIdInCondition(this.config.followup.ownerField, teamMemberIds), - { [this.config.followup.dateField]: { $dateNotAfter: asOf } }, - { [this.config.followup.statusField]: { $ne: INVALID_FOLLOWUP_STATUS } }, - ]), - fields: [ - this.config.followup.idField, - `${this.config.followup.customerField}.${this.config.customer.idField}`, - this.config.followup.dateField, - this.config.followup.statusField, - ], - sort: this.config.followup.sort, - }), + this.latestFollowupsByCustomer(followupFilter), ]); - const lastFollowup = new Map(); + const lastFollowup = new Map(); for (const followup of followups.rows) { - const date = parseDate(firstText(followup, [this.config.followup.dateField, ...FOLLOWUP_DATE_KEYS])); - if (!date) continue; - for (const customerId of relationIds( - followup[this.config.followup.customerField], - this.config.customer.idField, - )) { - const previous = lastFollowup.get(customerId); - if (!previous || date > previous) lastFollowup.set(customerId, date); - } + const customerId = followup.customer_id; + const date = parseDate(firstText(followup, ["last_followup_at"])); + if (customerId === undefined || customerId === null || !date) continue; + lastFollowup.set(String(customerId), date); } const asOfDate = new Date(`${asOf}T00:00:00Z`); const stale = customers.rows .map((customer) => { - const last = lastFollowup.get(customer[this.config.customer.idField]); + const last = lastFollowup.get(String(customer[this.config.customer.idField])); const days = last ? Math.floor((asOfDate.getTime() - dateOnlyUtc(last, this.config.timezone).getTime()) / 86_400_000) : null; @@ -773,7 +762,21 @@ export class CrmService { ); }); const result = paginatedRows(stale.slice(offset, offset + limit), stale.length, limit, offset); - Object.assign(result, { as_of: asOf, inactive_days: inactiveDays, timezone: this.config.timezone }); + const totalIsExact = !customers.truncation && !followups.truncation; + Object.assign(result, { + as_of: asOf, + inactive_days: inactiveDays, + timezone: this.config.timezone, + total_is_exact: totalIsExact, + }); + if (!totalIsExact) { + const scannedStaleCount = result.total; + delete result.total; + Object.assign(result, { + scanned_stale_count: scannedStaleCount, + pagination_scope: "scanned_records_only", + }); + } const warnings: string[] = []; const truncation: Record = {}; if (customers.truncation) { @@ -781,7 +784,7 @@ export class CrmService { truncation.customers = customers.truncation; } if (followups.truncation) { - warnings.push("follow-up scan was truncated"); + warnings.push("follow-up customer aggregation was truncated"); truncation.followups = followups.truncation; } if (warnings.length > 0) Object.assign(result, { data_quality_warnings: warnings, truncation }); @@ -864,6 +867,60 @@ export class CrmService { } } + private async latestFollowupsByCustomer(filter: Filter): Promise { + const followup = this.config.followup; + const rows: RecordRow[] = []; + let offset = 0; + const query = async (limit: number, queryOffset: number): Promise => + dataRows( + await this.client.queryRecords(followup.resource, { + dimensions: [ + { + field: [followup.customerField, this.config.customer.idField], + alias: "customer_id", + }, + ], + measures: [ + { + field: [followup.dateField], + aggregation: "max", + alias: "last_followup_at", + }, + ], + orders: [ + { + field: [followup.customerField, this.config.customer.idField], + alias: "customer_id", + order: "asc", + }, + ], + filter, + limit, + offset: queryOffset, + timezone: this.config.timezone, + }), + ); + + while (rows.length < MAX_SCAN_RECORDS) { + const batchLimit = Math.min(MAX_PAGE_SIZE, MAX_SCAN_RECORDS - rows.length); + const batch = await query(batchLimit, offset); + rows.push(...batch); + if (batch.length < batchLimit) return { rows }; + offset += batch.length; + } + const overflow = await query(1, offset); + if (overflow.length === 0) return { rows }; + return { + rows, + truncation: { + minimum_total: rows.length + 1, + returned: rows.length, + limit: MAX_SCAN_RECORDS, + next_offset: offset, + }, + }; + } + private decorateStatusCounts(rows: RecordRow[]): RecordRow[] { return decorateDimensionCounts( rows, @@ -1228,7 +1285,7 @@ export class CrmService { } } - private projectFields(): string[] { + private projectFields(includeContacts = true): string[] { const project = this.config.project; return fieldsWithProjections( [ @@ -1247,12 +1304,16 @@ export class CrmService { ], [ ...this.projection(project.customerRelationField, this.customerSpec()), - ...this.projection(project.contactField, this.contactSpec()), + ...(includeContacts ? this.projection(project.contactField, this.contactSpec()) : []), ], ); } - private compactProject(row: RecordRow, includeRaw = false): Record { + private compactProject( + row: RecordRow, + includeRaw = false, + includeContacts = true, + ): Record { const project = this.config.project; const progress = row[project.progressField]; const progressLabels = Object.fromEntries( @@ -1272,7 +1333,7 @@ export class CrmService { part_numbers: row[project.partNumbersField], period_usage: row[project.periodUsageField], details: row[project.detailsField], - contacts: relationRefs(row, [project.contactField], this.contactSpec()), + ...(includeContacts ? { contacts: relationRefs(row, [project.contactField], this.contactSpec()) } : {}), createdAt: toZonedIso(row.createdAt, this.config.timezone), updatedAt: toZonedIso(row.updatedAt, this.config.timezone), }); diff --git a/test/crm-service.test.ts b/test/crm-service.test.ts index dc6ed23..8abb53e 100644 --- a/test/crm-service.test.ts +++ b/test/crm-service.test.ts @@ -143,6 +143,154 @@ test("health never returns the Authorize Key", async () => { assert.doesNotMatch(JSON.stringify(result), /token|secret|authorizeKey/i); }); +test("getProject returns contacts only at the top level when requested", async () => { + const config = structuredClone(DEFAULT_CONFIG); + const requests: Array> = []; + const fake = { + getRecord: async (_resource: string, _projectId: unknown, options: Record) => { + requests.push(options); + return { + data: { + [config.project.idField]: 30, + [config.project.nameField]: "Project", + [config.project.contactField]: [{ [config.contact.idField]: 2, name: "Contact" }], + }, + }; + }, + } as unknown as NocoBaseClient; + const service = new CrmService(config, fake); + + const withoutContacts = await service.getProject(30, { + includeContacts: false, + includeFollowups: false, + includeUpdates: false, + }); + assert.equal("contacts" in withoutContacts, false); + assert.equal("contacts" in (withoutContacts.project as Record), false); + assert.equal( + ((requests[0]?.fields ?? []) as string[]).some( + (field) => field === config.project.contactField || field.startsWith(`${config.project.contactField}.`), + ), + false, + ); + + const withContacts = await service.getProject(30, { + includeContacts: true, + includeFollowups: false, + includeUpdates: false, + }); + assert.deepEqual(withContacts.contacts, [{ id: 2, name: "Contact" }]); + assert.equal("contacts" in (withContacts.project as Record), false); + assert.equal( + ((requests[1]?.fields ?? []) as string[]).some((field) => + field.startsWith(`${config.project.contactField}.`), + ), + true, + ); +}); + +test("listStaleCustomers uses paginated max aggregation and returns an exact total", async () => { + const config = structuredClone(DEFAULT_CONFIG); + const customers = Array.from({ length: 101 }, (_, index) => ({ + [config.customer.idField]: index + 1, + name: `Customer ${index + 1}`, + })); + const latest = customers.map((customer) => ({ + customer_id: customer[config.customer.idField], + last_followup_at: "2026-01-01T00:00:00Z", + })); + const queryRequests: Array> = []; + const fake = { + listRecords: async (resource: string, options: Record) => { + assert.equal(resource, config.customer.resource); + const pageSize = options.pageSize as number; + const start = ((options.page as number) - 1) * pageSize; + return { data: customers.slice(start, start + pageSize), meta: { count: customers.length } }; + }, + queryRecords: async (resource: string, options: Record) => { + assert.equal(resource, config.followup.resource); + queryRequests.push(options); + const offset = options.offset as number; + const limit = options.limit as number; + return { data: latest.slice(offset, offset + limit) }; + }, + } as unknown as NocoBaseClient; + + const result = await new CrmService(config, fake).listStaleCustomers({ + asOf: "2026-08-20", + inactiveDays: 30, + limit: 20, + }); + assert.equal(result.total, 101); + assert.equal(result.total_is_exact, true); + assert.deepEqual( + queryRequests.map((request) => request.offset), + [0, 100], + ); + assert.deepEqual(queryRequests[0]?.dimensions, [ + { + field: [config.followup.customerField, config.customer.idField], + alias: "customer_id", + }, + ]); + assert.deepEqual(queryRequests[0]?.measures, [ + { field: [config.followup.dateField], aggregation: "max", alias: "last_followup_at" }, + ]); + assert.equal(queryRequests[0]?.timezone, config.timezone); +}); + +test("listStaleCustomers never presents a truncated scan count as an exact total", async () => { + const config = structuredClone(DEFAULT_CONFIG); + const fake = { + listRecords: async () => ({ + data: [{ [config.customer.idField]: 1, name: "Customer" }], + meta: { count: 2 }, + }), + queryRecords: async () => ({ data: [] }), + } as unknown as NocoBaseClient; + + const result = await new CrmService(config, fake).listStaleCustomers({ + asOf: "2026-08-20", + inactiveDays: 30, + }); + assert.equal("total" in result, false); + assert.equal(result.total_is_exact, false); + assert.equal(result.scanned_stale_count, 1); + assert.equal(result.pagination_scope, "scanned_records_only"); + assert.deepEqual(result.data_quality_warnings, ["customer scan was truncated"]); +}); + +test("listStaleCustomers marks an overflowing follow-up aggregation as inexact", async () => { + const config = structuredClone(DEFAULT_CONFIG); + const latest = Array.from({ length: 5_001 }, (_, index) => ({ + customer_id: index + 1, + last_followup_at: "2026-01-01T00:00:00Z", + })); + const fake = { + listRecords: async () => ({ + data: [{ [config.customer.idField]: 1, name: "Customer" }], + meta: { count: 1 }, + }), + queryRecords: async (_resource: string, options: Record) => { + const offset = options.offset as number; + const limit = options.limit as number; + return { data: latest.slice(offset, offset + limit) }; + }, + } as unknown as NocoBaseClient; + + const result = await new CrmService(config, fake).listStaleCustomers({ + asOf: "2026-08-20", + inactiveDays: 30, + }); + assert.equal("total" in result, false); + assert.equal(result.total_is_exact, false); + assert.equal(result.scanned_stale_count, 1); + assert.deepEqual(result.data_quality_warnings, ["follow-up customer aggregation was truncated"]); + assert.deepEqual(result.truncation, { + followups: { minimum_total: 5_001, returned: 5_000, limit: 5_000, next_offset: 5_000 }, + }); +}); + test("all CRM service surfaces execute against the direct HTTP adapter contract", async () => { const config = structuredClone(DEFAULT_CONFIG); const rows: Record> = {