From d100fd9f57d19137c602d7c0024028c20fdc884e Mon Sep 17 00:00:00 2001 From: SandipBajracharya Date: Tue, 31 Mar 2026 18:23:42 +0545 Subject: [PATCH 1/2] perf(OUT-3183): batch Copilot API calls and parallelize page data fetching Replace N+1 per-mapping API calls (retrieveFileChannel, getCompany, getClient) with 3 bulk fetches (listFileChannels, getCompanies, getClients), reducing ~600 API calls to 3 for portals with ~200 channel syncs. Parallelize getSelectorClientsCompanies, getWorkspace, and listFormattedChannelMap via Promise.all on page load. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/app/(home)/page.tsx | 30 ++-- src/features/sync/lib/MapFiles.service.ts | 174 ++++++++++++++-------- 2 files changed, 125 insertions(+), 79 deletions(-) diff --git a/src/app/(home)/page.tsx b/src/app/(home)/page.tsx index c03483a..648443b 100644 --- a/src/app/(home)/page.tsx +++ b/src/app/(home)/page.tsx @@ -29,22 +29,24 @@ const Home = async ({ searchParams }: PageProps) => { const connection = await dpxConnectionService.getConnectionForWorkspace() const userService = new UserService(user) - const users = await userService.getSelectorClientsCompanies() - const workspace = await getWorkspace(token) - let mapList: MapList[] = [], - tempMapList: MapList[] = [] - if (connection.refreshToken && connection.accountId) { - const connectionToken = { - refreshToken: connection.refreshToken, - accountId: connection.accountId, - rootNamespaceId: connection.rootNamespaceId, - } + // Fetch user data, workspace, and channel maps in parallel + const mapListPromise = + connection.refreshToken && connection.accountId + ? new MapFilesService(user, { + refreshToken: connection.refreshToken, + accountId: connection.accountId, + rootNamespaceId: connection.rootNamespaceId, + }).listFormattedChannelMap() + : Promise.resolve([] as MapList[]) - const mapService = new MapFilesService(user, connectionToken) - mapList = await mapService.listFormattedChannelMap() - tempMapList = structuredClone(mapList) - } + const [users, workspace, mapList] = await Promise.all([ + userService.getSelectorClientsCompanies(), + getWorkspace(token), + mapListPromise, + ]) + + const tempMapList = structuredClone(mapList) return ( diff --git a/src/features/sync/lib/MapFiles.service.ts b/src/features/sync/lib/MapFiles.service.ts index a43dcb8..7c983b5 100644 --- a/src/features/sync/lib/MapFiles.service.ts +++ b/src/features/sync/lib/MapFiles.service.ts @@ -1,7 +1,7 @@ import { and, asc, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' import httpStatus from 'http-status' -import { ApiError } from 'node_modules/copilot-node-sdk/dist/codegen/api' import z from 'zod' +import { MAX_FETCH_COPILOT_RESOURCES } from '@/constants/limits' import db from '@/db' import { ObjectType } from '@/db/constants' import { @@ -22,7 +22,6 @@ import type { MapList, WhereClause, } from '@/features/sync/types' -import { copilotBottleneck } from '@/lib/copilot/bottleneck' import { type CopilotFileList, FileChannelMembership, @@ -421,80 +420,125 @@ export class MapFilesService extends AuthenticatedDropboxService { async listFormattedChannelMap(): Promise { const channelMaps = await this.getAllChannelMaps() + if (channelMaps.length === 0) return [] + + // Batch fetch all file channels in one API call + const fileChannels = await this.copilot.listFileChannels({ + limit: MAX_FETCH_COPILOT_RESOURCES, + }) + const fileChannelMap = new Map(fileChannels.map((fc) => [fc.id, fc])) + + // Collect unique company and client IDs from file channels that match our channel maps + const companyIds = new Set() + const clientIds = new Set() + const staleChannelMapIds: string[] = [] - const channelMapPromises = [] for (const channelMap of channelMaps) { - channelMapPromises.push( - copilotBottleneck.schedule(() => { - return this.formatChannelMap(channelMap) + const fc = fileChannelMap.get(channelMap.assemblyChannelId) + if (!fc) { + staleChannelMapIds.push(channelMap.id) + continue + } + if (fc.companyId) companyIds.add(fc.companyId) + if (fc.clientId) clientIds.add(fc.clientId) + } + + // Batch fetch all companies and clients in parallel (2 API calls total) + const [companiesResponse, clientsResponse] = await Promise.all([ + companyIds.size > 0 + ? this.copilot.getCompanies({ limit: MAX_FETCH_COPILOT_RESOURCES }) + : Promise.resolve({ data: [] }), + clientIds.size > 0 + ? this.copilot.getClients({ limit: MAX_FETCH_COPILOT_RESOURCES }) + : Promise.resolve({ data: [] }), + ]) + + const companyMap = new Map((companiesResponse.data ?? []).map((c) => [c.id, c])) + const clientMap = new Map((clientsResponse.data ?? []).map((c) => [c.id, c])) + + // Soft-delete stale channel maps in parallel + if (staleChannelMapIds.length > 0) { + await Promise.all( + staleChannelMapIds.map((id) => { + console.info('Soft delete channel map and make it inactive', id) + return this.deleteChannelMapById(id) }), ) } - return (await Promise.all(channelMapPromises)).filter((channelMap) => !!channelMap) - } - - async formatChannelMap(channelMap: ChannelSyncSelectType): Promise { - logger.info('MapFilesService#formatChannelMap :: Formatting channel map', channelMap) + // Format all channel maps locally using pre-fetched data + const results: MapList[] = [] + for (const channelMap of channelMaps) { + const formatted = this.formatChannelMapFromCache( + channelMap, + fileChannelMap, + companyMap, + clientMap, + ) + if (formatted) results.push(formatted) + } - try { - let fileChannelValue: UserCompanySelectorInputValue[] - const fileChannel = await this.copilot.retrieveFileChannel(channelMap.assemblyChannelId) + return results + } - if (fileChannel.membershipType === FileChannelMembership.COMPANY) { - if (!fileChannel.companyId) { - console.error('Company id not found') - return null - } - const companyDetails = await this.copilot.getCompany(fileChannel.companyId) - fileChannelValue = [ - { - id: companyDetails.id, - companyId: companyDetails.id, - object: 'company' as const, - }, - ] - } else { - if (!fileChannel.clientId) { - console.error('Client id not found') - return null - } - const clientDetails = await this.copilot.getClient(fileChannel.clientId) - fileChannelValue = [ - { - id: clientDetails.id, - companyId: z.string().parse(fileChannel.companyId), - object: 'client' as const, - }, - ] + private formatChannelMapFromCache( + channelMap: ChannelSyncSelectType, + fileChannelMap: Map< + string, + { id: string; membershipType: string; companyId?: string; clientId?: string } + >, + companyMap: Map, + clientMap: Map, + ): MapList | null { + const fileChannel = fileChannelMap.get(channelMap.assemblyChannelId) + if (!fileChannel) return null + + let fileChannelValue: UserCompanySelectorInputValue[] + + if (fileChannel.membershipType === FileChannelMembership.COMPANY) { + if (!fileChannel.companyId) { + console.error('Company id not found') + return null } - - // calculate synced percentage. - const syncedPercentage = this.getSyncedPercentage( - channelMap.status, - channelMap.syncedFilesCount, - channelMap.totalFilesCount, - ) - - const formattedChannelInfo = { - id: channelMap.id, - fileChannelValue, - dbxRootPath: channelMap.dbxRootPath, - status: channelMap.status, - fileChannelId: fileChannel.id, - lastSyncedAt: channelMap.lastSyncedAt, - syncedPercentage, + const company = companyMap.get(fileChannel.companyId) + if (!company) { + console.error('Company not found in batch response', fileChannel.companyId) + return null } - logger.info('MapFilesService#formatChannelMap :: Formatted channel map', formattedChannelInfo) - - return formattedChannelInfo - } catch (error: unknown) { - if (error instanceof ApiError && error.status === httpStatus.BAD_REQUEST) { - console.info('Soft delete channel map and make it inactive') - await this.deleteChannelMapById(channelMap.id) + fileChannelValue = [{ id: company.id, companyId: company.id, object: 'company' as const }] + } else { + if (!fileChannel.clientId) { + console.error('Client id not found') + return null } - logger.error('MapFilesService#formatChannelMap :: Error formatting channel map', error) - return null + const client = clientMap.get(fileChannel.clientId) + if (!client) { + console.error('Client not found in batch response', fileChannel.clientId) + return null + } + fileChannelValue = [ + { + id: client.id, + companyId: z.string().parse(fileChannel.companyId), + object: 'client' as const, + }, + ] + } + + const syncedPercentage = this.getSyncedPercentage( + channelMap.status, + channelMap.syncedFilesCount, + channelMap.totalFilesCount, + ) + + return { + id: channelMap.id, + fileChannelValue, + dbxRootPath: channelMap.dbxRootPath, + status: channelMap.status, + fileChannelId: fileChannel.id, + lastSyncedAt: channelMap.lastSyncedAt, + syncedPercentage, } } From 6cc416d40e0eb1b5d230e7d2081a27976b78bbc5 Mon Sep 17 00:00:00 2001 From: SandipBajracharya Date: Wed, 1 Apr 2026 15:56:15 +0545 Subject: [PATCH 2/2] refactor(OUT-3183): bulk soft delete channels --- src/features/sync/lib/MapFiles.service.ts | 24 +++++++++++------------ src/features/sync/lib/Sync.service.ts | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/features/sync/lib/MapFiles.service.ts b/src/features/sync/lib/MapFiles.service.ts index 7c983b5..7f32f68 100644 --- a/src/features/sync/lib/MapFiles.service.ts +++ b/src/features/sync/lib/MapFiles.service.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' +import { and, asc, eq, inArray, isNotNull, isNull, or, sql } from 'drizzle-orm' import httpStatus from 'http-status' import z from 'zod' import { MAX_FETCH_COPILOT_RESOURCES } from '@/constants/limits' @@ -277,8 +277,10 @@ export class MapFilesService extends AuthenticatedDropboxService { .where(eq(channelSync.id, id)) } - async deleteChannelMapById(id: string) { - logger.info('MapFilesService#deleteChannelMapById :: Deleting channel map', id) + async deleteChannelMapsByIds(ids: string[]) { + if (ids.length === 0) return + + logger.info('MapFilesService#deleteChannelMapsByIds :: Deleting channel maps', ids) await db.transaction(async (tx) => { const deletedAt = new Date() @@ -288,17 +290,17 @@ export class MapFilesService extends AuthenticatedDropboxService { deletedAt, status: false, }) - .where(eq(channelSync.id, id)) + .where(inArray(channelSync.id, ids)) await tx .update(fileFolderSync) .set({ deletedAt, }) - .where(eq(fileFolderSync.channelSyncId, id)) + .where(inArray(fileFolderSync.channelSyncId, ids)) }) - logger.info('MapFilesService#deleteChannelMapById :: Deleted channel map', id) + logger.info('MapFilesService#deleteChannelMapsByIds :: Deleted channel maps', ids) } async getAllChannelMaps(where?: WhereClause): Promise { @@ -456,14 +458,10 @@ export class MapFilesService extends AuthenticatedDropboxService { const companyMap = new Map((companiesResponse.data ?? []).map((c) => [c.id, c])) const clientMap = new Map((clientsResponse.data ?? []).map((c) => [c.id, c])) - // Soft-delete stale channel maps in parallel + // Bulk soft-delete stale channel maps if (staleChannelMapIds.length > 0) { - await Promise.all( - staleChannelMapIds.map((id) => { - console.info('Soft delete channel map and make it inactive', id) - return this.deleteChannelMapById(id) - }), - ) + console.info('Soft delete channel maps and make them inactive: ', staleChannelMapIds) + await this.deleteChannelMapsByIds(staleChannelMapIds) } // Format all channel maps locally using pre-fetched data diff --git a/src/features/sync/lib/Sync.service.ts b/src/features/sync/lib/Sync.service.ts index d7c4888..4f86f7c 100644 --- a/src/features/sync/lib/Sync.service.ts +++ b/src/features/sync/lib/Sync.service.ts @@ -444,6 +444,6 @@ export class SyncService extends AuthenticatedDropboxService { } async removeChannelSyncMapping(channelSyncId: string) { - await this.mapFilesService.deleteChannelMapById(channelSyncId) + await this.mapFilesService.deleteChannelMapsByIds([channelSyncId]) } }