From 54800b5cc8835acb4e1e2cd36465cdf933e98c1c Mon Sep 17 00:00:00 2001 From: SandipBajracharya Date: Mon, 13 Apr 2026 12:49:10 +0545 Subject: [PATCH 1/2] =?UTF-8?q?fix(OUT-3584):=20stream=20Dropbox=E2=86=92A?= =?UTF-8?q?ssembly=20uploads=20to=20prevent=20OOM=20crash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sync-dropbox-file-to-assembly task was OOMing on video files because `uploadFileInAssembly` called `filesDownload` solely to read file size — the Dropbox SDK buffers the entire file into a Buffer on that call, then the bytes were discarded. Actual upload already streamed via a separate path, so memory was wasted. - Remove the buffering `filesDownload` call. - `DropboxClient._downloadFile` now returns `{ body, contentLength }`, sourcing Content-Length from the download response headers so it always matches the streamed bytes (avoids S3 PUT mismatch on file types where listing-time size diverges from download size). - Add explicit access-token refresh in `_downloadFile`. The removed `filesDownload` SDK call was implicitly refreshing the token; without it the manual fetch ran with an unpopulated Bearer and Dropbox 400'd. - Guard resync path against non-file entries (folder/deleted) that could reach the upload path after a Dropbox rename/delete-and-recreate. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/features/sync/lib/Sync.service.ts | 17 ++++++-------- .../helper/resync-failed-files.helper.ts | 4 +++- src/lib/dropbox/DropboxClient.ts | 23 +++++++++++++++++-- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/features/sync/lib/Sync.service.ts b/src/features/sync/lib/Sync.service.ts index 4f86f7c..cea216e 100644 --- a/src/features/sync/lib/Sync.service.ts +++ b/src/features/sync/lib/Sync.service.ts @@ -268,23 +268,20 @@ export class SyncService extends AuthenticatedDropboxService { private async uploadFileInAssembly(dbxPath: string, uploadUrl: string, copilotApi: CopilotAPI) { logger.info('SyncService#uploadFileInAssembly :: Uploading file to Assembly', dbxPath) - const dbx = this.dbxClient.getDropboxClient() - const fileMetaData = await dbx.filesDownload({ path: dbxPath }) // get metadata for the files - logger.info('SyncService#uploadFileInAssembly :: File metadata downloaded', dbxPath) - - const downloadBody = await this.dbxClient.downloadFile({ + // Stream the file directly from Dropbox into Assembly's S3 upload URL. + // `contentLength` comes from the download response headers, guaranteeing it + // matches the exact bytes in the stream. Avoids the Dropbox SDK's + // `filesDownload` which buffers the full file in memory (OOMs on videos). + const { body: downloadBody, contentLength } = await this.dbxClient.downloadFile({ urlPath: DBX_URL_PATH.fileDownload, filePath: dbxPath, rootNamespaceId: z.string().parse(this.connectionToken.rootNamespaceId), + refreshToken: this.connectionToken.refreshToken, }) logger.info('SyncService#uploadFileInAssembly :: Found downloadBody', Boolean(downloadBody)) // upload file to assembly - const fileUploadResp = await copilotApi.uploadFile( - uploadUrl, - fileMetaData.result.size.toString(), - downloadBody, - ) + const fileUploadResp = await copilotApi.uploadFile(uploadUrl, contentLength, downloadBody) logger.info('SyncService#uploadFileInAssembly :: File uploaded to Assembly', dbxPath) if (fileUploadResp.status !== httpStatus.OK) { diff --git a/src/features/workers/resync-failed-files/helper/resync-failed-files.helper.ts b/src/features/workers/resync-failed-files/helper/resync-failed-files.helper.ts index 808aa1e..219a747 100644 --- a/src/features/workers/resync-failed-files/helper/resync-failed-files.helper.ts +++ b/src/features/workers/resync-failed-files/helper/resync-failed-files.helper.ts @@ -88,7 +88,9 @@ const processFailedSync = async ( if (file && file.status !== 'pending') return const fileInDropbox = await getFileFromDropbox(dbxClient, failedSync.dbxFileId ?? '') - if (!fileInDropbox) return + // Guard against non-file entries: a mapped dbxFileId could resolve to a folder + // or deleted entry after a Dropbox rename/delete-and-recreate. Only resync files. + if (!fileInDropbox || fileInDropbox['.tag'] !== 'file') return const channelSync = await db.query.channelSync.findFirst({ where: (channelSync, { eq }) => eq(channelSync.id, failedSync.channelSyncId), diff --git a/src/lib/dropbox/DropboxClient.ts b/src/lib/dropbox/DropboxClient.ts index 7a3b0c4..9cccde3 100644 --- a/src/lib/dropbox/DropboxClient.ts +++ b/src/lib/dropbox/DropboxClient.ts @@ -108,11 +108,18 @@ export class DropboxClient { urlPath, filePath, rootNamespaceId, + refreshToken, }: { urlPath: string filePath: string rootNamespaceId: string - }) { + refreshToken: string + }): Promise<{ body: NodeJS.ReadableStream | null; contentLength: string }> { + // Ensure a valid access token before the download. The DropboxAuth instance + // is created per-task with only the refresh token set; without an explicit + // refresh here, `getAccessToken()` returns undefined and Dropbox 400s. + await this.dbxAuthClient.refreshAccessToken(refreshToken) + const headers = { Authorization: `Bearer ${this.dbxAuthClient.authInstance.getAccessToken()}`, 'Dropbox-API-Path-Root': dropboxArgHeader({ @@ -126,7 +133,19 @@ export class DropboxClient { throw new DropboxResponseError(response.status, response.headers, { error_summary: 'DropboxClient#downloadFile. Failed to download file', // following the dropbox response error convention with snake case }) - return response.body + + // Use the Content-Length from the download response as the upload size. + // This is the exact byte count about to be streamed, guaranteed to match + // what the upstream (S3) sees — unlike the size in `filesListFolder` entries, + // which can diverge from the downloaded stream for certain file types. + const contentLength = response.headers.get('content-length') + if (!contentLength) { + throw new Error( + `DropboxClient#downloadFile. Missing Content-Length header for file: ${filePath}`, + ) + } + + return { body: response.body, contentLength } } /** From 52e84fe3b7c96cb56662f7008551b046ddddd4fd Mon Sep 17 00:00:00 2001 From: SandipBajracharya Date: Mon, 13 Apr 2026 12:51:45 +0545 Subject: [PATCH 2/2] fix(OUT-3584): add explicit access token refresh to _uploadFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the _downloadFile hardening from the previous commit. Previously _uploadFile relied on an implicit token refresh from a preceding SDK call (filesGetMetadata, filesMoveV2, filesCreateFolderV2) in the Assembly→Dropbox path. That contract was fragile: any future caller reaching _uploadFile without a prior SDK call would 400 on an unpopulated Bearer — the same regression we just shipped in _downloadFile. Make the refresh explicit so the contract lives with the method, not the call site. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/features/sync/lib/Sync.service.ts | 1 + src/lib/dropbox/DropboxClient.ts | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/features/sync/lib/Sync.service.ts b/src/features/sync/lib/Sync.service.ts index cea216e..b5bbfb2 100644 --- a/src/features/sync/lib/Sync.service.ts +++ b/src/features/sync/lib/Sync.service.ts @@ -427,6 +427,7 @@ export class SyncService extends AuthenticatedDropboxService { filePath: path, body: resp.body, rootNamespaceId: z.string().parse(this.connectionToken.rootNamespaceId), + refreshToken: this.connectionToken.refreshToken, }) logger.info('SyncService#uploadFileInDropbox :: File uploaded to', path) return { diff --git a/src/lib/dropbox/DropboxClient.ts b/src/lib/dropbox/DropboxClient.ts index 9cccde3..7d63942 100644 --- a/src/lib/dropbox/DropboxClient.ts +++ b/src/lib/dropbox/DropboxClient.ts @@ -157,12 +157,20 @@ export class DropboxClient { filePath, body, rootNamespaceId, + refreshToken, }: { urlPath: string filePath: string body: NodeJS.ReadableStream | null rootNamespaceId: string + refreshToken: string }): Promise { + // Explicit token refresh — mirrors `_downloadFile`. Previously relied on + // an implicit refresh from a preceding SDK call (e.g. filesGetMetadata), + // which is a fragile contract — any future caller reaching this method + // without a prior SDK call would 400 on an unpopulated Bearer. + await this.dbxAuthClient.refreshAccessToken(refreshToken) + const args = { path: filePath, autorename: false,