diff --git a/packages/cli/src/util/AssetUploader.ts b/packages/cli/src/util/AssetUploader.ts index 58e77b04..e656a98a 100644 --- a/packages/cli/src/util/AssetUploader.ts +++ b/packages/cli/src/util/AssetUploader.ts @@ -39,6 +39,7 @@ import { dirExists } from './files.js'; import { retryAsync } from './retryAsync.js'; const DEFAULT_PARALLEL_UPLOADS = 7; +const MAX_PARALLEL_ASSET_READS = 16; let PARALLEL_UPLOADS = parseInt(process.env['DEVVIT_PARALLEL_UPLOADS'] || '0'); if (isNaN(PARALLEL_UPLOADS) || PARALLEL_UPLOADS < 1) { @@ -608,8 +609,9 @@ export async function queryAssets( const assets = (await tinyglob(assetsGlob, { filesOnly: true, absolute: true })).filter( (asset) => allowedExtensions.length === 0 || allowedExtensions.includes(path.extname(asset)) ); - return await Promise.all( - assets.map(async (asset) => { + return await mapAsyncWithMaxConcurrency( + assets, + async (asset) => { const filename = path.relative(dir, asset).replaceAll(path.sep, '/'); let file = await fsp.readFile(asset); @@ -630,7 +632,8 @@ export async function queryAssets( isWebviewAsset: assetKind === 'Client', contents, }; - }) + }, + MAX_PARALLEL_ASSET_READS ); } diff --git a/packages/cli/src/util/getAppSourceZip.ts b/packages/cli/src/util/getAppSourceZip.ts index 2bd14df1..ba886c8d 100644 --- a/packages/cli/src/util/getAppSourceZip.ts +++ b/packages/cli/src/util/getAppSourceZip.ts @@ -1,12 +1,14 @@ import fsp from 'node:fs/promises'; import path from 'node:path'; +import { mapAsyncWithMaxConcurrency } from '@devvit/shared-types/mapAsyncWithMaxConcurrency.js'; import ignore from 'ignore'; import JSZip from 'jszip'; import { DevvitCommand } from './commands/DevvitCommand.js'; const ALWAYS_IGNORED_PATHS = Object.freeze(['node_modules', '.env', '.git']); +const MAX_PARALLEL_SOURCE_READS = 16; export async function getAppSourceZip(cmd: DevvitCommand): Promise { const zip = new JSZip(); @@ -51,7 +53,7 @@ async function addDirectoryToZip( const entries = await fsp.readdir(dir, { withFileTypes: true }); // Process entries in parallel - const filePromises: Promise[] = []; + const fileCallbacks: (() => Promise)[] = []; const dirCallbacks: (() => Promise)[] = []; for (const entry of entries) { @@ -71,18 +73,20 @@ async function addDirectoryToZip( ); } } else if (entry.isFile()) { - // Read files & add them in parallel though - filePromises.push( - (async () => { - const content = await fsp.readFile(fullPath); - zipFolder.file(entry.name, content); - })() - ); + // Defer file reads so concurrency can be bounded below. + fileCallbacks.push(async () => { + const content = await fsp.readFile(fullPath); + zipFolder.file(entry.name, content); + }); } } - // Wait for all file reads to complete - await Promise.all(filePromises); + // Read files concurrently without opening every file at once. + await mapAsyncWithMaxConcurrency( + fileCallbacks, + (readFile) => readFile(), + MAX_PARALLEL_SOURCE_READS + ); // Then, process the directories in serial for (const dirCallback of dirCallbacks) {