Merge pull request #1209 from gemini-testing/TESTPLANE-903.restrict_selectivity_fs_parallelism

fix(selectivity): reduce master-thread RAM usage

Author: KuznetsovRoman

package-lock.json

@@ -95,6 +95,7 @@
     "looks-same": "10.0.1",
     "micromatch": "4.0.5",
     "mocha": "10.2.0",
+    "p-limit": "3.1.0",
     "pirates": "4.0.7",
     "plugins-loader": "1.3.4",
     "png-validator": "1.1.0",

package.json

@@ -1,5 +1,6 @@
 import os from "node:os";
 import path from "node:path";
+import pLimit from "p-limit";
 import lockfile from "proper-lockfile";
 import fs from "fs-extra";
 import { getMD5 } from "../../../utils/crypto";
@@ -16,13 +17,16 @@ type CacheTypeValue = (typeof CacheType)[keyof typeof CacheType];
 const processStartTime = Number(new Date());
 const tmpDir = path.join(os.tmpdir(), SELECTIVITY_CACHE_DIRECTIRY);
 
+// https://nodejs.org/api/cli.html#uv_threadpool_sizesize
+const libUVLimited = pLimit((process.env.UV_THREADPOOL_SIZE && Number(process.env.UV_THREADPOOL_SIZE)) || 16);
+
 const ensureSelectivityCacheDirectory = async (): Promise<void> => {
-    await fs.ensureDir(tmpDir);
+    await libUVLimited(() => fs.ensureDir(tmpDir));
 };
 
 const wasModifiedAfterProcessStart = async (flagFilePath: string): Promise<boolean> => {
     try {
-        const stats = await fs.stat(flagFilePath);
+        const stats = await libUVLimited(() => fs.stat(flagFilePath));
         return stats.mtimeMs >= processStartTime;
     } catch {
         return false;
@@ -51,7 +55,7 @@ export const getCachedSelectivityFile = async (cacheType: CacheTypeValue, key: s
     const flagFilePath = cacheFilePath + SELECTIVITY_CACHE_READY_SUFFIX;
 
     if (await wasModifiedAfterProcessStart(flagFilePath)) {
-        const cacheContents = await fs.readFile(cacheFilePath, "utf8").catch(() => null);
+        const cacheContents = await libUVLimited(() => fs.readFile(cacheFilePath, "utf8")).catch(() => null);
 
         return cacheContents;
     }
@@ -60,7 +64,7 @@ export const getCachedSelectivityFile = async (cacheType: CacheTypeValue, key: s
     if (await wasModifiedAfterProcessStart(cacheFilePath)) {
         for (let i = 0; i < 10; i++) {
             if (await wasModifiedAfterProcessStart(flagFilePath)) {
-                const cacheContents = await fs.readFile(cacheFilePath, "utf8").catch(() => null);
+                const cacheContents = await libUVLimited(() => fs.readFile(cacheFilePath, "utf8")).catch(() => null);
 
                 return cacheContents;
             }
@@ -113,12 +117,12 @@ export const setCachedSelectivityFile = async (
     }
 
     try {
-        await fs.writeFile(cacheFilePath, utf8Contents, { encoding: "utf8" }).catch(cause => {
+        await libUVLimited(() => fs.writeFile(cacheFilePath, utf8Contents, { encoding: "utf8" })).catch(cause => {
             throw new Error(`Couldn't write cache to "${cacheFilePath}"`, { cause });
         });
 
         // Using "writeFile" to trigger "mtime" update even if file exists
-        await fs.writeFile(flagFilePath, "").catch(cause => {
+        await libUVLimited(() => fs.writeFile(flagFilePath, "")).catch(cause => {
             throw new Error(`Couldn't mark cache as fresh at "${cacheFilePath}"`, { cause });
         });
     } finally {

src/browser/cdp/selectivity/fs-cache.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import crypto from "node:crypto";
 import fs from "node:fs";
+import pLimit from "p-limit";
 
 const calculateFileMd5Hash = (filePath: string): Promise<string> =>
     new Promise((resolve, reject) => {
@@ -17,6 +18,7 @@ const calculateFileMd5Hash = (filePath: string): Promise<string> =>
 export class HashProvider {
     private static readonly _fileHashStore: Map<string, Promise<string>> = new Map();
     private static readonly _patternHashStore: Map<string, Promise<string>> = new Map();
+    private static readonly _limited = pLimit(10);
 
     async calculateForFile(filePath: string): Promise<string> {
         const cachedHash = HashProvider._fileHashStore.get(filePath);
@@ -25,7 +27,7 @@ export class HashProvider {
             return cachedHash;
         }
 
-        const hashPromise = calculateFileMd5Hash(filePath);
+        const hashPromise = HashProvider._limited(() => calculateFileMd5Hash(filePath));
 
         HashProvider._fileHashStore.set(filePath, hashPromise);
 
@@ -55,7 +57,7 @@ export class HashProvider {
             let promiseQue = Promise.resolve();
 
             for (const filePath of filesSorted) {
-                const fileHashPromise = calculateFileMd5Hash(filePath);
+                const fileHashPromise = HashProvider._limited(() => calculateFileMd5Hash(filePath));
                 const cwdRelativePath = path.relative(cwd, filePath);
                 const posixRelativePath =
                     path.sep === path.posix.sep

src/browser/cdp/selectivity/hash-provider.ts

@@ -240,12 +240,12 @@ export class JSSelectivity {
                         );
                     }
 
-                    const sourceString = isCachedOnFs(source)
-                        ? await getCachedSelectivityFile(CacheType.Asset, sourceUrl)
-                        : source;
-                    const sourceMapsString = isCachedOnFs(sourceMaps)
-                        ? await getCachedSelectivityFile(CacheType.Asset, sourceMapUrl as string)
-                        : sourceMaps;
+                    const [sourceString, sourceMapsString] = await Promise.all([
+                        isCachedOnFs(source) ? getCachedSelectivityFile(CacheType.Asset, sourceUrl) : source,
+                        isCachedOnFs(sourceMaps)
+                            ? getCachedSelectivityFile(CacheType.Asset, sourceMapUrl as string)
+                            : sourceMaps,
+                    ]);
 
                     if (!sourceString || !sourceMapsString) {
                         throw new Error(`JS Selectivity: fs-cache is broken for ${sourceUrl}`);
@@ -261,7 +261,7 @@ export class JSSelectivity {
                         });
                     });
 
-                    const dependingSourceFiles = await extractSourceFilesDeps(
+                    const dependingSourceFiles = extractSourceFilesDeps(
                         sourceString,
                         sourceMapsString,
                         Array.from(startOffsetsSet),