Skip to content

Commit 18cd769

Browse files
KuznetsovRomanKuznetsovRoman
authored
Merge pull request #1204 from gemini-testing/TESTPLANE-883.selectivity_fs_caching
feat(selectivity): implement fs-caching
2 parents 03a5228 + 665e4df commit 18cd769

16 files changed

Lines changed: 1138 additions & 290 deletions

File tree

‎src/browser/cdp/selectivity/constants.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
export const WEBPACK_PROTOCOL = "webpack://";
2+
export const SELECTIVITY_CACHE_DIRECTIRY = "testplane-selectivity-cache";
3+
export const SELECTIVITY_CACHE_READY_SUFFIX = "-ready";

‎src/browser/cdp/selectivity/css-selectivity.ts‎

Lines changed: 154 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,28 @@
11
import { groupBy } from "lodash";
22
import path from "node:path";
3-
import { resolve as urlResolve, URL } from "node:url";
3+
import { resolve as urlResolve } from "node:url";
44
import { CSS_SOURCE_MAP_URL_COMMENT } from "../../../error-snippets/constants";
5-
import { fetchTextWithBrowserFallback, hasProtocol, patchSourceMapSources } from "./utils";
5+
import {
6+
fetchTextWithBrowserFallback,
7+
hasProtocol,
8+
isCachedOnFs,
9+
isDataProtocol,
10+
patchSourceMapSources,
11+
} from "./utils";
12+
import { CacheType, getCachedSelectivityFile, hasCachedSelectivityFile, setCachedSelectivityFile } from "./fs-cache";
13+
import { debugSelectivity } from "./debug";
614
import type { CDP } from "..";
7-
import type { CDPRuntimeScriptId, CDPSessionId } from "../types";
15+
import type { CDPStyleSheetId, CDPSessionId } from "../types";
816
import type { CssEvents } from "../domains/css";
17+
import type { SelectivityAssetState } from "./types";
918

1019
export class CSSSelectivity {
1120
private readonly _cdp: CDP;
1221
private readonly _sessionId: CDPSessionId;
1322
private readonly _sourceRoot: string;
1423
private _cssOnStyleSheetAddedFn: ((params: CssEvents["styleSheetAdded"]) => void) | null = null;
15-
private _stylesSourceMap: Record<CDPRuntimeScriptId, null | Promise<string | Error>> = {};
24+
private _stylesSourceMap: Record<CDPStyleSheetId, SelectivityAssetState> = {};
25+
private _styleSheetIdToSourceMapUrl: Record<CDPStyleSheetId, string | null> = {};
1626

1727
constructor(cdp: CDP, sessionId: CDPSessionId, sourceRoot = "") {
1828
this._cdp = cdp;
@@ -26,15 +36,43 @@ export class CSSSelectivity {
2636
}
2737

2838
if (!sourceURL || !sourceMapURL) {
29-
this._stylesSourceMap[styleSheetId] ||= null;
39+
this._stylesSourceMap[styleSheetId] ||= Promise.resolve(null);
3040
return;
3141
}
3242

33-
this._stylesSourceMap[styleSheetId] ||= fetchTextWithBrowserFallback(
34-
urlResolve(sourceURL, sourceMapURL),
35-
this._cdp.runtime,
36-
this._sessionId,
37-
);
43+
const sourceMapResolvedUrl = urlResolve(sourceURL, sourceMapURL);
44+
45+
// Embedded source maps are not cached on file system because of their large cache key
46+
if (isDataProtocol(sourceMapResolvedUrl)) {
47+
this._styleSheetIdToSourceMapUrl[styleSheetId] = null;
48+
this._stylesSourceMap[styleSheetId] ||= fetchTextWithBrowserFallback(
49+
sourceMapResolvedUrl,
50+
this._cdp.runtime,
51+
this._sessionId,
52+
).catch((err: Error) => err);
53+
} else {
54+
this._styleSheetIdToSourceMapUrl[styleSheetId] = sourceMapResolvedUrl;
55+
this._stylesSourceMap[styleSheetId] ||= hasCachedSelectivityFile(
56+
CacheType.Asset,
57+
sourceMapResolvedUrl,
58+
).then(isCached => {
59+
return isCached
60+
? true
61+
: fetchTextWithBrowserFallback(sourceMapResolvedUrl, this._cdp.runtime, this._sessionId)
62+
.then(data =>
63+
setCachedSelectivityFile(CacheType.Asset, sourceMapResolvedUrl, data)
64+
.then(() => true as const)
65+
.catch(err => {
66+
debugSelectivity(
67+
`Couldn't offload asset from "${sourceMapResolvedUrl}" to fs-cache: %O`,
68+
err,
69+
);
70+
return data;
71+
}),
72+
)
73+
.catch((err: Error) => err);
74+
});
75+
}
3876
}
3977

4078
async start(): Promise<void> {
@@ -53,101 +91,120 @@ export class CSSSelectivity {
5391

5492
/** @param drop only performs cleanup without providing actual deps. Should be "true" if test is failed */
5593
async stop(drop?: boolean): Promise<Set<string> | null> {
56-
if (drop) {
57-
this._cssOnStyleSheetAddedFn && this._cdp.css.off("styleSheetAdded", this._cssOnStyleSheetAddedFn);
58-
59-
return null;
60-
}
61-
62-
const coverage = await this._cdp.css.stopRuleUsageTracking(this._sessionId);
63-
64-
// If we haven't got "styleSheetAdded" event for the script, pull up styles + source map manually
65-
coverage.ruleUsage.forEach(({ styleSheetId }) => {
66-
if (Object.hasOwn(this._stylesSourceMap, styleSheetId)) {
67-
return;
94+
try {
95+
if (drop) {
96+
return null;
6897
}
6998

70-
const scriptSourcePromise = this._cdp.css
71-
.getStyleSheetText(this._sessionId, styleSheetId)
72-
.then(res => res.text)
73-
.catch((err: Error) => err);
99+
const coverage = await this._cdp.css.stopRuleUsageTracking(this._sessionId);
74100

75-
this._stylesSourceMap[styleSheetId] ||= scriptSourcePromise.then(sourceCode => {
76-
if (sourceCode instanceof Error) {
77-
return sourceCode;
78-
}
79-
80-
const sourceMapsStartIndex = sourceCode.lastIndexOf(CSS_SOURCE_MAP_URL_COMMENT);
81-
const sourceMapsEndIndex = sourceCode.indexOf("*/", sourceMapsStartIndex);
82-
83-
if (sourceMapsStartIndex === -1) {
84-
return new Error("Source maping url comment is missing");
85-
}
86-
87-
const sourceMapURL =
88-
sourceMapsEndIndex === -1
89-
? sourceCode.slice(sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length)
90-
: sourceCode.slice(
91-
sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length,
92-
sourceMapsEndIndex,
93-
);
94-
95-
const isSourceMapEmbedded = new URL(sourceMapURL).protocol === "data:";
96-
97-
// If we encounter css stylesheet, that was not reported by "styleSheetAdded"
98-
// We can only get sourcemaps if they are inlined
99-
// Otherwise, we can't resolve actual sourcemaps url because we dont know css styles url itself.
100-
if (!isSourceMapEmbedded) {
101-
return new Error(
102-
[
103-
`Missed stylesheet url for stylesheet id ${styleSheetId}.`,
104-
"Switching to inline sourcemaps for CSS will help",
105-
].join("\n"),
106-
);
107-
}
108-
109-
return fetchTextWithBrowserFallback(sourceMapURL, this._cdp.runtime, this._sessionId);
110-
});
111-
});
112-
113-
const totalDependingSourceFiles = new Set<string>();
114-
const grouppedByStyleSheetCoverage = groupBy(coverage.ruleUsage, "styleSheetId");
115-
const styleSheetIds = Object.keys(grouppedByStyleSheetCoverage);
116-
117-
await Promise.all(
118-
styleSheetIds.map(async styleSheetId => {
119-
const sourceMapString = await this._stylesSourceMap[styleSheetId];
120-
121-
if (!sourceMapString) {
101+
// If we haven't got "styleSheetAdded" event for the script, pull up styles + source map manually
102+
coverage.ruleUsage.forEach(({ styleSheetId }) => {
103+
if (Object.hasOwn(this._stylesSourceMap, styleSheetId)) {
122104
return;
123105
}
124106

125-
if (sourceMapString instanceof Error) {
126-
throw new Error(
127-
`CSS Selectivity: Couldn't load source maps for stylesheet id ${styleSheetId}: ${sourceMapString}`,
107+
const scriptSourcePromise = this._cdp.css
108+
.getStyleSheetText(this._sessionId, styleSheetId)
109+
.then(res => res.text)
110+
.catch((err: Error) => err);
111+
112+
this._stylesSourceMap[styleSheetId] ||= scriptSourcePromise.then(sourceCode => {
113+
if (sourceCode instanceof Error) {
114+
return sourceCode;
115+
}
116+
117+
const sourceMapsStartIndex = sourceCode.lastIndexOf(CSS_SOURCE_MAP_URL_COMMENT);
118+
const sourceMapsEndIndex = sourceCode.indexOf("*/", sourceMapsStartIndex);
119+
120+
// Source maps are not generated for this source file
121+
if (sourceMapsStartIndex === -1) {
122+
return null;
123+
}
124+
125+
const sourceMapURL =
126+
sourceMapsEndIndex === -1
127+
? sourceCode.slice(sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length)
128+
: sourceCode.slice(
129+
sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length,
130+
sourceMapsEndIndex,
131+
);
132+
133+
// If we encounter css stylesheet, that was not reported by "styleSheetAdded"
134+
// We can only get sourcemaps if they are inlined
135+
// Otherwise, we can't resolve actual sourcemaps url because we dont know css styles url itself.
136+
if (!isDataProtocol(sourceMapURL)) {
137+
return new Error(
138+
[
139+
`Missed stylesheet url for stylesheet id ${styleSheetId}.`,
140+
"Looks like Chrome Devtools 'styleSheetAdded' event was lost",
141+
"It could happen due to network instability",
142+
"Switching to inline sourcemaps for CSS will help at the cost of increased RAM usage",
143+
].join("\n"),
144+
);
145+
}
146+
147+
return fetchTextWithBrowserFallback(sourceMapURL, this._cdp.runtime, this._sessionId).catch(
148+
(err: Error) => err,
128149
);
129-
}
130-
131-
const rawSourceMap = patchSourceMapSources(JSON.parse(sourceMapString), this._sourceRoot);
132-
133-
// We could check "if stylesheet was used" with utils.extractSourceFilesDeps
134-
// But we dont, because if stylesheet was not used, it could be used after change
135-
// So its safe to think "if stylesheet was loaded, it was used"
136-
rawSourceMap.sources.forEach(sourceFilePath => {
137-
// "Each entry is either a string that is a (potentially relative) URL", so we are using posix.jojn
138-
// https://tc39.es/ecma426/#sec-source-map-format
139-
// Except for file path with protocol ("turbopack://", "file://")
140-
const sourceRootBasedPath = hasProtocol(sourceFilePath)
141-
? sourceFilePath
142-
: path.posix.join(rawSourceMap.sourceRoot || "", sourceFilePath);
143-
144-
totalDependingSourceFiles.add(sourceRootBasedPath);
145150
});
146-
}),
147-
);
148-
149-
this._cssOnStyleSheetAddedFn && this._cdp.css.off("styleSheetAdded", this._cssOnStyleSheetAddedFn);
151+
});
150152

151-
return totalDependingSourceFiles;
153+
const totalDependingSourceFiles = new Set<string>();
154+
const grouppedByStyleSheetCoverage = groupBy(coverage.ruleUsage, "styleSheetId");
155+
const styleSheetIds = Object.keys(grouppedByStyleSheetCoverage);
156+
157+
await Promise.all(
158+
styleSheetIds.map(async styleSheetId => {
159+
const sourceMap = await this._stylesSourceMap[styleSheetId];
160+
const sourceMapUrl = this._styleSheetIdToSourceMapUrl[styleSheetId];
161+
162+
if (!sourceMap) {
163+
return;
164+
}
165+
166+
if (sourceMap instanceof Error) {
167+
throw new Error(
168+
`CSS Selectivity: Couldn't load source maps for stylesheet id ${styleSheetId}`,
169+
{ cause: sourceMap },
170+
);
171+
}
172+
173+
if (isCachedOnFs(sourceMap) && !sourceMapUrl) {
174+
throw new Error(
175+
"Assertation failed: souce map url has to present if source maps are fs-cached",
176+
);
177+
}
178+
179+
const sourceMapString = isCachedOnFs(sourceMap)
180+
? await getCachedSelectivityFile(CacheType.Asset, sourceMapUrl as string)
181+
: sourceMap;
182+
183+
if (!sourceMapString) {
184+
throw new Error(`CSS Selectivity: fs-cache is broken for ${sourceMapUrl}`);
185+
}
186+
187+
const rawSourceMap = patchSourceMapSources(JSON.parse(sourceMapString), this._sourceRoot);
188+
189+
// We could check "if stylesheet was used" with utils.extractSourceFilesDeps
190+
// But we dont, because if stylesheet was not used, it could be used after change
191+
// So its safe to think "if stylesheet was loaded, it was used"
192+
rawSourceMap.sources.forEach(sourceFilePath => {
193+
// "Each entry is either a string that is a (potentially relative) URL", so we are using posix.jojn
194+
// https://tc39.es/ecma426/#sec-source-map-format
195+
// Except for file path with protocol ("turbopack://", "file://")
196+
const sourceRootBasedPath = hasProtocol(sourceFilePath)
197+
? sourceFilePath
198+
: path.posix.join(rawSourceMap.sourceRoot || "", sourceFilePath);
199+
200+
totalDependingSourceFiles.add(sourceRootBasedPath);
201+
});
202+
}),
203+
);
204+
205+
return totalDependingSourceFiles;
206+
} finally {
207+
this._cssOnStyleSheetAddedFn && this._cdp.css.off("styleSheetAdded", this._cssOnStyleSheetAddedFn);
208+
}
152209
}
153210
}

0 commit comments

Comments
 (0)