Skip to content

Commit 18ef7b4

Browse files
KuznetsovRomanKuznetsovRoman
committed
feat(selectivity): implement fs-caching
1 parent 463d373 commit 18ef7b4

13 files changed

Lines changed: 1048 additions & 286 deletions

File tree

‎src/browser/cdp/selectivity/constants.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
export const WEBPACK_PROTOCOL = "webpack://";
2+
export const SELECTIVITY_CACHE_DIRECTIRY = "testplane-selectivity-cache";
3+
export const SELECTIVITY_CACHE_READY_SUFFIX = "-ready";

‎src/browser/cdp/selectivity/css-selectivity.ts‎

Lines changed: 143 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,27 @@
11
import { groupBy } from "lodash";
22
import path from "node:path";
3-
import { resolve as urlResolve, URL } from "node:url";
3+
import { resolve as urlResolve } from "node:url";
44
import { CSS_SOURCE_MAP_URL_COMMENT } from "../../../error-snippets/constants";
5-
import { fetchTextWithBrowserFallback, hasProtocol, patchSourceMapSources } from "./utils";
5+
import {
6+
fetchTextWithBrowserFallback,
7+
hasProtocol,
8+
isCachedOnFs,
9+
isDataProtocol,
10+
patchSourceMapSources,
11+
} from "./utils";
12+
import { CacheType, getCachedSelectivityFile, hasCachedSelectivityFile, setCachedSelectivityFile } from "./fs-cache";
613
import type { CDP } from "..";
7-
import type { CDPRuntimeScriptId, CDPSessionId } from "../types";
14+
import type { CDPStyleSheetId, CDPSessionId } from "../types";
815
import type { CssEvents } from "../domains/css";
16+
import type { SelectivityAssetState } from "./types";
917

1018
export class CSSSelectivity {
1119
private readonly _cdp: CDP;
1220
private readonly _sessionId: CDPSessionId;
1321
private readonly _sourceRoot: string;
1422
private _cssOnStyleSheetAddedFn: ((params: CssEvents["styleSheetAdded"]) => void) | null = null;
15-
private _stylesSourceMap: Record<CDPRuntimeScriptId, null | Promise<string | Error>> = {};
23+
private _stylesSourceMap: Record<CDPStyleSheetId, SelectivityAssetState> = {};
24+
private _styleSheetIdToSourceMapUrl: Record<CDPStyleSheetId, string | null> = {};
1625

1726
constructor(cdp: CDP, sessionId: CDPSessionId, sourceRoot = "") {
1827
this._cdp = cdp;
@@ -26,15 +35,34 @@ export class CSSSelectivity {
2635
}
2736

2837
if (!sourceURL || !sourceMapURL) {
29-
this._stylesSourceMap[styleSheetId] ||= null;
38+
this._stylesSourceMap[styleSheetId] ||= Promise.resolve(null);
3039
return;
3140
}
3241

33-
this._stylesSourceMap[styleSheetId] ||= fetchTextWithBrowserFallback(
34-
urlResolve(sourceURL, sourceMapURL),
35-
this._cdp.runtime,
36-
this._sessionId,
37-
);
42+
const sourceMapResolvedUrl = urlResolve(sourceURL, sourceMapURL);
43+
44+
// Embedded source maps are not cached on file system because of their large cache key
45+
if (isDataProtocol(sourceMapResolvedUrl)) {
46+
this._styleSheetIdToSourceMapUrl[styleSheetId] = null;
47+
this._stylesSourceMap[styleSheetId] ||= fetchTextWithBrowserFallback(
48+
sourceMapResolvedUrl,
49+
this._cdp.runtime,
50+
this._sessionId,
51+
).catch((err: Error) => err);
52+
} else {
53+
this._styleSheetIdToSourceMapUrl[styleSheetId] = sourceMapResolvedUrl;
54+
this._stylesSourceMap[styleSheetId] ||= hasCachedSelectivityFile(
55+
CacheType.Asset,
56+
sourceMapResolvedUrl,
57+
).then(isCached => {
58+
return isCached
59+
? true
60+
: fetchTextWithBrowserFallback(sourceMapResolvedUrl, this._cdp.runtime, this._sessionId)
61+
.then(data => setCachedSelectivityFile(CacheType.Asset, sourceMapResolvedUrl, data))
62+
.then(() => true as const)
63+
.catch((err: Error) => err);
64+
});
65+
}
3866
}
3967

4068
async start(): Promise<void> {
@@ -53,101 +81,119 @@ export class CSSSelectivity {
5381

5482
/** @param drop only performs cleanup without providing actual deps. Should be "true" if test is failed */
5583
async stop(drop?: boolean): Promise<Set<string> | null> {
56-
if (drop) {
57-
this._cssOnStyleSheetAddedFn && this._cdp.css.off("styleSheetAdded", this._cssOnStyleSheetAddedFn);
58-
59-
return null;
60-
}
61-
62-
const coverage = await this._cdp.css.stopRuleUsageTracking(this._sessionId);
63-
64-
// If we haven't got "styleSheetAdded" event for the script, pull up styles + source map manually
65-
coverage.ruleUsage.forEach(({ styleSheetId }) => {
66-
if (Object.hasOwn(this._stylesSourceMap, styleSheetId)) {
67-
return;
84+
try {
85+
if (drop) {
86+
return null;
6887
}
6988

70-
const scriptSourcePromise = this._cdp.css
71-
.getStyleSheetText(this._sessionId, styleSheetId)
72-
.then(res => res.text)
73-
.catch((err: Error) => err);
89+
const coverage = await this._cdp.css.stopRuleUsageTracking(this._sessionId);
7490

75-
this._stylesSourceMap[styleSheetId] ||= scriptSourcePromise.then(sourceCode => {
76-
if (sourceCode instanceof Error) {
77-
return sourceCode;
78-
}
79-
80-
const sourceMapsStartIndex = sourceCode.lastIndexOf(CSS_SOURCE_MAP_URL_COMMENT);
81-
const sourceMapsEndIndex = sourceCode.indexOf("*/", sourceMapsStartIndex);
82-
83-
if (sourceMapsStartIndex === -1) {
84-
return new Error("Source maping url comment is missing");
85-
}
86-
87-
const sourceMapURL =
88-
sourceMapsEndIndex === -1
89-
? sourceCode.slice(sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length)
90-
: sourceCode.slice(
91-
sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length,
92-
sourceMapsEndIndex,
93-
);
94-
95-
const isSourceMapEmbedded = new URL(sourceMapURL).protocol === "data:";
96-
97-
// If we encounter css stylesheet, that was not reported by "styleSheetAdded"
98-
// We can only get sourcemaps if they are inlined
99-
// Otherwise, we can't resolve actual sourcemaps url because we dont know css styles url itself.
100-
if (!isSourceMapEmbedded) {
101-
return new Error(
102-
[
103-
`Missed stylesheet url for stylesheet id ${styleSheetId}.`,
104-
"Switching to inline sourcemaps for CSS will help",
105-
].join("\n"),
106-
);
107-
}
108-
109-
return fetchTextWithBrowserFallback(sourceMapURL, this._cdp.runtime, this._sessionId);
110-
});
111-
});
112-
113-
const totalDependingSourceFiles = new Set<string>();
114-
const grouppedByStyleSheetCoverage = groupBy(coverage.ruleUsage, "styleSheetId");
115-
const styleSheetIds = Object.keys(grouppedByStyleSheetCoverage);
116-
117-
await Promise.all(
118-
styleSheetIds.map(async styleSheetId => {
119-
const sourceMapString = await this._stylesSourceMap[styleSheetId];
120-
121-
if (!sourceMapString) {
91+
// If we haven't got "styleSheetAdded" event for the script, pull up styles + source map manually
92+
coverage.ruleUsage.forEach(({ styleSheetId }) => {
93+
if (Object.hasOwn(this._stylesSourceMap, styleSheetId)) {
12294
return;
12395
}
12496

125-
if (sourceMapString instanceof Error) {
126-
throw new Error(
127-
`CSS Selectivity: Couldn't load source maps for stylesheet id ${styleSheetId}: ${sourceMapString}`,
97+
const scriptSourcePromise = this._cdp.css
98+
.getStyleSheetText(this._sessionId, styleSheetId)
99+
.then(res => res.text)
100+
.catch((err: Error) => err);
101+
102+
this._stylesSourceMap[styleSheetId] ||= scriptSourcePromise.then(sourceCode => {
103+
if (sourceCode instanceof Error) {
104+
return sourceCode;
105+
}
106+
107+
const sourceMapsStartIndex = sourceCode.lastIndexOf(CSS_SOURCE_MAP_URL_COMMENT);
108+
const sourceMapsEndIndex = sourceCode.indexOf("*/", sourceMapsStartIndex);
109+
110+
// Source maps are not generated for this source file
111+
if (sourceMapsStartIndex === -1) {
112+
return null;
113+
}
114+
115+
const sourceMapURL =
116+
sourceMapsEndIndex === -1
117+
? sourceCode.slice(sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length)
118+
: sourceCode.slice(
119+
sourceMapsStartIndex + CSS_SOURCE_MAP_URL_COMMENT.length,
120+
sourceMapsEndIndex,
121+
);
122+
123+
// If we encounter css stylesheet, that was not reported by "styleSheetAdded"
124+
// We can only get sourcemaps if they are inlined
125+
// Otherwise, we can't resolve actual sourcemaps url because we dont know css styles url itself.
126+
if (!isDataProtocol(sourceMapURL)) {
127+
return new Error(
128+
[
129+
`Missed stylesheet url for stylesheet id ${styleSheetId}.`,
130+
"Looks like Chrome Devtools 'styleSheetAdded' event was lost",
131+
"It could happen due to network instability",
132+
"Switching to inline sourcemaps for CSS will help at the cost of increased RAM usage",
133+
].join("\n"),
134+
);
135+
}
136+
137+
return fetchTextWithBrowserFallback(sourceMapURL, this._cdp.runtime, this._sessionId).catch(
138+
(err: Error) => err,
128139
);
129-
}
130-
131-
const rawSourceMap = patchSourceMapSources(JSON.parse(sourceMapString), this._sourceRoot);
132-
133-
// We could check "if stylesheet was used" with utils.extractSourceFilesDeps
134-
// But we dont, because if stylesheet was not used, it could be used after change
135-
// So its safe to think "if stylesheet was loaded, it was used"
136-
rawSourceMap.sources.forEach(sourceFilePath => {
137-
// "Each entry is either a string that is a (potentially relative) URL", so we are using posix.jojn
138-
// https://tc39.es/ecma426/#sec-source-map-format
139-
// Except for file path with protocol ("turbopack://", "file://")
140-
const sourceRootBasedPath = hasProtocol(sourceFilePath)
141-
? sourceFilePath
142-
: path.posix.join(rawSourceMap.sourceRoot || "", sourceFilePath);
143-
144-
totalDependingSourceFiles.add(sourceRootBasedPath);
145140
});
146-
}),
147-
);
148-
149-
this._cssOnStyleSheetAddedFn && this._cdp.css.off("styleSheetAdded", this._cssOnStyleSheetAddedFn);
141+
});
150142

151-
return totalDependingSourceFiles;
143+
const totalDependingSourceFiles = new Set<string>();
144+
const grouppedByStyleSheetCoverage = groupBy(coverage.ruleUsage, "styleSheetId");
145+
const styleSheetIds = Object.keys(grouppedByStyleSheetCoverage);
146+
147+
await Promise.all(
148+
styleSheetIds.map(async styleSheetId => {
149+
const sourceMap = await this._stylesSourceMap[styleSheetId];
150+
const sourceMapUrl = this._styleSheetIdToSourceMapUrl[styleSheetId];
151+
152+
if (!sourceMap) {
153+
return;
154+
}
155+
156+
if (sourceMap instanceof Error) {
157+
throw new Error(
158+
`CSS Selectivity: Couldn't load source maps for stylesheet id ${styleSheetId}:\n${sourceMap}`,
159+
);
160+
}
161+
162+
if (isCachedOnFs(sourceMap) && !sourceMapUrl) {
163+
throw new Error(
164+
"Assertation failed: souce map url has to present if source maps are fs-cached",
165+
);
166+
}
167+
168+
const sourceMapString = isCachedOnFs(sourceMap)
169+
? await getCachedSelectivityFile(CacheType.Asset, sourceMapUrl as string)
170+
: sourceMap;
171+
172+
if (!sourceMapString) {
173+
throw new Error(`CSS Selectivity: fs-cache is broken for ${sourceMapUrl}`);
174+
}
175+
176+
const rawSourceMap = patchSourceMapSources(JSON.parse(sourceMapString), this._sourceRoot);
177+
178+
// We could check "if stylesheet was used" with utils.extractSourceFilesDeps
179+
// But we dont, because if stylesheet was not used, it could be used after change
180+
// So its safe to think "if stylesheet was loaded, it was used"
181+
rawSourceMap.sources.forEach(sourceFilePath => {
182+
// "Each entry is either a string that is a (potentially relative) URL", so we are using posix.jojn
183+
// https://tc39.es/ecma426/#sec-source-map-format
184+
// Except for file path with protocol ("turbopack://", "file://")
185+
const sourceRootBasedPath = hasProtocol(sourceFilePath)
186+
? sourceFilePath
187+
: path.posix.join(rawSourceMap.sourceRoot || "", sourceFilePath);
188+
189+
totalDependingSourceFiles.add(sourceRootBasedPath);
190+
});
191+
}),
192+
);
193+
194+
return totalDependingSourceFiles;
195+
} finally {
196+
this._cssOnStyleSheetAddedFn && this._cdp.css.off("styleSheetAdded", this._cssOnStyleSheetAddedFn);
197+
}
152198
}
153199
}

0 commit comments

Comments
 (0)