Skip to content

Commit 651ba1a

Browse files
mstangemstange
committed
Add merge-android-profiles script
1 parent 450e3f6 commit 651ba1a

4 files changed

Lines changed: 351 additions & 1 deletion

File tree

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@
1919
"build-l10n-prod": "cross-env NODE_ENV=production L10N=1 node scripts/build.mjs && yarn build-sw && yarn build-photon",
2020
"build-photon": "cross-env NODE_ENV=production node scripts/build-photon.mjs",
2121
"build-sw": "workbox generateSW workbox-config.js",
22-
"build-symbolicator-cli": "cross-env NODE_ENV=production node scripts/build-symbolicator.mjs",
2322
"build-prod:quiet": "yarn build-prod",
23+
"build-symbolicator-cli": "cross-env NODE_ENV=production node scripts/build-symbolicator.mjs",
2424
"build-symbolicator-cli:quiet": "yarn build-symbolicator-cli",
25+
"build-merge-android-profiles": "cross-env NODE_ENV=production node scripts/build-merge-android-profiles.mjs",
2526
"lint": "node bin/output-fixing-commands.js run-p lint-js lint-css prettier-run",
2627
"lint-fix": "run-p lint-fix-js lint-fix-css prettier-fix",
2728
"lint-js": "node bin/output-fixing-commands.js eslint . --report-unused-disable-directives --cache --cache-strategy content",

‎scripts/build-merge-android-profiles.mjs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/* This Source Code Form is subject to the terms of the Mozilla Public
2+
* License, v. 2.0. If a copy of the MPL was not distributed with this
3+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4+
import esbuild from 'esbuild';
5+
import { nodeBaseConfig } from './lib/esbuild-configs.mjs';
6+
7+
const symbolicatorConfig = {
8+
...nodeBaseConfig,
9+
metafile: true,
10+
entryPoints: ['src/merge-android-profiles/index.ts'],
11+
outfile: 'dist/merge-android-profiles.js',
12+
};
13+
14+
async function build() {
15+
await esbuild.build(symbolicatorConfig);
16+
console.log('✅ merge-android-profiles build completed');
17+
}
18+
19+
build().catch(console.error);

‎src/merge-android-profiles/index.ts‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
/**
2+
* Merge two existing profiles, taking the samples from the first profile and
3+
* the markers from the second profile.
4+
*
5+
* This was useful during early 2025 when the Mozilla Performance team was
6+
* doing a lot of Android startup profiling:
7+
*
8+
* - The "samples" profile would be collected using simpleperf and converted
9+
* with samply import.
10+
* - The "markers" profile would be collected using the Gecko profiler.
11+
*
12+
* To use this script, it first needs to be built:
13+
* yarn build-merge-android-profiles
14+
*
15+
* Then it can be run from the `dist` directory:
16+
* node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
17+
*
18+
* For example:
19+
* yarn build-merge-android-profiles && node dist/merge-android-profiles.js --samples-hash warg8azfac0z5b5sy92h4a69bfrj2fqsjc6ty58 --markers-hash mb6220c2rx3mmhegv82d84tsvgn6a5p8r7g4je8 --output-file ~/Downloads/merged-profile.json
20+
*
21+
*/
22+
23+
import fs from 'fs';
24+
import minimist from 'minimist';
25+
26+
import {
27+
unserializeProfileOfArbitraryFormat,
28+
adjustMarkerTimestamps,
29+
} from '../profile-logic/process-profile';
30+
import { GOOGLE_STORAGE_BUCKET } from 'firefox-profiler/app-logic/constants';
31+
import { computeStringIndexMarkerFieldsByDataType } from '../profile-logic/marker-schema';
32+
import { ensureExists } from '../utils/types';
33+
import { StringTable } from '../utils/string-table';
34+
35+
import type { Profile, RawThread, Tid } from '../types/profile';
36+
import { compress } from 'firefox-profiler/utils/gz';
37+
38+
type ProfileSource =
39+
| {
40+
type: 'HASH';
41+
hash: string;
42+
}
43+
| {
44+
type: 'FILE';
45+
file: string;
46+
};
47+
48+
interface CliOptions {
49+
samplesProf: ProfileSource;
50+
markersProf: ProfileSource;
51+
filterByProcessPrefix: string | undefined;
52+
assumeSamplesProfileHasStartTimeZero: boolean;
53+
outputFile: string;
54+
}
55+
56+
export function getProfileUrlForHash(hash: string): string {
57+
// See https://cloud.google.com/storage/docs/access-public-data
58+
// The URL is https://storage.googleapis.com/<BUCKET>/<FILEPATH>.
59+
// https://<BUCKET>.storage.googleapis.com/<FILEPATH> seems to also work but
60+
// is not documented nowadays.
61+
62+
// By convention, "profile-store" is the name of our bucket, and the file path
63+
// is the hash we receive in the URL.
64+
return `https://storage.googleapis.com/${GOOGLE_STORAGE_BUCKET}/${hash}`;
65+
}
66+
67+
async function fetchProfileWithHash(hash: string): Promise<Profile> {
68+
const response = await fetch(getProfileUrlForHash(hash));
69+
const serializedProfile = await response.json();
70+
return unserializeProfileOfArbitraryFormat(serializedProfile);
71+
}
72+
73+
async function loadProfileFromFile(path: string): Promise<Profile> {
74+
const uint8Array = fs.readFileSync(path, null);
75+
return unserializeProfileOfArbitraryFormat(uint8Array.buffer);
76+
}
77+
78+
async function loadProfile(source: ProfileSource): Promise<Profile> {
79+
switch (source.type) {
80+
case 'HASH':
81+
return fetchProfileWithHash(source.hash);
82+
case 'FILE':
83+
return loadProfileFromFile(source.file);
84+
default:
85+
return source;
86+
}
87+
}
88+
89+
export async function run(options: CliOptions) {
90+
const profileWithSamples: Profile = await loadProfile(options.samplesProf);
91+
const profileWithMarkers: Profile = await loadProfile(options.markersProf);
92+
93+
// const referenceSampleTime = 169912951.547432; // filteredThread.samples.time[0] after zooming in on samples in mozilla::dom::indexedDB::BackgroundTransactionChild::RecvComplete
94+
// const referenceMarkerTime = 664.370158 ; // selectedMarker.start after selecting the marker for the "complete" DOMEvent
95+
96+
// console.log(profileWithSamples.meta);
97+
// console.log(profileWithMarkers.meta);
98+
99+
if (
100+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot ===
101+
undefined &&
102+
options.assumeSamplesProfileHasStartTimeZero
103+
) {
104+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot = 0;
105+
}
106+
107+
let timeDelta =
108+
profileWithMarkers.meta.startTime - profileWithSamples.meta.startTime;
109+
if (
110+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
111+
undefined &&
112+
profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot !==
113+
undefined
114+
) {
115+
timeDelta =
116+
(profileWithMarkers.meta.startTimeAsClockMonotonicNanosecondsSinceBoot -
117+
profileWithSamples.meta.startTimeAsClockMonotonicNanosecondsSinceBoot) /
118+
1000000;
119+
}
120+
121+
// console.log({ timeDelta });
122+
123+
const profile = profileWithSamples;
124+
profile.meta.markerSchema = profileWithMarkers.meta.markerSchema;
125+
profile.pages = profileWithMarkers.pages;
126+
127+
const markerProfileCategoryToCategory = new Map();
128+
const markerProfileCategories = ensureExists(
129+
profileWithMarkers.meta.categories
130+
);
131+
const profileCategories = ensureExists(profile.meta.categories);
132+
for (
133+
let markerCategoryIndex = 0;
134+
markerCategoryIndex < markerProfileCategories.length;
135+
markerCategoryIndex++
136+
) {
137+
const category = markerProfileCategories[markerCategoryIndex];
138+
let categoryIndex = profileCategories.findIndex(
139+
(c) => c.name === category.name
140+
);
141+
if (categoryIndex === -1) {
142+
categoryIndex = profileCategories.length;
143+
profileCategories[categoryIndex] = {
144+
name: category.name,
145+
color: category.color,
146+
subcategories: ['Other'],
147+
};
148+
}
149+
markerProfileCategoryToCategory.set(markerCategoryIndex, categoryIndex);
150+
}
151+
152+
const markerThreadsByTid = new Map<Tid, RawThread>(
153+
profileWithMarkers.threads.map((thread) => ['' + thread.tid, thread])
154+
);
155+
// console.log([...markerThreadsByTid.keys()]);
156+
157+
// console.log(profile.threads.map((thread) => thread.tid));
158+
159+
const stringIndexMarkerFieldsByDataType =
160+
computeStringIndexMarkerFieldsByDataType(profile.meta.markerSchema);
161+
162+
const sampleThreadTidsWithoutCorrespondingMarkerThreads = new Set();
163+
164+
const stringTable = StringTable.withBackingArray(profile.shared.stringArray);
165+
const markerStringArray = profileWithMarkers.shared.stringArray;
166+
const keptThreads = [];
167+
for (const thread of profile.threads) {
168+
if (options.filterByProcessPrefix !== undefined) {
169+
if (!thread.processName!.startsWith(options.filterByProcessPrefix)) {
170+
continue;
171+
}
172+
}
173+
keptThreads.push(thread);
174+
const tid = thread.tid;
175+
const markerThread = markerThreadsByTid.get(tid);
176+
if (markerThread === undefined) {
177+
sampleThreadTidsWithoutCorrespondingMarkerThreads.add(tid);
178+
continue;
179+
}
180+
markerThreadsByTid.delete(tid);
181+
182+
thread.markers = adjustMarkerTimestamps(markerThread.markers, timeDelta);
183+
for (let i = 0; i < thread.markers.length; i++) {
184+
thread.markers.category[i] = ensureExists(
185+
markerProfileCategoryToCategory.get(thread.markers.category[i])
186+
);
187+
thread.markers.name[i] = stringTable.indexForString(
188+
markerStringArray[thread.markers.name[i]]
189+
);
190+
const data = thread.markers.data[i];
191+
if (data !== null && data.type) {
192+
const markerType = data.type;
193+
const stringIndexMarkerFields =
194+
stringIndexMarkerFieldsByDataType.get(markerType);
195+
if (stringIndexMarkerFields !== undefined) {
196+
for (const fieldKey of stringIndexMarkerFields) {
197+
const stringIndex = (data as any)[fieldKey];
198+
if (typeof stringIndex === 'number') {
199+
const newStringIndex = stringTable.indexForString(
200+
markerStringArray[stringIndex]
201+
);
202+
(data as any)[fieldKey] = newStringIndex;
203+
}
204+
}
205+
}
206+
}
207+
}
208+
}
209+
210+
profile.threads = keptThreads;
211+
212+
// console.log(
213+
// `Have ${markerThreadsByTid.size} marker threads left over which weren't slurped up by sample threads:`,
214+
// [...markerThreadsByTid.keys()]
215+
// );
216+
// if (markerThreadsByTid.size !== 0) {
217+
// console.log(
218+
// `Have ${sampleThreadTidsWithoutCorrespondingMarkerThreads.size} sample threads which didn't find corresponding marker threads:`,
219+
// [...sampleThreadTidsWithoutCorrespondingMarkerThreads]
220+
// );
221+
// }
222+
223+
if (options.outputFile.endsWith('.gz')) {
224+
fs.writeFileSync(
225+
options.outputFile,
226+
await compress(JSON.stringify(profile))
227+
);
228+
} else {
229+
fs.writeFileSync(options.outputFile, JSON.stringify(profile));
230+
}
231+
}
232+
233+
export function makeOptionsFromArgv(processArgv: string[]): CliOptions {
234+
const argv = minimist(processArgv.slice(2));
235+
236+
const hasSamplesHash =
237+
'samples-hash' in argv && typeof argv['samples-hash'] === 'string';
238+
const hasSamplesFile =
239+
'samples-file' in argv && typeof argv['samples-file'] === 'string';
240+
const hasMarkersHash =
241+
'markers-hash' in argv && typeof argv['markers-hash'] === 'string';
242+
const hasMarkersFile =
243+
'markers-file' in argv && typeof argv['markers-file'] === 'string';
244+
245+
if (!hasSamplesHash && !hasSamplesFile) {
246+
throw new Error('Either --samples-file or --samples-hash must be supplied');
247+
}
248+
if (hasSamplesHash && hasSamplesFile) {
249+
throw new Error(
250+
'Only one of --samples-file or --samples-hash can be supplied'
251+
);
252+
}
253+
if (!hasMarkersHash && !hasMarkersFile) {
254+
throw new Error('Either --markers-file or --markers-hash must be supplied');
255+
}
256+
if (hasMarkersHash && hasMarkersFile) {
257+
throw new Error(
258+
'Only one of --markers-file or --markers-hash can be supplied'
259+
);
260+
}
261+
262+
const samplesProf: ProfileSource = hasSamplesHash
263+
? { type: 'HASH', hash: argv['samples-hash'] }
264+
: { type: 'FILE', file: argv['samples-file'] };
265+
const markersProf: ProfileSource = hasMarkersHash
266+
? { type: 'HASH', hash: argv['markers-hash'] }
267+
: { type: 'FILE', file: argv['markers-file'] };
268+
269+
return {
270+
samplesProf,
271+
markersProf,
272+
filterByProcessPrefix: argv['filter-by-process-prefix'],
273+
assumeSamplesProfileHasStartTimeZero: 'assume-samply' in argv,
274+
outputFile: argv['output-file'],
275+
};
276+
}
277+
278+
if (!module.parent) {
279+
try {
280+
const options = makeOptionsFromArgv(process.argv);
281+
run(options).catch((err) => {
282+
throw err;
283+
});
284+
} catch (e) {
285+
console.error(e);
286+
process.exit(1);
287+
}
288+
}

‎src/merge-android-profiles/webpack.config.js‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
const path = require('path');
2+
const projectRoot = path.join(__dirname, '../..');
3+
const includes = [path.join(projectRoot, 'src')];
4+
5+
module.exports = {
6+
name: 'merge-android-profiles',
7+
target: 'node',
8+
mode: process.env.NODE_ENV,
9+
resolve: {
10+
extensions: ['.js', '.jsx', '.ts', '.tsx'],
11+
alias: {
12+
'firefox-profiler-res': path.resolve(projectRoot, 'res'),
13+
},
14+
},
15+
output: {
16+
path: path.resolve(projectRoot, 'dist'),
17+
filename: 'merge-android-profiles.js',
18+
},
19+
entry: './src/merge-android-profiles/index.ts',
20+
module: {
21+
rules: [
22+
{
23+
test: /\.(ts|tsx)$/,
24+
use: ['babel-loader'],
25+
include: includes,
26+
},
27+
{
28+
test: /\.js$/,
29+
include: path.resolve(projectRoot, 'res'),
30+
type: 'asset/resource',
31+
},
32+
{
33+
test: /\.svg$/,
34+
type: 'asset/resource',
35+
},
36+
],
37+
},
38+
experiments: {
39+
// Make WebAssembly work just like in webpack v4
40+
syncWebAssembly: true,
41+
},
42+
};

0 commit comments

Comments
 (0)