Give the frameTable a lib column

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,14 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 70
+
+The `lib` column moved off the `resourceTable` and onto the `frameTable`. Previously a frame's library was reached indirectly, via `frame -> func -> resource -> lib`; now the frame points at its library directly, and `resourceTable` has no `lib` column at all. Frames with no library have their lib set to the `-1` sentinel.
+
+This decouples resources from libraries.
+
+The `lib` column can optionally be stored as an `Int32Array`, for profiles loaded from [JsonSlabs](https://github.com/mstange/json-slabs/) files (.jslb, .jslb.gz). Regular JS / JSON arrays are still accepted - but note that `-1` (not `null`) must be used regardless of format.
+
 ### Version 69
 
 A new marker schema display location, `timeline-network`, was added. A marker schema can list `timeline-network` in its `display` array to have markers of that type surfaced in the Network track.

profiler-cli/src/test/unit/call-tree-formatting.test.ts

@@ -52,9 +52,8 @@ function buildTopDownResult(
   const state = store.getState();
   const threadSelectors = getThreadSelectors(0);
   const callTree = threadSelectors.getCallTree(state);
-  const libs = profile.libs;
 
-  const regularCallTree = collectCallTree(callTree, libs, options);
+  const regularCallTree = collectCallTree(callTree, options);
 
   return {
     type: 'thread-samples-top-down',
@@ -76,7 +75,6 @@ function buildBottomUpResult(
   const store = storeWithProfile(profile);
   const state = store.getState();
   const threadSelectors = getThreadSelectors(0);
-  const libs = profile.libs;
 
   // Build inverted call tree (bottom-up view)
   let collectedInvertedTree = null;
@@ -118,7 +116,7 @@ function buildBottomUpResult(
       weightType
     );
 
-    collectedInvertedTree = collectCallTree(invertedTree, libs, options);
+    collectedInvertedTree = collectCallTree(invertedTree, options);
   } catch (e) {
     // Failed to create inverted tree
     console.error('Failed to create inverted call tree:', e);

src/app-logic/constants.ts

@@ -12,7 +12,7 @@ export const GECKO_PROFILE_VERSION = 36;
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 69;
+export const PROCESSED_PROFILE_VERSION = 70;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/profile-logic/bottom-box.ts

@@ -2,12 +2,11 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-import { ResourceType } from 'firefox-profiler/types';
-
 import type {
   Thread,
   IndexIntoStackTable,
   IndexIntoCallNodeTable,
+  IndexIntoLibs,
   BottomBoxInfo,
   SamplesLikeTable,
 } from 'firefox-profiler/types';
@@ -37,14 +36,8 @@ export function getBottomBoxInfoForCallNode(
   thread: Thread,
   samples: SamplesLikeTable
 ): BottomBoxInfo {
-  const {
-    stackTable,
-    frameTable,
-    funcTable,
-    stringTable,
-    resourceTable,
-    nativeSymbols,
-  } = thread;
+  const { stackTable, frameTable, funcTable, stringTable, nativeSymbols } =
+    thread;
 
   const funcIndex = callNodeInfo.funcForNode(callNodeIndex);
   const { source: sourceIndex, line: funcLine } = getOriginalPositionForFrame(
@@ -54,17 +47,24 @@ export function getBottomBoxInfoForCallNode(
     funcTable,
     thread.sourceLocationTable
   );
-  const resource = funcTable.resource[funcIndex];
-  const libIndex =
-    resource !== -1 && resourceTable.type[resource] === ResourceType.Library
-      ? resourceTable.lib[resource]
-      : null;
   const callNodeFramePerStack = getCallNodeFramePerStack(
     callNodeIndex,
     callNodeInfo,
     stackTable
   );
 
+  // All frames of a call node share the same func, but in principle they can
+  // come from different libraries (e.g. two builds of the same library in a
+  // comparison profile). Use the library of the first frame that has one.
+  let libIndex: IndexIntoLibs | null = null;
+  for (const frameIndex of callNodeFramePerStack.values()) {
+    const frameLib = frameTable.lib[frameIndex];
+    if (frameLib !== -1) {
+      libIndex = frameLib;
+      break;
+    }
+  }
+
   // If we have at least one native symbol to show assembly for, pick
   // the one with the highest total. But first, create the full list of
   // native symbols for this call node, including even those symbols
@@ -148,14 +148,8 @@ export function getBottomBoxInfoForStackFrame(
   stackIndex: IndexIntoStackTable,
   thread: Thread
 ): BottomBoxInfo {
-  const {
-    stackTable,
-    frameTable,
-    funcTable,
-    resourceTable,
-    nativeSymbols,
-    stringTable,
-  } = thread;
+  const { stackTable, frameTable, funcTable, nativeSymbols, stringTable } =
+    thread;
 
   const frameIndex = stackTable.frame[stackIndex];
   const funcIndex = frameTable.func[frameIndex];
@@ -166,11 +160,8 @@ export function getBottomBoxInfoForStackFrame(
     funcTable,
     thread.sourceLocationTable
   );
-  const resource = funcTable.resource[funcIndex];
-  const libIndex =
-    resource !== -1 && resourceTable.type[resource] === ResourceType.Library
-      ? resourceTable.lib[resource]
-      : null;
+  const frameLib = frameTable.lib[frameIndex];
+  const libIndex = frameLib !== -1 ? frameLib : null;
 
   // Get native symbol for this frame
   const nativeSymbol = frameTable.nativeSymbol[frameIndex];

src/profile-logic/data-structures.ts

@@ -35,6 +35,7 @@ import type {
   IndexIntoSubcategoryListForCategory,
   IndexIntoNativeSymbolTable,
   IndexIntoSourceLocationTable,
+  IndexIntoLibs,
   InnerWindowID,
   Address,
   Bytes,
@@ -117,6 +118,7 @@ export type RawFrameTableBuilder = {
   category: (IndexIntoCategoryList | null)[];
   subcategory: (IndexIntoSubcategoryListForCategory | null)[];
   func: IndexIntoFuncTable[];
+  lib: Array<IndexIntoLibs | -1>;
   nativeSymbol: (IndexIntoNativeSymbolTable | null)[];
   innerWindowID: (InnerWindowID | null)[];
   line: (number | null)[];
@@ -289,6 +291,7 @@ export function getRawFrameTableBuilder(): RawFrameTableBuilder {
     category: [],
     subcategory: [],
     func: [],
+    lib: [],
     nativeSymbol: [],
     innerWindowID: [],
     line: [],
@@ -311,6 +314,7 @@ export function getRawFrameTableBuilderWithExistingContents(
     category: frameTable.category.slice(),
     subcategory: frameTable.subcategory.slice(),
     func: Array.from(frameTable.func),
+    lib: Array.from(frameTable.lib),
     nativeSymbol: frameTable.nativeSymbol.slice(),
     innerWindowID: frameTable.innerWindowID.slice(),
     line: frameTable.line.slice(),
@@ -409,7 +413,6 @@ export function getEmptyResourceTable(): ResourceTable {
     // If modifying this structure, please update all callers of this function to ensure
     // that they are pushing on correctly to the data structure. These pushes may not
     // be caught by the type system.
-    lib: [],
     name: [],
     host: [],
     type: [],

src/profile-logic/global-data-collector.ts

@@ -61,8 +61,6 @@ export class GlobalDataCollector {
   _funcKeyToFuncIndex: Map<string, IndexIntoFuncTable> = new Map();
   _nativeSymbolKeyToNativeSymbolIndex: Map<string, IndexIntoNativeSymbolTable> =
     new Map();
-  _libIndexToResourceIndex: Map<IndexIntoLibs, IndexIntoResourceTable> =
-    new Map();
   _libNameToResourceIndex: Map<IndexIntoStringTable, IndexIntoResourceTable> =
     new Map();
   _originToResourceIndex: Map<string, IndexIntoResourceTable> = new Map();
@@ -179,7 +177,6 @@ export class GlobalDataCollector {
 
         const idIndex = this._stringTable.indexForString(extensions.id[i]);
 
-        resourceTable.lib[resourceIndex] = null;
         resourceTable.name[resourceIndex] =
           this._stringTable.indexForString(name);
         resourceTable.host[resourceIndex] = idIndex;
@@ -223,7 +220,6 @@ export class GlobalDataCollector {
     this._originToResourceIndex.set(origin, resourceIndex);
     if (host) {
       // This is a webhost URL.
-      resourceTable.lib[resourceIndex] = null;
       resourceTable.name[resourceIndex] =
         this._stringTable.indexForString(origin);
       resourceTable.host[resourceIndex] =
@@ -232,7 +228,6 @@ export class GlobalDataCollector {
     } else {
       // This is a URL, but it doesn't point to something on the web, e.g. a
       // chrome url.
-      resourceTable.lib[resourceIndex] = null;
       resourceTable.name[resourceIndex] =
         this._stringTable.indexForString(scriptURI);
       resourceTable.host[resourceIndex] = null;
@@ -241,25 +236,9 @@ export class GlobalDataCollector {
     return resourceIndex;
   }
 
-  indexForLibResource(libIndex: IndexIntoLibs): IndexIntoResourceTable {
-    let resourceIndex = this._libIndexToResourceIndex.get(libIndex);
-    if (resourceIndex !== undefined) {
-      return resourceIndex;
-    }
-
-    const resourceTable = this._resourceTable;
-
-    resourceIndex = this._resourceTable.length++;
-    this._libIndexToResourceIndex.set(libIndex, resourceIndex);
-    resourceTable.lib[resourceIndex] = libIndex;
-    resourceTable.name[resourceIndex] = this._stringTable.indexForString(
-      this._libs[libIndex].name
-    );
-    resourceTable.host[resourceIndex] = null;
-    resourceTable.type[resourceIndex] = ResourceType.Library;
-    return resourceIndex;
-  }
-
+  // Returns the resource for a library, identified only by its name. Two
+  // different libs with the same name share one resource; the frames keep the
+  // libs apart via their own `lib` column.
   indexForNameOnlyLibResource(
     libNameStringIndex: IndexIntoStringTable
   ): IndexIntoResourceTable {
@@ -272,13 +251,21 @@ export class GlobalDataCollector {
 
     resourceIndex = this._resourceTable.length++;
     this._libNameToResourceIndex.set(libNameStringIndex, resourceIndex);
-    resourceTable.lib[resourceIndex] = null;
     resourceTable.name[resourceIndex] = libNameStringIndex;
     resourceTable.host[resourceIndex] = null;
     resourceTable.type[resourceIndex] = ResourceType.Library;
     return resourceIndex;
   }
 
+  // Same as indexForNameOnlyLibResource, for when you have a lib index rather
+  // than just a name. Note that the resulting resource only records the lib's
+  // name; the lib itself belongs on the frame.
+  indexForLibResource(libIndex: IndexIntoLibs): IndexIntoResourceTable {
+    return this.indexForNameOnlyLibResource(
+      this._stringTable.indexForString(this._libs[libIndex].name)
+    );
+  }
+
   indexForNativeSymbol(
     libIndex: IndexIntoLibs,
     address: Address,

src/profile-logic/import/chrome.ts

@@ -699,6 +699,7 @@ async function processTracingEvents(
           frameTable.category[frameIndex] = category;
           frameTable.subcategory[frameIndex] = 0;
           frameTable.func[frameIndex] = funcId;
+          frameTable.lib[frameIndex] = -1;
           frameTable.nativeSymbol[frameIndex] = null;
           frameTable.innerWindowID[frameIndex] = 0;
           frameTable.line[frameIndex] =

src/profile-logic/import/dhat.ts

@@ -212,6 +212,7 @@ export function attemptToConvertDhat(json: unknown): Profile | null {
   frameTable.category.push(otherCategory);
   frameTable.subcategory.push(otherSubCategory);
   frameTable.innerWindowID.push(null);
+  frameTable.lib.push(-1);
   frameTable.nativeSymbol.push(null);
   frameTable.inlineDepth.push(0);
   frameTable.func.push(rootFuncIndex);
@@ -279,6 +280,7 @@ export function attemptToConvertDhat(json: unknown): Profile | null {
     frameTable.category.push(otherCategory);
     frameTable.subcategory.push(otherSubCategory);
     frameTable.innerWindowID.push(null);
+    frameTable.lib.push(-1);
     frameTable.nativeSymbol.push(null);
     frameTable.inlineDepth.push(0);
     frameTable.func.push(funcIndex);

src/profile-logic/import/flame-graph.ts

@@ -120,6 +120,7 @@ export function convertFlameGraphProfile(profileText: string): Profile {
     frameTable.category.push(category);
     frameTable.subcategory.push(0);
     frameTable.func.push(funcIndex);
+    frameTable.lib.push(-1);
     frameTable.nativeSymbol.push(null);
     frameTable.innerWindowID.push(null);
     frameTable.line.push(null);

src/profile-logic/import/simpleperf.ts

@@ -99,7 +99,6 @@ class FirefoxResourceTable {
   findOrAddResource(file: report.IFile): IndexIntoResourceTable {
     let resourceIndex = this.resourcesMap.get(file.id!);
     if (!resourceIndex) {
-      this.resourceTable.lib.push(null);
       this.resourceTable.name.push(this.strings.indexForString(file.path!));
       this.resourceTable.host.push(null);
       this.resourceTable.type.push(1); // Library
@@ -177,6 +176,7 @@ class FirefoxFrameTable {
       this.frameTable.category.push(category);
       this.frameTable.subcategory.push(0);
       this.frameTable.func.push(funcIndex);
+      this.frameTable.lib.push(-1);
       this.frameTable.nativeSymbol.push(null);
       this.frameTable.innerWindowID.push(null);
       this.frameTable.line.push(null);

src/profile-logic/insert-stack-labels.ts

@@ -136,6 +136,7 @@ export function insertStackLabels(
     frameTable.func[frameIndex] = funcIndex;
     frameTable.category[frameIndex] = labelCategoryIndex;
     frameTable.subcategory[frameIndex] = 0;
+    frameTable.lib[frameIndex] = -1;
     frameTable.nativeSymbol[frameIndex] = null;
     frameTable.address[frameIndex] = 0;
     frameTable.inlineDepth[frameIndex] = 0;

src/profile-logic/js-tracer.ts

@@ -610,6 +610,7 @@ export function convertJsTracerToThreadWithoutSamples(
     frameTable.inlineDepth.push(0);
     frameTable.category.push(otherCategory);
     frameTable.func.push(funcIndex);
+    frameTable.lib.push(-1);
     frameTable.nativeSymbol.push(null);
     frameTable.innerWindowID.push(0);
     frameTable.line.push(line);