Change profile.shared.stackTable.prefix to prefixOffset (delta from current index)

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,25 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 67
+
+The `prefix` column of `profile.shared.stackTable` was replaced with a `prefixOffset` column, to improve compressibility.
+
+The `prefixOffset` values have the following meanings:
+
+- `prefixOffset[i] === 0` means: stack `i` is a root.
+- Otherwise, `prefixOffset[i] === k` (k > 0) means: `i`'s parent has index `i - k`.
+
+Parents always come before their children in the stack table (the stack table is stored in topological order), so these offsets are always positive and always point backwards.
+
+The `prefixOffset` column can be stored as an array of numbers or as an `Int32Array` (when using JSLB).
+
+### Version 66
+
+The `prefix` column of `profile.shared.stackTable` now uses `-1` instead of `null` to indicate "this stack node is a root".
+Furthermore, for profiles loaded from [JsonSlabs](https://github.com/mstange/json-slabs/) files (.jslb, .jslb.gz),
+`profile.shared.stackTable.prefix` can now optionally be stored as an `Int32Array`. Regular JS / JSON arrays are still accepted.
+
 ### Version 65
 
 The stack table's `frame` column (stored at `profile.shared.stackTable.frame`) can now optionally be stored as an `Int32Array`, for profiles loaded from [JsonSlabs](https://github.com/mstange/json-slabs/) files (.jslb, .jslb.gz). Regular JS / JSON arrays are still accepted.

src/app-logic/constants.ts

@@ -12,7 +12,7 @@ export const GECKO_PROFILE_VERSION = 34;
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 65;
+export const PROCESSED_PROFILE_VERSION = 67;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/app-logic/url-handling.ts

@@ -1649,19 +1649,20 @@ function getStackIndexFromVersion3JSCallNodePath(
   oldCallNodePath: CallNodePath
 ): IndexIntoStackTable | null {
   const { stackTable, funcTable, frameTable } = shared;
-  const stackIndexDepth: Map<IndexIntoStackTable | null, number> = new Map();
-  stackIndexDepth.set(null, -1);
+  const stackIndexDepth: Map<IndexIntoStackTable, number> = new Map();
+  // Map key -1 represents the virtual root above all stack indexes.
+  stackIndexDepth.set(-1, -1);
 
   for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
-    const prefix = stackTable.prefix[stackIndex];
+    const offset = stackTable.prefixOffset[stackIndex];
+    const prefix = offset === 0 ? -1 : stackIndex - offset;
     const frameIndex = stackTable.frame[stackIndex];
     const funcIndex = frameTable.func[frameIndex];
     const isJS = funcTable.isJS[funcIndex];
     // We know that at this point stack table is sorted and the following
     // condition holds:
-    // assert(prefixStack === null || prefixStack < stackIndex);
-    const doesPrefixMatchCallNodePath =
-      prefix === null || stackIndexDepth.has(prefix);
+    // assert(prefixStack === -1 || prefixStack < stackIndex);
+    const doesPrefixMatchCallNodePath = stackIndexDepth.has(prefix);
 
     if (!doesPrefixMatchCallNodePath) {
       continue;
@@ -1696,14 +1697,15 @@ function getVersion4JSCallNodePathFromStackIndex(
 ): CallNodePath {
   const { funcTable, stackTable, frameTable } = shared;
   const callNodePath = [];
-  let nextStackIndex: IndexIntoStackTable | null = stackIndex;
-  while (nextStackIndex !== null) {
+  let nextStackIndex: IndexIntoStackTable = stackIndex;
+  while (nextStackIndex !== -1) {
     const frameIndex: IndexIntoFrameTable = stackTable.frame[nextStackIndex];
     const funcIndex = frameTable.func[frameIndex];
     if (funcTable.isJS[funcIndex] || funcTable.relevantForJS[funcIndex]) {
       callNodePath.unshift(funcIndex);
     }
-    nextStackIndex = stackTable.prefix[nextStackIndex];
+    const offset = stackTable.prefixOffset[nextStackIndex];
+    nextStackIndex = offset === 0 ? -1 : nextStackIndex - offset;
   }
   return callNodePath;
 }

src/profile-logic/address-timings.ts

@@ -127,7 +127,7 @@ export function getStackAddressInfo(
   for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
     const prefixStack = stackTable.prefix[stackIndex];
     const prefixAddressSet: IndexIntoAddressSetTable | -1 =
-      prefixStack !== null ? stackIndexToAddressSetIndex[prefixStack] : -1;
+      prefixStack !== -1 ? stackIndexToAddressSetIndex[prefixStack] : -1;
 
     const frame = stackTable.frame[stackIndex];
     const nativeSymbolOfThisStack = frameTable.nativeSymbol[frame];

src/profile-logic/data-structures.ts

@@ -70,9 +70,14 @@ export function getRawStackTableBuilder(): RawStackTableBuilder {
 export function getRawStackTableBuilderWithExistingContents(
   existing: RawStackTable
 ): RawStackTableBuilder {
+  const prefix = new Array<IndexIntoStackTable | null>(existing.length);
+  for (let i = 0; i < existing.length; i++) {
+    const offset = existing.prefixOffset[i];
+    prefix[i] = offset === 0 ? null : i - offset;
+  }
   return {
     frame: [...existing.frame],
-    prefix: [...existing.prefix],
+    prefix,
     length: existing.length,
   };
 }
@@ -81,9 +86,14 @@ export function finishRawStackTableBuilder(
   builder: RawStackTableBuilder
 ): RawStackTable {
   const { frame, prefix, length } = builder;
+  const prefixOffset = new Int32Array(length);
+  for (let i = 0; i < length; i++) {
+    const p = prefix[i];
+    prefixOffset[i] = p === null ? 0 : i - p;
+  }
   return {
     frame: new Int32Array(frame),
-    prefix,
+    prefixOffset,
     length,
   };
 }

src/profile-logic/import/chrome.ts

@@ -5,12 +5,15 @@
 import type {
   Profile,
   RawThread,
-  RawStackTable,
   IndexIntoStackTable,
   MixedObject,
 } from 'firefox-profiler/types';
 
-import { getEmptyProfile, getEmptyThread } from '../data-structures';
+import {
+  getEmptyProfile,
+  getEmptyThread,
+  type RawStackTableBuilder,
+} from '../data-structures';
 import type { StringTable } from '../../utils/string-table';
 import { ensureExists, coerce } from '../../utils/types';
 import {
@@ -868,7 +871,7 @@ function getImageSize(
  * For sanity, check that stacks are ordered where the prefix stack
  * always preceeds the current stack index in the StackTable.
  */
-function assertStackOrdering(stackTable: RawStackTable) {
+function assertStackOrdering(stackTable: RawStackTableBuilder) {
   const visitedStacks = new Set<number | null>([null]);
   for (let i = 0; i < stackTable.length; i++) {
     if (!visitedStacks.has(stackTable.prefix[i])) {

src/profile-logic/insert-stack-labels.ts

@@ -4,7 +4,6 @@
 
 import type {
   IndexIntoFrameTable,
-  IndexIntoStackTable,
   RawStackTable,
   IndexIntoFuncTable,
   Profile,
@@ -197,10 +196,10 @@ export function insertStackLabels(
   );
   let stacksToInsertCount = 0;
   for (let stackIndex = 0; stackIndex < oldStackTable.length; stackIndex++) {
-    const parentStackIndex = oldStackTable.prefix[stackIndex];
+    const prefixOffset = oldStackTable.prefixOffset[stackIndex];
     const inheritedLabelFrameIndex =
-      parentStackIndex !== null
-        ? inheritedLabelFrameIndexAtStack[parentStackIndex]
+      prefixOffset !== 0
+        ? inheritedLabelFrameIndexAtStack[stackIndex - prefixOffset]
         : null;
     const frameIndex = oldStackTable.frame[stackIndex];
     const funcIndex = oldFrameTable.func[frameIndex];
@@ -218,7 +217,7 @@ export function insertStackLabels(
     ) {
       labelFrameIndexToInsertAtStack[stackIndex] = null;
       inheritedLabelFrameIndexAtStack[stackIndex] = null;
-    } else if (parentStackIndex === null) {
+    } else if (prefixOffset === 0) {
       labelFrameIndexToInsertAtStack[stackIndex] = rootLabelFrameIndex;
       inheritedLabelFrameIndexAtStack[stackIndex] = rootLabelFrameIndex;
       stacksToInsertCount++;
@@ -230,7 +229,7 @@ export function insertStackLabels(
 
   // Now compute the new stack table.
   const newStackCount = oldStackTable.length + stacksToInsertCount;
-  const newPrefixCol = new Array<IndexIntoStackTable | null>(newStackCount);
+  const newPrefixOffsetCol = new Int32Array(newStackCount);
   const newFrameCol = new Array<IndexIntoFrameTable>(newStackCount);
   const oldStackToNewStackPlusOne = new Int32Array(oldStackTable.length);
   let nextNewStackIndex = 0;
@@ -241,18 +240,22 @@ export function insertStackLabels(
   ) {
     const labelFrameIndexToInsert =
       labelFrameIndexToInsertAtStack[oldStackIndex];
-    const oldPrefix = oldStackTable.prefix[oldStackIndex];
+    const oldPrefixOffset = oldStackTable.prefixOffset[oldStackIndex];
+    const oldPrefix =
+      oldPrefixOffset !== 0 ? oldStackIndex - oldPrefixOffset : -1;
     let newPrefix =
-      oldPrefix !== null ? oldStackToNewStackPlusOne[oldPrefix] - 1 : null;
+      oldPrefix !== -1 ? oldStackToNewStackPlusOne[oldPrefix] - 1 : -1;
     const frameIndex = oldStackTable.frame[oldStackIndex];
     if (labelFrameIndexToInsert !== null) {
       const insertedStackIndex = nextNewStackIndex++;
-      newPrefixCol[insertedStackIndex] = newPrefix;
+      newPrefixOffsetCol[insertedStackIndex] =
+        newPrefix === -1 ? 0 : insertedStackIndex - newPrefix;
       newFrameCol[insertedStackIndex] = labelFrameIndexToInsert;
       newPrefix = insertedStackIndex;
     }
     const newStackIndex = nextNewStackIndex++;
-    newPrefixCol[newStackIndex] = newPrefix;
+    newPrefixOffsetCol[newStackIndex] =
+      newPrefix === -1 ? 0 : newStackIndex - newPrefix;
     newFrameCol[newStackIndex] = frameIndex;
     oldStackToNewStackPlusOne[oldStackIndex] = newStackIndex + 1;
   }
@@ -266,7 +269,7 @@ export function insertStackLabels(
   }
 
   const stackTable: RawStackTable = {
-    prefix: newPrefixCol,
+    prefixOffset: newPrefixOffsetCol,
     frame: newFrameCol,
     length: newStackCount,
   };

src/profile-logic/line-timings.ts

@@ -57,7 +57,7 @@ export function getStackLineInfo(
   for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
     const prefixStack = stackTable.prefix[stackIndex];
     const prefixLineSet: IndexIntoLineSetTable | -1 =
-      prefixStack !== null ? stackIndexToLineSetIndex[prefixStack] : -1;
+      prefixStack !== -1 ? stackIndexToLineSetIndex[prefixStack] : -1;
 
     const frame = stackTable.frame[stackIndex];
     const func = frameTable.func[frame];

src/profile-logic/merge-compare.ts

@@ -1135,9 +1135,10 @@ function mergeStackTables(
         stackTable.frame[i],
         oldFrameToNewFramePlusOne
       );
-      const prefix = stackTable.prefix[i];
+      const offset = stackTable.prefixOffset[i];
+      const prefix = offset === 0 ? -1 : i - offset;
       const newPrefix =
-        prefix === null ? null : oldStackToNewStackPlusOne[prefix] - 1;
+        prefix === -1 ? null : oldStackToNewStackPlusOne[prefix] - 1;
 
       newStackTable.frame.push(frameIndex);
       newStackTable.prefix.push(newPrefix);

src/profile-logic/process-profile.ts

@@ -2086,7 +2086,6 @@ export function serializeProfileToJsonSlabsFile(
     // JSON" for these tables will become much smaller and we won't need to split
     // out those tables anymore.
     profile.threads,
-    profile.shared.stackTable,
     profile.shared.frameTable,
     profile.shared.funcTable,
     profile.shared.stringArray,

src/profile-logic/processed-profile-versioning.ts

@@ -81,10 +81,6 @@ export function attemptToUpgradeProcessedProfileThroughMutation(
       ? meta.preprocessedProfileVersion
       : UNANNOTATED_VERSION;
 
-  if (profileVersion === PROCESSED_PROFILE_VERSION) {
-    return profile;
-  }
-
   if (profileVersion > PROCESSED_PROFILE_VERSION) {
     throw new ProfileVersionError(
       'processed',
@@ -104,12 +100,24 @@ export function attemptToUpgradeProcessedProfileThroughMutation(
     }
   }
 
+  _normalizeAfterUpgrade(profile);
+
   const upgradedProfile = profile as Profile;
   upgradedProfile.meta.preprocessedProfileVersion = PROCESSED_PROFILE_VERSION;
 
   return upgradedProfile;
 }
 
+// Fixups for things that don't survive JSON roundtripping at the current
+// profile version, such as typed array columns being parsed as regular
+// arrays. Idempotent; safe to run on an already-normalized profile.
+function _normalizeAfterUpgrade(profile: any): void {
+  const stackTable = profile.shared?.stackTable ?? null;
+  if (stackTable && !(stackTable.prefixOffset instanceof Int32Array)) {
+    stackTable.prefixOffset = new Int32Array(stackTable.prefixOffset);
+  }
+}
+
 function _archFromAbi(abi: string): string {
   if (abi === 'x86_64-gcc3') {
     return 'x86_64';
@@ -3261,6 +3269,39 @@ const _upgraders: {
     // `IndexIntoFrameTable[]` to `IndexIntoFrameTable[] | Int32Array<ArrayBuffer>`.
     // All valid v64 profiles are valid v65 profiles, so no upgrader is needed.
   },
+  [66]: (profile: any) => {
+    // The stackTable.prefix column is now stored as an Int32Array with -1
+    // meaning "root", instead of an Array<number | null> with null meaning
+    // "root". Convert any null entries to -1 and turn the column into an
+    // Int32Array.
+    const stackTable = profile.shared?.stackTable ?? null;
+    if (stackTable && Array.isArray(stackTable.prefix)) {
+      const oldPrefix = stackTable.prefix;
+      const newPrefix = new Int32Array(oldPrefix.length);
+      for (let i = 0; i < oldPrefix.length; i++) {
+        const p = oldPrefix[i];
+        newPrefix[i] = p === null ? -1 : p;
+      }
+      stackTable.prefix = newPrefix;
+    }
+  },
+  [67]: (profile: any) => {
+    // The stackTable.prefix column was replaced with a stackTable.prefixOffset
+    // column. For each stack i, prefixOffset[i] is 0 if i is a root, otherwise
+    // it is i's offset from its parent (parent index = i - prefixOffset[i]).
+    const stackTable = profile.shared?.stackTable ?? null;
+    if (stackTable && stackTable.prefix !== undefined) {
+      const oldPrefix = stackTable.prefix;
+      const length = oldPrefix.length;
+      const prefixOffset = new Int32Array(length);
+      for (let i = 0; i < length; i++) {
+        const p = oldPrefix[i];
+        prefixOffset[i] = p === -1 || p === null ? 0 : i - p;
+      }
+      stackTable.prefixOffset = prefixOffset;
+      delete stackTable.prefix;
+    }
+  },
   // If you add a new upgrader here, please document the change in
   // `docs-developer/CHANGELOG-formats.md`.
 };