Fix #1629: resolve memory leaks in LazilyTransformingAstService and UndoRedo

CHANGELOG.md

@@ -7,8 +7,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+- Added `compactionThreshold` configuration option to control how often the engine compacts accumulated formula transformations. [#1629](https://github.com/handsontable/hyperformula/issues/1629)
+
 ### Fixed
 
+- Fixed a memory leak in `LazilyTransformingAstService` where the transformations array grew unboundedly, causing increasing memory usage over time. [#1629](https://github.com/handsontable/hyperformula/issues/1629)
+- Fixed a memory leak in `UndoRedo` where `oldData` entries for evicted undo stack entries were never cleaned up, causing increasing memory usage over time. [#1629](https://github.com/handsontable/hyperformula/issues/1629)
 - Fixed the IRR function returning `#NUM!` error when the initial investment significantly exceeds the sum of returns. [#1628](https://github.com/handsontable/hyperformula/issues/1628)
 - Fixed the ADDRESS function ignoring `defaultValue` when arguments are syntactically empty (e.g., `=ADDRESS(2,3,,FALSE())`). [#1632](https://github.com/handsontable/hyperformula/issues/1632)
 

src/BuildEngineFactory.ts

@@ -72,7 +72,7 @@ export class BuildEngineFactory {
 
     const namedExpressions = new NamedExpressions()
     const functionRegistry = new FunctionRegistry(config)
-    const lazilyTransformingAstService = new LazilyTransformingAstService(stats)
+    const lazilyTransformingAstService = new LazilyTransformingAstService(stats, config.compactionThreshold)
     const dependencyGraph = DependencyGraph.buildEmpty(lazilyTransformingAstService, config, functionRegistry, namedExpressions, stats)
     const columnSearch = buildColumnSearchStrategy(dependencyGraph, config, stats)
     const sheetMapping = dependencyGraph.sheetMapping

src/Config.ts

@@ -62,6 +62,7 @@ export class Config implements ConfigParams, ParserConfig {
     timeFormats: ['hh:mm', 'hh:mm:ss.sss'],
     thousandSeparator: '',
     undoLimit: 20,
+    compactionThreshold: 50,
     useRegularExpressions: false,
     useWildcards: true,
     useColumnIndex: false,
@@ -135,6 +136,8 @@ export class Config implements ConfigParams, ParserConfig {
   /** @inheritDoc */
   public readonly undoLimit: number
   /** @inheritDoc */
+  public readonly compactionThreshold: number
+  /** @inheritDoc */
   public readonly context: unknown
 
   /**
@@ -198,6 +201,7 @@ export class Config implements ConfigParams, ParserConfig {
       useArrayArithmetic,
       useStats,
       undoLimit,
+      compactionThreshold,
       useColumnIndex,
       useRegularExpressions,
       useWildcards,
@@ -244,10 +248,12 @@ export class Config implements ConfigParams, ParserConfig {
     this.nullDate = configValueFromParamCheck(nullDate, instanceOfSimpleDate, 'IDate', 'nullDate')
     this.leapYear1900 = configValueFromParam(leapYear1900, 'boolean', 'leapYear1900')
     this.undoLimit = configValueFromParam(undoLimit, 'number', 'undoLimit')
+    this.compactionThreshold = configValueFromParam(compactionThreshold, 'number', 'compactionThreshold')
     this.useRegularExpressions = configValueFromParam(useRegularExpressions, 'boolean', 'useRegularExpressions')
     this.useWildcards = configValueFromParam(useWildcards, 'boolean', 'useWildcards')
     this.matchWholeCell = configValueFromParam(matchWholeCell, 'boolean', 'matchWholeCell')
     validateNumberToBeAtLeast(this.undoLimit, 'undoLimit', 0)
+    validateNumberToBeAtLeast(this.compactionThreshold, 'compactionThreshold', 1)
     this.maxRows = configValueFromParam(maxRows, 'number', 'maxRows')
     validateNumberToBeAtLeast(this.maxRows, 'maxRows', 1)
     this.maxColumns = configValueFromParam(maxColumns, 'number', 'maxColumns')

src/ConfigParams.ts

@@ -402,6 +402,18 @@ export interface ConfigParams {
    * @category Undo and Redo
    */
   undoLimit: number,
+  /**
+   * Sets the number of accumulated formula transformations that triggers compaction
+   * of the LazilyTransformingAstService. When the number of pending transformations
+   * reaches this threshold, all formula vertices and column indexes are forced to
+   * apply their postponed transformations, and the transformation history is cleared.
+   *
+   * Lower values cause more frequent compaction (useful for testing), while higher
+   * values reduce overhead at the cost of more memory usage.
+   * @default 50
+   * @category Engine
+   */
+  compactionThreshold: number,
   /**
    * When set to `true`, criteria in functions (SUMIF, COUNTIF, ...) are allowed to use regular expressions.
    * @default false

src/HyperFormula.ts

@@ -4595,6 +4595,20 @@ export class HyperFormula implements TypedEmitter {
     this._functionRegistry = newEngine.functionRegistry
   }
 
+  /**
+   * When enough transformations have accumulated, forces all formula vertices and
+   * column index entries to apply pending lazy transformations, then compacts the
+   * transformation history and cleans up orphaned undo oldData entries.
+   */
+  private compactLazyTransformationsIfNeeded(): void {
+    if (this._lazilyTransformingAstService.needsCompaction()) {
+      this._dependencyGraph.forceApplyPostponedTransformations()
+      this._columnSearch.forceApplyPostponedTransformations()
+      this._lazilyTransformingAstService.compact()
+      this._lazilyTransformingAstService.undoRedo?.cleanupOrphanedOldData()
+    }
+  }
+
   /**
    * Runs a recomputation starting from recently changed vertices.
    *
@@ -4610,6 +4624,8 @@ export class HyperFormula implements TypedEmitter {
       const verticesToRecomputeFrom = this.dependencyGraph.verticesToRecompute()
       this.dependencyGraph.clearDirtyVertices()
 
+      this.compactLazyTransformationsIfNeeded()
+
       if (verticesToRecomputeFrom.length > 0) {
         changes.addAll(this.evaluator.partialRun(verticesToRecomputeFrom))
       }

src/LazilyTransformingAstService.ts

@@ -11,21 +11,56 @@ import {StatType} from './statistics'
 import {Statistics} from './statistics/Statistics'
 import {UndoRedo} from './UndoRedo'
 
+/**
+ * Manages lazy application of formula AST transformations.
+ *
+ * ## Problem
+ * Structural operations (adding/removing rows/columns, moving cells, renaming sheets)
+ * require updating every formula that references the affected area. Applying these
+ * transformations eagerly to all formulas after every operation is expensive, especially
+ * for large spreadsheets with many formulas.
+ *
+ * ## Solution: Lazy Transformation
+ * Instead of transforming all formulas immediately, this service stores transformations
+ * in a queue. Each formula vertex (FormulaVertex) and column index entry (ValueIndex)
+ * tracks its own version number. When a consumer needs up-to-date data, it calls
+ * `applyTransformations()` with its current version and receives all transformations
+ * accumulated since that version.
+ *
+ * ## Compaction
+ * Over time, the transformations array grows unboundedly. To prevent this memory leak,
+ * the engine periodically triggers compaction when the number of accumulated
+ * transformations reaches the configurable `compactionThreshold`:
+ *
+ * 1. All FormulaVertex instances are forced to apply pending transformations
+ *    (via `DependencyGraph.forceApplyPostponedTransformations()`).
+ * 2. All ColumnIndex entries are forced to apply pending transformations
+ *    (via `ColumnSearchStrategy.forceApplyPostponedTransformations()`).
+ * 3. `compact()` is called, which advances `versionOffset` and clears the
+ *    transformations array.
+ * 4. `UndoRedo.cleanupOrphanedOldData()` removes any oldData entries that were
+ *    written during forced application but belong to already-evicted undo entries.
+ *
+ * The `versionOffset` ensures that version numbers remain globally consistent
+ * after compaction: `version() = versionOffset + transformations.length`.
+ */
 export class LazilyTransformingAstService {
 
   public parser?: ParserWithCaching
   public undoRedo?: UndoRedo
 
   private transformations: FormulaTransformer[] = []
+  private versionOffset: number = 0
   private combinedTransformer?: CombinedTransformer
 
   constructor(
     private readonly stats: Statistics,
+    private readonly compactionThreshold: number,
   ) {
   }
 
   public version(): number {
-    return this.transformations.length
+    return this.versionOffset + this.transformations.length
   }
 
   public addTransformation(transformation: FormulaTransformer): number {
@@ -53,8 +88,9 @@ export class LazilyTransformingAstService {
   public applyTransformations(ast: Ast, address: SimpleCellAddress, version: number): [Ast, SimpleCellAddress, number] {
     this.stats.start(StatType.TRANSFORM_ASTS_POSTPONED)
 
-    for (let v = version; v < this.transformations.length; v++) {
-      const transformation = this.transformations[v]
+    const currentVersion = this.version()
+    for (let v = Math.max(version, this.versionOffset); v < currentVersion; v++) {
+      const transformation = this.transformations[v - this.versionOffset]
       if (transformation.isIrreversible()) {
         this.undoRedo!.storeDataForVersion(v, address, this.parser!.computeHashFromAst(ast))
         this.parser!.rememberNewAst(ast)
@@ -67,15 +103,37 @@ export class LazilyTransformingAstService {
     const cachedAst = this.parser!.rememberNewAst(ast)
 
     this.stats.end(StatType.TRANSFORM_ASTS_POSTPONED)
-    return [cachedAst, address, this.transformations.length]
+    return [cachedAst, address, currentVersion]
   }
 
   public* getTransformationsFrom(version: number, filter?: (transformation: FormulaTransformer) => boolean): IterableIterator<FormulaTransformer> {
-    for (let v = version; v < this.transformations.length; v++) {
-      const transformation = this.transformations[v]
+    const currentVersion = this.version()
+    for (let v = Math.max(version, this.versionOffset); v < currentVersion; v++) {
+      const transformation = this.transformations[v - this.versionOffset]
       if (!filter || filter(transformation)) {
         yield transformation
       }
     }
   }
+
+  /**
+   * Returns true when enough transformations have accumulated to justify the cost
+   * of forcing all consumers (FormulaVertex, ColumnIndex) to apply pending changes.
+   */
+  public needsCompaction(): boolean {
+    return this.transformations.length >= this.compactionThreshold
+  }
+
+  /**
+   * Compacts the transformations array by discarding all entries that have already
+   * been applied by every consumer. Safe to call only after all FormulaVertex and
+   * ColumnIndex consumers have been brought up to the current version.
+   * After calling, UndoRedo.cleanupOrphanedOldData() must be invoked to remove
+   * oldData entries written during forceApplyPostponedTransformations for
+   * already-evicted undo entries.
+   */
+  public compact(): void {
+    this.versionOffset += this.transformations.length
+    this.transformations = []
+  }
 }

src/Lookup/ColumnBinarySearch.ts

@@ -53,6 +53,16 @@ export class ColumnBinarySearch extends AdvancedFind implements ColumnSearchStra
   public removeValues(range: IterableIterator<[RawScalarValue, SimpleCellAddress]>): void {
   }
 
+  /**
+   * No-op: ColumnBinarySearch reads cell values directly from the dependency graph
+   * on every lookup, so it has no cached data that could become stale.
+   * Unlike ColumnIndex, which maintains a separate value-to-address index that
+   * must be kept in sync with lazy transformations, binary search always operates
+   * on the current graph state.
+   */
+  public forceApplyPostponedTransformations(): void {
+  }
+
   /*
    * WARNING: Finding lower/upper bounds in unordered ranges is not supported. When ordering === 'none', assumes matchExactly === true
    */

src/Lookup/ColumnIndex.ts

@@ -184,6 +184,24 @@ export class ColumnIndex implements ColumnSearchStrategy {
     this.index.delete(sheetId)
   }
 
+  /**
+   * Forces all ValueIndex entries to apply any pending lazy transformations,
+   * bringing every entry up to the current LazilyTransformingAstService version.
+   * Must be called before compacting LazilyTransformingAstService.
+   */
+  public forceApplyPostponedTransformations(): void {
+    for (const [sheet, sheetIndex] of this.index) {
+      sheetIndex.forEach((columnMap, col) => {
+        if (!columnMap) {
+          return
+        }
+        for (const value of columnMap.keys()) {
+          this.ensureRecentData(sheet, col, value)
+        }
+      })
+    }
+  }
+
   public getColumnMap(sheet: number, col: number): ColumnMap {
     if (!this.index.has(sheet)) {
       this.index.set(sheet, [])

src/Lookup/SearchStrategy.ts

@@ -51,6 +51,13 @@ export interface ColumnSearchStrategy extends SearchStrategy {
   moveValues(range: IterableIterator<[RawScalarValue, SimpleCellAddress]>, toRight: number, toBottom: number, toSheet: number): void,
 
   removeValues(range: IterableIterator<[RawScalarValue, SimpleCellAddress]>): void,
+
+  /**
+   * Forces all lazily-tracked ValueIndex entries to apply any pending transformations,
+   * bringing every entry's version up to the current LazilyTransformingAstService version.
+   * Must be called before compacting LazilyTransformingAstService.
+   */
+  forceApplyPostponedTransformations(): void,
 }
 
 export function buildColumnSearchStrategy(dependencyGraph: DependencyGraph, config: Config, statistics: Statistics): ColumnSearchStrategy {

src/Operations.ts

@@ -737,8 +737,14 @@ export class Operations {
     return changes
   }
 
+  /**
+   * Forces all formula vertices and column index entries to apply pending lazy
+   * transformations, bringing them up to the current LazilyTransformingAstService version.
+   * Called before undo of move operations and before compaction.
+   */
   public forceApplyPostponedTransformations(): void {
     this.dependencyGraph.forceApplyPostponedTransformations()
+    this.columnSearch.forceApplyPostponedTransformations()
   }
 
   /**