feat(utils): add file sink classes

packages/utils/src/lib/file-sink-json-trace.ts

@@ -0,0 +1,167 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js';
+import { getCompleteEvent, getStartTracing } from './trace-file-utils.js';
+import type {
+  InstantEvent,
+  SpanEvent,
+  TraceEvent,
+  TraceEventRaw,
+  UserTimingDetail,
+} from './trace-file.type.js';
+
+const tryJson = <T>(v: unknown): T | unknown => {
+  if (typeof v !== 'string') return v;
+  try {
+    return JSON.parse(v) as T;
+  } catch {
+    return v;
+  }
+};
+
+const toJson = (v: unknown): unknown => {
+  if (v === undefined) return undefined;
+  try {
+    return JSON.stringify(v);
+  } catch {
+    return v;
+  }
+};
+
+export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent {
+  if (!args) return rest as TraceEvent;
+
+  const out: any = { ...args };
+  if ('detail' in out) out.detail = tryJson<UserTimingDetail>(out.detail);
+  if (out.data?.detail)
+    out.data.detail = tryJson<UserTimingDetail>(out.data.detail);
+
+  return { ...rest, args: out } as TraceEvent;
+}
+
+export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw {
+  if (!args) return rest as TraceEventRaw;
+
+  const out: any = { ...args };
+  if ('detail' in out) out.detail = toJson(out.detail);
+  if (out.data?.detail) out.data.detail = toJson(out.data.detail);
+
+  return { ...rest, args: out } as TraceEventRaw;
+}
+
+function getTraceMetadata(
+  startDate?: Date,
+  metadata?: Record<string, unknown>,
+) {
+  return {
+    source: 'DevTools',
+    startTime: startDate?.toISOString() ?? new Date().toISOString(),
+    hardwareConcurrency: 1,
+    dataOrigin: 'TraceEvents',
+    ...metadata,
+  };
+}
+
+function createTraceFileContent(
+  traceEventsContent: string,
+  startDate?: Date,
+  metadata?: Record<string, unknown>,
+): string {
+  return `{
+  "metadata": ${JSON.stringify(getTraceMetadata(startDate, metadata))},
+  "traceEvents": [
+${traceEventsContent}
+  ]
+}`;
+}
+
+function finalizeTraceFile(
+  events: (SpanEvent | InstantEvent)[],
+  outputPath: string,
+  metadata?: Record<string, unknown>,
+): void {
+  const { writeFileSync } = fs;
+
+  const sortedEvents = events.sort((a, b) => a.ts - b.ts);
+  const first = sortedEvents[0];
+  const last = sortedEvents[sortedEvents.length - 1];
+
+  // Use performance.now() as fallback when no events exist
+  const fallbackTs = performance.now();
+  const firstTs = first?.ts ?? fallbackTs;
+  const lastTs = last?.ts ?? fallbackTs;
+
+  // Add margins for readability
+  const tsMargin = 1000;
+  const startTs = firstTs - tsMargin;
+  const endTs = lastTs + tsMargin;
+  const startDate = new Date().toISOString();
+
+  const traceEventsJson = [
+    // Preamble
+    encodeTraceEvent(
+      getStartTracing({
+        ts: startTs,
+        url: outputPath,
+      }),
+    ),
+    encodeTraceEvent(
+      getCompleteEvent({
+        ts: startTs,
+        dur: 20,
+      }),
+    ),
+    // Events
+    ...events.map(encodeTraceEvent),
+    encodeTraceEvent(
+      getCompleteEvent({
+        ts: endTs,
+        dur: 20,
+      }),
+    ),
+  ].join(',\n');
+
+  const jsonOutput = createTraceFileContent(
+    traceEventsJson,
+    new Date(),
+    metadata,
+  );
+  writeFileSync(outputPath, jsonOutput, 'utf8');
+}
+
+export interface TraceFileSinkOptions {
+  filename: string;
+  directory?: string;
+  metadata?: Record<string, unknown>;
+}
+
+export class TraceFileSink extends JsonlFileSink<SpanEvent | InstantEvent> {
+  readonly #filePath: string;
+  readonly #getFilePathForExt: (ext: 'json' | 'jsonl') => string;
+  readonly #metadata: Record<string, unknown> | undefined;
+
+  constructor(opts: TraceFileSinkOptions) {
+    const { filename, directory = '.', metadata } = opts;
+
+    const traceJsonlPath = path.join(directory, `${filename}.jsonl`);
+
+    super({
+      filePath: traceJsonlPath,
+      recover: () => recoverJsonlFile<SpanEvent | InstantEvent>(traceJsonlPath),
+    });
+
+    this.#metadata = metadata;
+    this.#filePath = path.join(directory, `${filename}.json`);
+    this.#getFilePathForExt = (ext: 'json' | 'jsonl') =>
+      path.join(directory, `${filename}.${ext}`);
+  }
+
+  override finalize(): void {
+    finalizeTraceFile(this.recover().records, this.#filePath, this.#metadata);
+  }
+
+  getFilePathForExt(ext: 'json' | 'jsonl'): string {
+    return this.#getFilePathForExt(ext);
+  }
+}

packages/utils/src/lib/file-sink-jsonl.int.test.ts

@@ -0,0 +1,138 @@
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { teardownTestFolder } from '@code-pushup/test-utils';
+import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js';
+
+describe('JsonlFileSink integration', () => {
+  const baseDir = path.join(os.tmpdir(), 'file-sink-json-int-tests');
+  const testFile = path.join(baseDir, 'test-data.jsonl');
+
+  beforeAll(async () => {
+    await fs.promises.mkdir(baseDir, { recursive: true });
+  });
+
+  beforeEach(async () => {
+    try {
+      await fs.promises.unlink(testFile);
+    } catch {
+      // File doesn't exist, which is fine
+    }
+  });
+
+  afterAll(async () => {
+    await teardownTestFolder(baseDir);
+  });
+
+  describe('file operations', () => {
+    const testData = [
+      { id: 1, name: 'Alice', active: true },
+      { id: 2, name: 'Bob', active: false },
+      { id: 3, name: 'Charlie', active: true },
+    ];
+
+    it('should write and read JSONL files', async () => {
+      const sink = new JsonlFileSink({ filePath: testFile });
+
+      // Open and write data
+      sink.open();
+      testData.forEach(item => sink.write(item));
+      sink.close();
+
+      expect(fs.existsSync(testFile)).toBe(true);
+      const fileContent = fs.readFileSync(testFile, 'utf8');
+      const lines = fileContent.trim().split('\n');
+      expect(lines).toStrictEqual([
+        '{"id":1,"name":"Alice","active":true}',
+        '{"id":2,"name":"Bob","active":false}',
+        '{"id":3,"name":"Charlie","active":true}',
+      ]);
+
+      lines.forEach((line, index) => {
+        const parsed = JSON.parse(line);
+        expect(parsed).toStrictEqual(testData[index]);
+      });
+    });
+
+    it('should recover data from JSONL files', async () => {
+      const jsonlContent = `${testData.map(item => JSON.stringify(item)).join('\n')}\n`;
+      fs.writeFileSync(testFile, jsonlContent);
+
+      expect(recoverJsonlFile(testFile)).toStrictEqual({
+        records: testData,
+        errors: [],
+        partialTail: null,
+      });
+    });
+
+    it('should handle JSONL files with parse errors', async () => {
+      const mixedContent =
+        '{"id":1,"name":"Alice"}\n' +
+        'invalid json line\n' +
+        '{"id":2,"name":"Bob"}\n' +
+        '{"id":3,"name":"Charlie","incomplete":\n';
+
+      fs.writeFileSync(testFile, mixedContent);
+
+      expect(recoverJsonlFile(testFile)).toStrictEqual({
+        records: [
+          { id: 1, name: 'Alice' },
+          { id: 2, name: 'Bob' },
+        ],
+        errors: [
+          expect.objectContaining({ line: 'invalid json line' }),
+          expect.objectContaining({
+            line: '{"id":3,"name":"Charlie","incomplete":',
+          }),
+        ],
+        partialTail: '{"id":3,"name":"Charlie","incomplete":',
+      });
+    });
+
+    it('should recover data using JsonlFileSink.recover()', async () => {
+      const sink = new JsonlFileSink({ filePath: testFile });
+      sink.open();
+      testData.forEach(item => sink.write(item));
+      sink.close();
+
+      expect(sink.recover()).toStrictEqual({
+        records: testData,
+        errors: [],
+        partialTail: null,
+      });
+    });
+
+    describe('edge cases', () => {
+      it('should handle empty files', async () => {
+        fs.writeFileSync(testFile, '');
+
+        expect(recoverJsonlFile(testFile)).toStrictEqual({
+          records: [],
+          errors: [],
+          partialTail: null,
+        });
+      });
+
+      it('should handle files with only whitespace', async () => {
+        fs.writeFileSync(testFile, '   \n  \n\t\n');
+
+        expect(recoverJsonlFile(testFile)).toStrictEqual({
+          records: [],
+          errors: [],
+          partialTail: null,
+        });
+      });
+
+      it('should handle non-existent files', async () => {
+        const nonExistentFile = path.join(baseDir, 'does-not-exist.jsonl');
+
+        expect(recoverJsonlFile(nonExistentFile)).toStrictEqual({
+          records: [],
+          errors: [],
+          partialTail: null,
+        });
+      });
+    });
+  });
+});

packages/utils/src/lib/file-sink-jsonl.ts

@@ -0,0 +1,60 @@
+import * as fs from 'node:fs';
+import {
+  type FileOutput,
+  FileSink,
+  type FileSinkOptions,
+  stringDecode,
+  stringEncode,
+  stringRecover,
+} from './file-sink-text.js';
+import type { RecoverOptions, RecoverResult } from './sink-source.types.js';
+
+export const jsonlEncode = <
+  T extends Record<string, unknown> = Record<string, unknown>,
+>(
+  input: T,
+): FileOutput => JSON.stringify(input);
+
+export const jsonlDecode = <
+  T extends Record<string, unknown> = Record<string, unknown>,
+>(
+  output: FileOutput,
+): T => JSON.parse(stringDecode(output)) as T;
+
+export function recoverJsonlFile<
+  T extends Record<string, unknown> = Record<string, unknown>,
+>(filePath: string, opts: RecoverOptions = {}): RecoverResult<T> {
+  return stringRecover(filePath, jsonlDecode<T>, opts);
+}
+
+export class JsonlFileSink<
+  T extends Record<string, unknown> = Record<string, unknown>,
+> extends FileSink<T> {
+  constructor(options: FileSinkOptions) {
+    const { filePath, ...fileOptions } = options;
+    super({
+      ...fileOptions,
+      filePath,
+      recover: () => recoverJsonlFile<T>(filePath),
+      finalize: () => {
+        // No additional finalization needed for JSONL files
+      },
+    });
+  }
+
+  override encode(input: T): FileOutput {
+    return stringEncode(jsonlEncode(input));
+  }
+
+  override decode(output: FileOutput): T {
+    return jsonlDecode(stringDecode(output));
+  }
+
+  override repack(outputPath?: string): void {
+    const { records } = this.recover();
+    fs.writeFileSync(
+      outputPath ?? this.getFilePath(),
+      records.map(this.encode).join(''),
+    );
+  }
+}