diff --git a/crates/next-napi-bindings/src/next_api/project.rs b/crates/next-napi-bindings/src/next_api/project.rs index 1f4ec6fcb81898..9fd58ba767e415 100644 --- a/crates/next-napi-bindings/src/next_api/project.rs +++ b/crates/next-napi-bindings/src/next_api/project.rs @@ -2152,6 +2152,16 @@ pub fn project_compilation_events_subscribe( break; } } + // Signal the JS side that the subscription has ended (e.g. after + // project shutdown drops all senders). This allows the async + // iterator to exit promptly instead of hanging forever. + let _ = tsfn.call( + Err(napi::Error::new( + Status::Cancelled, + "compilation events subscription closed", + )), + ThreadsafeFunctionCallMode::Blocking, + ); }); Ok(()) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 842e141a78ceeb..42c071e4eff326 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1009,16 +1009,6 @@ export default async function build( // Reading the config can modify environment variables that influence the bundler selection. bundler = finalizeBundlerFromConfig(bundler) nextBuildSpan.setAttribute('bundler', getBundlerForTelemetry(bundler)) - // Install the native bindings early so we can have synchronous access later. - await installBindings(config.experimental?.useWasmBinary) - - // Set up code frame renderer for error formatting - const { installCodeFrameSupport } = - require('../server/lib/install-code-frame') as typeof import('../server/lib/install-code-frame') - installCodeFrameSupport() - - process.env.NEXT_DEPLOYMENT_ID = config.deploymentId || '' - NextBuildContext.config = config let configOutDir = 'out' if (hasCustomExportOutput(config)) { @@ -1030,6 +1020,26 @@ export default async function build( setGlobal('phase', PHASE_PRODUCTION_BUILD) setGlobal('distDir', distDir) + // Check for build cache before initializing telemetry, because the + // Telemetry constructor creates the cache directory in CI environments. + const cacheDir = getCacheDir(distDir) + + // Initialize telemetry before installBindings so that SWC load failure + // events are captured if native bindings fail to load. + const telemetry = new Telemetry({ distDir }) + setGlobal('telemetry', telemetry) + + // Install the native bindings early so we can have synchronous access later. + await installBindings(config.experimental?.useWasmBinary) + + // Set up code frame renderer for error formatting + const { installCodeFrameSupport } = + require('../server/lib/install-code-frame') as typeof import('../server/lib/install-code-frame') + installCodeFrameSupport() + + process.env.NEXT_DEPLOYMENT_ID = config.deploymentId || '' + NextBuildContext.config = config + const buildId = await getBuildId( isGenerateMode, distDir, @@ -1117,12 +1127,6 @@ export default async function build( ) } - const cacheDir = getCacheDir(distDir) - - const telemetry = new Telemetry({ distDir }) - - setGlobal('telemetry', telemetry) - const publicDir = path.join(dir, 'public') const { pagesDir, appDir } = findPagesDir(dir) diff --git a/packages/next/src/build/turbopack-build/impl.ts b/packages/next/src/build/turbopack-build/impl.ts index 9fb86d566ec34a..20ccd34cd09704 100644 --- a/packages/next/src/build/turbopack-build/impl.ts +++ b/packages/next/src/build/turbopack-build/impl.ts @@ -146,15 +146,17 @@ export async function turbopackBuild(): Promise<{ } : undefined ) - try { - const buildEventsSpan = trace('turbopack-build-events') - // Stop immediately: this span is only used as a parent for - // manualTraceChild calls which carry their own timestamps. - buildEventsSpan.stop() - backgroundLogCompilationEvents(project, { - parentSpan: buildEventsSpan, - }) + const buildEventsSpan = trace('turbopack-build-events') + // Stop immediately: this span is only used as a parent for + // manualTraceChild calls which carry their own timestamps. + buildEventsSpan.stop() + const shutdownController = new AbortController() + const compilationEvents = backgroundLogCompilationEvents(project, { + parentSpan: buildEventsSpan, + signal: shutdownController.signal, + }) + try { // Write an empty file in a known location to signal this was built with Turbopack await fs.writeFile(path.join(distDir, 'turbopack'), '') @@ -264,7 +266,14 @@ export async function turbopackBuild(): Promise<{ await project.writeAnalyzeData(appDirOnly) } - const shutdownPromise = project.shutdown() + // Shutdown may trigger final compilation events (e.g. persistence, + // compaction trace spans). This is the last chance to capture them. + // After shutdown resolves we abort the signal to close the iterator + // and drain any remaining buffered events. + const shutdownPromise = project.shutdown().then(() => { + shutdownController.abort() + return compilationEvents.catch(() => {}) + }) const time = process.hrtime(startTime) return { @@ -274,6 +283,8 @@ export async function turbopackBuild(): Promise<{ } } catch (err) { await project.shutdown() + shutdownController.abort() + await compilationEvents.catch(() => {}) throw err } } @@ -283,9 +294,7 @@ export async function workerMain(workerData: { buildContext: typeof NextBuildContext traceState: TraceState & { shouldSaveTraceEvents: boolean } }): Promise< - Omit>, 'shutdownPromise'> & { - debugTraceEvents?: ReturnType - } + Omit>, 'shutdownPromise'> > { // setup new build context from the serialized data passed from the parent Object.assign(NextBuildContext, workerData.buildContext) @@ -324,14 +333,9 @@ export async function workerMain(workerData: { duration, } = await turbopackBuild() shutdownPromise = resultShutdownPromise - // Wait for shutdown to complete so that all compilation events - // (e.g. persistence trace spans) have been processed before we - // collect the saved trace events. - await shutdownPromise return { buildTraceContext, duration, - debugTraceEvents: getTraceEvents(), } } finally { // Always flush telemetry before worker exits (waits for async operations like setTimeout in debug mode) @@ -341,8 +345,13 @@ export async function workerMain(workerData: { } } -export async function waitForShutdown(): Promise { +export async function waitForShutdown(): Promise<{ + debugTraceEvents?: ReturnType +}> { if (shutdownPromise) { await shutdownPromise } + // Collect trace events after shutdown completes so that all compilation + // events (e.g. persistence trace spans) have been processed. + return { debugTraceEvents: getTraceEvents() } } diff --git a/packages/next/src/build/turbopack-build/index.ts b/packages/next/src/build/turbopack-build/index.ts index d1479ef30447cc..1f26a9c41bd81f 100644 --- a/packages/next/src/build/turbopack-build/index.ts +++ b/packages/next/src/build/turbopack-build/index.ts @@ -35,24 +35,22 @@ async function turbopackBuildWithWorker(): ReturnType< config: _config, ...prunedBuildContext } = NextBuildContext - const { buildTraceContext, duration, debugTraceEvents } = - await worker.workerMain({ - buildContext: prunedBuildContext, - traceState: { - ...exportTraceState(), - defaultParentSpanId: nextBuildSpan.getId(), - shouldSaveTraceEvents: true, - }, - }) - - if (debugTraceEvents) { - recordTraceEvents(debugTraceEvents) - } + const { buildTraceContext, duration } = await worker.workerMain({ + buildContext: prunedBuildContext, + traceState: { + ...exportTraceState(), + defaultParentSpanId: nextBuildSpan.getId(), + shouldSaveTraceEvents: true, + }, + }) return { // destroy worker when Turbopack has shutdown so it's not sticking around using memory // We need to wait for shutdown to make sure filesystem cache is flushed - shutdownPromise: worker.waitForShutdown().then(() => { + shutdownPromise: worker.waitForShutdown().then(({ debugTraceEvents }) => { + if (debugTraceEvents) { + recordTraceEvents(debugTraceEvents) + } worker.end() }), buildTraceContext, diff --git a/packages/next/src/shared/lib/turbopack/compilation-events.ts b/packages/next/src/shared/lib/turbopack/compilation-events.ts index 264967ea559f73..0f171333663c57 100644 --- a/packages/next/src/shared/lib/turbopack/compilation-events.ts +++ b/packages/next/src/shared/lib/turbopack/compilation-events.ts @@ -15,8 +15,10 @@ export function msToNs(ms: number): bigint { * When `parentSpan` is provided, `TraceEvent` compilation events are recorded * as trace spans in the `.next/trace` file. * - * The `signal` argument is partially implemented. The abort may not happen until the next - * compilation event arrives. + * Returns a promise that resolves when the subscription ends. Abort the + * `signal` to close the underlying async iterator and settle the promise + * promptly. The iterator also closes automatically when the Rust side + * drops the subscription (e.g. after project shutdown). */ export function backgroundLogCompilationEvents( project: Project, @@ -26,12 +28,16 @@ export function backgroundLogCompilationEvents( parentSpan, }: { eventTypes?: string[]; signal?: AbortSignal; parentSpan?: Span } = {} ): Promise { - const promise = (async function () { - for await (const event of project.compilationEventsSubscribe(eventTypes)) { - if (signal?.aborted) { - return - } + const iterator = project.compilationEventsSubscribe(eventTypes) + // Close the iterator as soon as the signal fires so the for-await loop + // exits without waiting for the next compilation event. + signal?.addEventListener('abort', () => iterator.return?.(undefined as any), { + once: true, + }) + + const promise = (async function () { + for await (const event of iterator) { // Record TraceEvent compilation events as trace spans in .next/trace. if (parentSpan && event.typeName === 'TraceEvent' && event.eventJson) { try { diff --git a/packages/next/src/trace/report/to-json-build.ts b/packages/next/src/trace/report/to-json-build.ts index 08c7142d00ec88..dfa16b0cc6c681 100644 --- a/packages/next/src/trace/report/to-json-build.ts +++ b/packages/next/src/trace/report/to-json-build.ts @@ -15,6 +15,9 @@ const allowlistedEvents = new Set([ 'adapter-handle-build-complete', 'output-standalone', 'telemetry-flush', + 'turbopack-build-events', + 'turbopack-persistence', + 'turbopack-compaction', ]) export default createJsonReporter({ diff --git a/test/development/app-dir/enabled-features-trace/enabled-features-trace.test.ts b/test/development/app-dir/enabled-features-trace/enabled-features-trace.test.ts index 1d1c45e1068dc2..b94a17b58397d8 100644 --- a/test/development/app-dir/enabled-features-trace/enabled-features-trace.test.ts +++ b/test/development/app-dir/enabled-features-trace/enabled-features-trace.test.ts @@ -3,46 +3,7 @@ import { join } from 'path' import { existsSync, readFileSync } from 'fs' import { createServer } from 'http' import { spawn } from 'child_process' -import type { TraceEvent } from 'next/dist/trace' - -interface TraceStructure { - events: TraceEvent[] - eventsByName: Map - eventsById: Map -} - -function parseTraceFile(tracePath: string): TraceStructure { - const traceContent = readFileSync(tracePath, 'utf8') - const traceLines = traceContent - .trim() - .split('\n') - .filter((line) => line.trim()) - - const allEvents: TraceEvent[] = [] - - for (const line of traceLines) { - const events = JSON.parse(line) as TraceEvent[] - allEvents.push(...events) - } - - const eventsByName = new Map() - const eventsById = new Map() - - // Index all events - for (const event of allEvents) { - if (!eventsByName.has(event.name)) { - eventsByName.set(event.name, []) - } - eventsByName.get(event.name)!.push(event) - eventsById.set(event.id.toString(), event) - } - - return { - events: allEvents, - eventsByName, - eventsById, - } -} +import { parseTraceFile } from '../../../lib/parse-trace-file' describe('enabled features in trace', () => { const { next, isNextDev } = nextTestSetup({ diff --git a/test/development/app-dir/hmr-trace-timing/hmr-trace-timing.test.ts b/test/development/app-dir/hmr-trace-timing/hmr-trace-timing.test.ts index feac7cad98d4d7..21c76d5bd0241d 100644 --- a/test/development/app-dir/hmr-trace-timing/hmr-trace-timing.test.ts +++ b/test/development/app-dir/hmr-trace-timing/hmr-trace-timing.test.ts @@ -1,46 +1,7 @@ import { nextTestSetup } from 'e2e-utils' import { join } from 'path' -import { existsSync, readFileSync } from 'fs' -import type { TraceEvent } from 'next/dist/trace' - -interface TraceStructure { - events: TraceEvent[] - eventsByName: Map - eventsById: Map -} - -function parseTraceFile(tracePath: string): TraceStructure { - const traceContent = readFileSync(tracePath, 'utf8') - const traceLines = traceContent - .trim() - .split('\n') - .filter((line) => line.trim()) - - const allEvents: TraceEvent[] = [] - - for (const line of traceLines) { - const events = JSON.parse(line) as TraceEvent[] - allEvents.push(...events) - } - - const eventsByName = new Map() - const eventsById = new Map() - - // Index all events - for (const event of allEvents) { - if (!eventsByName.has(event.name)) { - eventsByName.set(event.name, []) - } - eventsByName.get(event.name)!.push(event) - eventsById.set(event.id.toString(), event) - } - - return { - events: allEvents, - eventsByName, - eventsById, - } -} +import { existsSync } from 'fs' +import { parseTraceFile } from '../../../lib/parse-trace-file' describe('render-path tracing', () => { const { next, isNextDev } = nextTestSetup({ diff --git a/test/e2e/app-dir/trace-build-file/next.config.js b/test/e2e/app-dir/trace-build-file/next.config.js index 807126e4cf0bf5..570fc70d912f55 100644 --- a/test/e2e/app-dir/trace-build-file/next.config.js +++ b/test/e2e/app-dir/trace-build-file/next.config.js @@ -1,6 +1,10 @@ /** * @type {import('next').NextConfig} */ -const nextConfig = {} +const nextConfig = { + experimental: { + turbopackFileSystemCacheForBuild: true, + }, +} module.exports = nextConfig diff --git a/test/e2e/app-dir/trace-build-file/trace-build-file.test.ts b/test/e2e/app-dir/trace-build-file/trace-build-file.test.ts index d65bd283d885ad..6edc50892e5654 100644 --- a/test/e2e/app-dir/trace-build-file/trace-build-file.test.ts +++ b/test/e2e/app-dir/trace-build-file/trace-build-file.test.ts @@ -1,67 +1,20 @@ import { nextTestSetup, isNextDev, isNextStart } from 'e2e-utils' import { join } from 'path' -import { existsSync, readFileSync } from 'fs' -import type { TraceEvent } from 'next/dist/trace' - -interface TraceStructure { - events: TraceEvent[] - eventsByName: Map - eventsById: Map - rootEvents: TraceEvent[] - orphanedEvents: TraceEvent[] -} - -function parseTraceFile(traceBuildPath: string): TraceStructure { - const traceContent = readFileSync(traceBuildPath, 'utf8') - const traceLines = traceContent - .trim() - .split('\n') - .filter((line) => line.trim()) - - const allEvents: TraceEvent[] = [] - - for (const line of traceLines) { - const events = JSON.parse(line) as TraceEvent[] - allEvents.push(...events) - } - - const eventsByName = new Map() - const eventsById = new Map() - const rootEvents: TraceEvent[] = [] - const orphanedEvents: TraceEvent[] = [] - - // Index all events - for (const event of allEvents) { - if (!eventsByName.has(event.name)) { - eventsByName.set(event.name, []) - } - eventsByName.get(event.name).push(event) - eventsById.set(event.id.toString(), event) - } - - // Categorize events as root or orphaned - for (const event of allEvents) { - if (!event.parentId) { - rootEvents.push(event) - } else if (!eventsById.has(event.parentId.toString())) { - orphanedEvents.push(event) - } - } - - return { - events: allEvents, - eventsByName, - eventsById, - rootEvents, - orphanedEvents, - } -} +import { existsSync } from 'fs' +import { parseTraceFile } from '../../../lib/parse-trace-file' describe('trace-build-file', () => { const { next } = nextTestSetup({ files: __dirname, skipStart: !isNextDev, skipDeployment: true, + env: { + // Enable persistent caching even when the git working directory is + // dirty (e.g. when developing Next.js itself). Without this, the + // cache falls back to a temp directory and persistence/compaction + // spans are not emitted. + TURBO_ENGINE_IGNORE_DIRTY: '1', + }, }) if (isNextStart) { @@ -133,6 +86,8 @@ describe('trace-build-file', () => { "static-check", "static-generation", "telemetry-flush", + "turbopack-build-events", + "turbopack-persistence", ] `) } else { diff --git a/test/e2e/filesystem-cache/filesystem-cache.test.ts b/test/e2e/filesystem-cache/filesystem-cache.test.ts index b94f0a1a383dcc..dece69e8102983 100644 --- a/test/e2e/filesystem-cache/filesystem-cache.test.ts +++ b/test/e2e/filesystem-cache/filesystem-cache.test.ts @@ -2,24 +2,9 @@ import { nextTestSetup, isNextDev } from 'e2e-utils' import { waitFor } from 'next-test-utils' import fs from 'fs/promises' -import { readFileSync, existsSync } from 'fs' +import { existsSync } from 'fs' import path from 'path' -import type { TraceEvent } from 'next/dist/trace' - -function parseTraceFile(tracePath: string): TraceEvent[] { - const traceContent = readFileSync(tracePath, 'utf8') - const traceLines = traceContent - .trim() - .split('\n') - .filter((line) => line.trim()) - - const allEvents: TraceEvent[] = [] - for (const line of traceLines) { - const events = JSON.parse(line) as TraceEvent[] - allEvents.push(...events) - } - return allEvents -} +import { parseTraceEvents } from '../../lib/parse-trace-file' async function getDirectorySize(dirPath: string): Promise { try { @@ -440,7 +425,7 @@ for (const cacheEnabled of [false, true]) { const tracePath = path.join(next.testDir, traceDir, 'trace') expect(existsSync(tracePath)).toBe(true) - const events = parseTraceFile(tracePath) + const events = parseTraceEvents(tracePath) const persistenceEvents = events.filter( (e) => e.name === 'turbopack-persistence' ) diff --git a/test/lib/parse-trace-file.ts b/test/lib/parse-trace-file.ts new file mode 100644 index 00000000000000..8feabf3229c2ca --- /dev/null +++ b/test/lib/parse-trace-file.ts @@ -0,0 +1,63 @@ +import { readFileSync } from 'fs' +import type { TraceEvent } from 'next/dist/trace' + +export interface TraceStructure { + events: TraceEvent[] + eventsByName: Map + eventsById: Map + rootEvents: TraceEvent[] + orphanedEvents: TraceEvent[] +} + +/** + * Parses a Next.js trace file (e.g. `.next/trace` or `.next/trace-build`) + * and returns the flat list of trace events. + */ +export function parseTraceEvents(tracePath: string): TraceEvent[] { + const traceContent = readFileSync(tracePath, 'utf8') + const allEvents: TraceEvent[] = [] + for (const line of traceContent.trim().split('\n')) { + if (!line.trim()) continue + allEvents.push(...(JSON.parse(line) as TraceEvent[])) + } + return allEvents +} + +/** + * Parses a Next.js trace file and returns a structured representation + * with events indexed by name and id, plus root/orphaned classification. + */ +export function parseTraceFile(tracePath: string): TraceStructure { + const allEvents = parseTraceEvents(tracePath) + + const eventsByName = new Map() + const eventsById = new Map() + const rootEvents: TraceEvent[] = [] + const orphanedEvents: TraceEvent[] = [] + + for (const event of allEvents) { + const byName = eventsByName.get(event.name) + if (byName) { + byName.push(event) + } else { + eventsByName.set(event.name, [event]) + } + eventsById.set(event.id.toString(), event) + } + + for (const event of allEvents) { + if (!event.parentId) { + rootEvents.push(event) + } else if (!eventsById.has(event.parentId.toString())) { + orphanedEvents.push(event) + } + } + + return { + events: allEvents, + eventsByName, + eventsById, + rootEvents, + orphanedEvents, + } +} diff --git a/test/production/build-failed-trace/build-failed-trace.test.ts b/test/production/build-failed-trace/build-failed-trace.test.ts index d55a56a6d7ed7e..a51ca9414a6ea8 100644 --- a/test/production/build-failed-trace/build-failed-trace.test.ts +++ b/test/production/build-failed-trace/build-failed-trace.test.ts @@ -1,16 +1,6 @@ import { nextTestSetup, isNextStart } from 'e2e-utils' import { join } from 'path' -import { readFileSync } from 'fs' -import type { TraceEvent } from 'next/dist/trace' - -function parseTraceFile(tracePath: string): TraceEvent[] { - const content = readFileSync(tracePath, 'utf8') - const events: TraceEvent[] = [] - for (const line of content.trim().split('\n').filter(Boolean)) { - events.push(...(JSON.parse(line) as TraceEvent[])) - } - return events -} +import { parseTraceEvents } from '../../lib/parse-trace-file' describe('build-failed-trace', () => { if (!isNextStart) { @@ -28,7 +18,7 @@ describe('build-failed-trace', () => { expect(exitCode).not.toBe(0) const tracePath = join(next.testDir, '.next', 'trace') - const events = parseTraceFile(tracePath) + const events = parseTraceEvents(tracePath) const nextBuildEvent = events.find((e) => e.name === 'next-build') expect(nextBuildEvent).toBeDefined()