Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 77 additions & 10 deletions app/lib/services/preview_generator.dart
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,27 @@ import '../models/processing_pipeline.dart';
import '../models/video_job.dart';
import 'field_order_detector.dart';
import 'temp_directory_service.dart';
import 'process_tree.dart';
import 'tool_locator.dart';

/// Service for generating video thumbnails and processed previews.
class PreviewGenerator {
Process? _thumbnailProcess;
Process? _previewProcess;

/// Every preview worker still running, including ones already superseded.
///
/// `_previewProcess` alone is not enough to guarantee cleanup. It is assigned
/// *after* `await Process.start(...)` returns, so a seek that arrives inside
/// that window cancels whatever the field held at the time — not the process
/// currently being spawned — and the next assignment then overwrites the
/// reference to it. That process is never tracked and never killed. Scrubbing
/// the timeline cancels a preview on every move, so those strays accumulate,
/// which is what "a bunch of vspipe and ffmpeg processes" looks like.
///
/// Registering at spawn and discarding on exit closes that window: a process
/// is reachable for cancellation from the moment it exists.
final Set<Process> _livePreviews = {};
String? _ffmpegPath;
String? _ffprobePath;
String? _workerPath;
Expand Down Expand Up @@ -283,10 +298,14 @@ class PreviewGenerator {
workingDirectory: path.dirname(_workerPath!),
);
_previewProcess = process;
_livePreviews.add(process);

if (cancelToken?.isCancelled ?? false) {
process.kill();
// Whole group: killing the worker alone strands vspipe/ffmpeg.
ProcessTree.killTree(process);
_previewProcess = null;
_livePreviews.remove(process);
await ProcessTree.waitForExit(process);
return null;
}

Expand All @@ -309,8 +328,12 @@ class PreviewGenerator {

await for (final chunk in process.stdout) {
if (cancelToken?.isCancelled ?? false) {
process.kill();
// Whole group: killing the worker alone strands vspipe/ffmpeg.
ProcessTree.killTree(process);
_previewProcess = null;
_livePreviews.remove(process);
_livePreviews.remove(process);
await ProcessTree.waitForExit(process);
return null;
}
pngBytes.addAll(chunk);
Expand All @@ -323,6 +346,7 @@ class PreviewGenerator {
final exitCode = await process.exitCode;
if (_previewProcess == process) {
_previewProcess = null;
_livePreviews.remove(process);
}

// Log the result
Expand All @@ -347,6 +371,7 @@ class PreviewGenerator {
} finally {
if (_previewProcess == process) {
_previewProcess = null;
_livePreviews.remove(process);
}
// Clean up config file on error
try {
Expand All @@ -358,20 +383,62 @@ class PreviewGenerator {
}

/// Cancel any ongoing preview generation.
///
/// Seeking the scrubber cancels the in-flight preview on every move, so this
/// runs far more often than a job cancel does — and anything it fails to clean
/// up accumulates. Killing the worker's pid alone left `vspipe` and `ffmpeg`
/// running: preview mode installs no signal handler (it returns before ctrlc
/// is set up in `worker/src/main.rs`), so SIGTERM kills the worker outright
/// without unwinding, and `generate_preview` holds its children in locals that
/// `PipelineExecutor::terminate()` never sees. Nothing killed them
/// deliberately; they just tended to die on EPIPE once their pipes closed,
/// which does not happen while they are blocked reading a slow source.
///
/// Signal the whole process group instead, and wait for the worker to actually
/// go — a fire-and-forget kill cannot tell a clean shutdown from a stray.
/// Signals every live preview and returns as soon as they have been told to
/// stop — it does NOT wait for them to exit.
///
/// A seek calls this before starting the next preview, so blocking here would
/// add the full shutdown grace to every scrubber movement and make seeking
/// feel broken. Signalling is immediate and ordered; reaping is not, so it
/// runs unawaited. Use [awaitPreviewShutdown] when the wait actually matters.
Future<void> cancelPreviewGeneration() async {
if (_previewProcess != null) {
_previewProcess!.kill();
_previewProcess = null;
}
if (_thumbnailProcess != null) {
_thumbnailProcess!.kill();
_thumbnailProcess = null;
final doomed = <Process>{
..._livePreviews,
if (_previewProcess != null) _previewProcess!,
if (_thumbnailProcess != null) _thumbnailProcess!,
};
_livePreviews.clear();
_previewProcess = null;
_thumbnailProcess = null;

for (final p in doomed) {
// The whole group: signalling the worker alone strands vspipe/ffmpeg,
// and preview mode has no handler that would clean up after itself.
ProcessTree.killTree(p);
// Reap in the background so a slow shutdown cannot stall the next seek.
unawaited(ProcessTree.waitForExit(p));
}
}

/// Cancel and wait for everything to actually be gone.
///
/// For shutdown, where leaving strays behind is worse than a brief pause.
Future<void> awaitPreviewShutdown() async {
final doomed = <Process>{
..._livePreviews,
if (_previewProcess != null) _previewProcess!,
if (_thumbnailProcess != null) _thumbnailProcess!,
};
await cancelPreviewGeneration();
await Future.wait(doomed.map(ProcessTree.waitForExit));
}

/// Clean up resources.
Future<void> dispose() async {
await cancelPreviewGeneration();
// Shutdown is the one place the wait is worth it: strays outlive the app.
await awaitPreviewShutdown();
_thumbnailCache.clear();

// Clean up temp directory
Expand Down
66 changes: 66 additions & 0 deletions app/lib/services/process_tree.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import 'dart:io';

/// Terminating a worker without leaving its pipeline behind.
///
/// Signalling the worker's pid alone does not stop `vspipe` and `ffmpeg`. They
/// usually die shortly afterwards, but only incidentally — their pipes close
/// with the worker and they take EPIPE the next time they write. A child blocked
/// on slow input (a source on a network share is the reported case) writes
/// nothing for minutes, never notices, and is left running at full CPU on work
/// nobody is waiting for. Preview seeking makes this worse than it sounds: every
/// scrub cancels an in-flight preview, so the strays accumulate.
///
/// The worker puts itself in its own process group at startup (`setpgid` in
/// `worker/src/main.rs`), so its whole tree can be signalled at once by sending
/// to the negated pid. That is a deliberate teardown rather than a hopeful one.
class ProcessTree {
/// Signal [process] and everything it spawned.
///
/// Falls back to signalling the process alone when the group cannot be
/// reached — an older worker that predates the `setpgid` call, or a platform
/// without process groups. That fallback is exactly the previous behaviour, so
/// this is never worse than what it replaced.
///
/// Returns true if the group signal landed.
static bool killTree(Process process, [ProcessSignal signal = ProcessSignal.sigterm]) {
if (Platform.isWindows) {
// No process groups; taskkill /T walks the tree instead. Callers on
// Windows use that directly.
return process.kill(signal);
}
// A negative pid addresses the process group. Dart forwards this to kill(2)
// unchanged, and kill(2) defines negative pids as group targets.
try {
if (Process.killPid(-process.pid, signal)) return true;
} on Object {
// Fall through — some platforms reject a negative pid outright.
}
process.kill(signal);
return false;
}

/// Wait for [process] to exit, escalating to SIGKILL if it outstays [grace].
///
/// Returns true if it exited without needing to be forced.
static Future<bool> waitForExit(
Process process, {
Duration grace = const Duration(seconds: 5),
Duration forceGrace = const Duration(seconds: 3),
}) async {
try {
await process.exitCode.timeout(grace);
return true;
} on Object {
// Still alive. Forcing it here can orphan the children, which is the very
// thing this class exists to avoid — so force the whole group, not just
// the leader.
killTree(process, ProcessSignal.sigkill);
try {
await process.exitCode.timeout(forceGrace);
} on Object {
// Nothing further we can do from here.
}
return false;
}
}
}
9 changes: 7 additions & 2 deletions app/lib/services/worker_manager.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import 'dart:io';

import '../models/progress_info.dart';
import '../models/video_job.dart';
import 'process_tree.dart';
import 'temp_directory_service.dart';
import 'tool_locator.dart';

Expand Down Expand Up @@ -194,7 +195,11 @@ class WorkerManager {
// orphaned; /F is unavoidable there.
await Process.run('taskkill', ['/PID', '${process.pid}', '/T', '/F']);
} else {
process.kill(ProcessSignal.sigterm);
// Signal the whole process group. The worker still tears its own pipeline
// down when it gets the chance, but that only covers the children it
// tracks, and it cannot run at all if it is forced below — so do not rely
// on it alone. See ProcessTree.
ProcessTree.killTree(process);
}

// Wait for the process to actually exit. `exitCode` completes once it has
Expand All @@ -210,7 +215,7 @@ class WorkerManager {
// Genuinely wedged. Forcing it here orphans the children — the same
// failure described above — but by now the alternative is a job that
// never stops at all, so take the lesser problem and say so.
process.kill(ProcessSignal.sigkill);
ProcessTree.killTree(process, ProcessSignal.sigkill);
try {
await process.exitCode.timeout(_forceKillGrace);
} on TimeoutException {
Expand Down
Loading
Loading