Skip to content

Commit 80be18a

Browse files
myieyehahn-kev
andauthored
Add ffmpeg wasm (#1822)
* Add ffmpeg wasm * setup audio pipeline for the audio editor to process and modify audio * abort file operations when the audio editor is destroyed * cleanup and teardown ffmpeg with the audio editor dialog * add a download button to avoid losing audio when it's too big --------- Co-authored-by: Kevin Hahn <kevin_hahn@sil.org>
1 parent 7c8749c commit 80be18a

15 files changed

Lines changed: 564 additions & 226 deletions

File tree

frontend/pnpm-lock.yaml

Lines changed: 35 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

frontend/viewer/package.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"scripts": {
1212
"dev": "vite",
1313
"build": "vite build",
14+
"build-ffmpeg-worker": "vite build --config vite.config.ffmpeg-worker.ts",
1415
"preview": "vite preview",
1516
"pretest:playwright": "playwright install",
1617
"test:playwright": "playwright test",
@@ -89,6 +90,9 @@
8990
"@microsoft/signalr": "^8.0.7",
9091
"autoprefixer": "^10.4.21",
9192
"fast-json-patch": "^3.1.1",
93+
"@ffmpeg/ffmpeg": "0.12.15",
94+
"@ffmpeg/util": "0.12.2",
95+
"@ffmpeg/core": "0.12.10",
9296
"jsdom": "^26.1.0",
9397
"just-throttle": "^4.2.0",
9498
"postcss": "catalog:",

frontend/viewer/src/lib/components/audio/AudioDialog.svelte

Lines changed: 16 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,8 @@
55
import {useDialogsService} from '$lib/services/dialogs-service.js';
66
import {useBackHandler} from '$lib/utils/back-handler.svelte';
77
import {watch} from 'runed';
8-
import {delay} from '$lib/utils/time';
98
import AudioProvider from './audio-provider.svelte';
109
import AudioEditor from './audio-editor.svelte';
11-
import Loading from '$lib/components/Loading.svelte';
1210
import {useLexboxApi} from '$lib/services/service-provider';
1311
import {UploadFileResult} from '$lib/dotnet-types/generated-types/MiniLcm/Media/UploadFileResult';
1412
import {AppNotification} from '$lib/notifications/notifications';
@@ -21,8 +19,8 @@
2119
2220
let submitting = $state(false);
2321
let selectedFile = $state<File>();
24-
let audio = $state<Blob>();
25-
const tooBig = $derived((audio?.size ?? 0) > 10 * 1024 * 1024);
22+
let finalAudio = $state<File>();
23+
const tooBig = $derived((finalAudio?.size ?? 0) > 10 * 1024 * 1024);
2624
2725
let requester: {
2826
resolve: (mediaUri: string | undefined) => void
@@ -41,6 +39,10 @@
4139
if (!open) reset();
4240
});
4341
42+
watch(() => selectedFile, () => {
43+
if (!selectedFile) finalAudio = undefined;
44+
})
45+
4446
function close() {
4547
open = false;
4648
reset();
@@ -53,12 +55,12 @@
5355
}
5456
5557
function clearAudio() {
56-
audio = selectedFile = undefined;
58+
selectedFile = undefined;
5759
submitting = false;
5860
}
5961
6062
async function submitAudio() {
61-
if (!audio) throw new Error('No audio to upload');
63+
if (!selectedFile) throw new Error('No audio to upload');
6264
if (!requester) throw new Error('No requester');
6365
6466
submitting = true;
@@ -72,8 +74,8 @@
7274
}
7375
7476
async function uploadAudio() {
75-
if (!audio || !selectedFile) throw new Error($t`No file selected`);
76-
const response = await lexboxApi.saveFile(audio, {filename: selectedFile.name, mimeType: audio.type});
77+
if (!finalAudio) throw new Error($t`No file to upload`);
78+
const response = await lexboxApi.saveFile(finalAudio, {filename: finalAudio.name, mimeType: finalAudio.type});
7779
switch (response.result) {
7880
case UploadFileResult.SavedLocally:
7981
AppNotification.display($t`Audio saved locally`, 'success');
@@ -94,16 +96,13 @@
9496
return response.mediaUri;
9597
}
9698
97-
async function onFileSelected(file: File) {
99+
function onFileSelected(file: File) {
98100
selectedFile = file;
99-
audio = await processAudio(file);
100101
}
101102
102-
async function onRecordingComplete(blob: Blob) {
103+
function onRecordingComplete(blob: Blob) {
103104
let fileExt = mimeTypeToFileExtension(blob.type);
104105
selectedFile = new File([blob], `recording-${Date.now()}.${fileExt}`, {type: blob.type});
105-
if (!open) return;
106-
audio = await processAudio(blob);
107106
}
108107
109108
function mimeTypeToFileExtension(mimeType: string) {
@@ -133,17 +132,8 @@
133132
}
134133
135134
function onDiscard() {
136-
audio = undefined;
137135
selectedFile = undefined;
138136
}
139-
140-
let loading = $state(false);
141-
async function processAudio(blob: Blob): Promise<Blob> {
142-
loading = true;
143-
await delay(1000); // Simulate processing delay
144-
loading = false;
145-
return blob;
146-
}
147137
</script>
148138

149139

@@ -152,20 +142,16 @@
152142
<Dialog.DialogHeader>
153143
<Dialog.DialogTitle>{$t`Add audio`}</Dialog.DialogTitle>
154144
</Dialog.DialogHeader>
155-
{#if !audio || !selectedFile}
156-
{#if loading}
157-
<Loading class="self-center justify-self-center size-16"/>
158-
{:else}
159-
<AudioProvider {onFileSelected} {onRecordingComplete}/>
160-
{/if}
145+
{#if !selectedFile}
146+
<AudioProvider {onFileSelected} {onRecordingComplete}/>
161147
{:else}
162-
<AudioEditor {audio} name={selectedFile.name} onDiscard={onDiscard}/>
148+
<AudioEditor audio={selectedFile} bind:finalAudio onDiscard={onDiscard}/>
163149
{#if tooBig}
164150
<p class="text-destructive text-lg text-end">{$t`File too big`}</p>
165151
{/if}
166152
<Dialog.DialogFooter>
167153
<Button onclick={() => open = false} variant="secondary">{$t`Cancel`}</Button>
168-
<Button onclick={() => submitAudio()} disabled={tooBig} loading={submitting}>
154+
<Button onclick={() => submitAudio()} disabled={tooBig || !finalAudio} loading={submitting}>
169155
{$t`Save audio`}
170156
</Button>
171157
</Dialog.DialogFooter>
Lines changed: 89 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,119 @@
11
<script lang="ts">
2-
import { t } from 'svelte-i18n-lingui';
2+
import {t} from 'svelte-i18n-lingui';
33
import Button from '../ui/button/button.svelte';
44
import Waveform from './wavesurfer/waveform.svelte';
55
import type WaveSurfer from 'wavesurfer.js';
66
import {formatDigitalDuration} from '../ui/format/format-duration';
77
import DevContent from '$lib/layout/DevContent.svelte';
88
import {Label} from '../ui/label';
9+
import {FFmpegApi} from './ffmpeg';
10+
import Loading from '$lib/components/Loading.svelte';
11+
import {resource, watch} from 'runed';
12+
import {onDestroy} from 'svelte';
913
1014
type Props = {
11-
audio: Blob;
12-
name: string
15+
audio: File;
16+
finalAudio: File | undefined;
1317
onDiscard: () => void;
1418
};
1519
16-
let { audio, name, onDiscard }: Props = $props();
20+
let {
21+
audio,
22+
finalAudio = $bindable(undefined),
23+
onDiscard
24+
}: Props = $props();
1725
1826
let audioApi = $state<WaveSurfer>();
1927
let playing = $state(false);
2028
let duration = $state<number | null>(null);
21-
const mb = $derived((audio.size / 1024 / 1024).toFixed(2));
22-
const formatedDuration = $derived(duration ? formatDigitalDuration({ seconds: duration }) : 'unknown');
29+
const mb = $derived(!finalAudio ? '0' : (finalAudio.size / 1024 / 1024).toFixed(2));
30+
const formattedDuration = $derived(duration ? formatDigitalDuration({seconds: duration}) : 'unknown');
31+
let ffmpegApi: FFmpegApi | undefined;
2332
33+
let ffmpegFile = resource(() => audio, async (audio, _, {signal}) => {
34+
ffmpegApi ??= await FFmpegApi.create();
35+
return await ffmpegApi.toFFmpegFile(audio, AbortSignal.any([signal, abortController.signal]));
36+
});
37+
38+
let flacFile = resource(() => [ffmpegFile.current], async ([file], _, {signal}) => {
39+
if (!file) return;
40+
ffmpegApi ??= await FFmpegApi.create();
41+
return await ffmpegApi.convertToFlac(file, AbortSignal.any([signal, abortController.signal]));
42+
});
43+
44+
let readFile = resource(() => [flacFile.current], async ([file], _, {signal}) => {
45+
if (!file) return;
46+
ffmpegApi ??= await FFmpegApi.create();
47+
return await ffmpegApi.readFile(file, AbortSignal.any([signal, abortController.signal]));
48+
});
49+
50+
watch(() => [readFile.current, readFile.loading] as const, ([file, loading]) => {
51+
if (loading || !file) {
52+
finalAudio = undefined;
53+
} else {
54+
finalAudio = file;
55+
}
56+
});
57+
58+
const loading = $derived(ffmpegFile.loading || flacFile.loading || readFile.loading);
59+
const error = $derived((ffmpegFile.error || flacFile.error || readFile.error)?.toString());
60+
61+
const abortController = new AbortController();
62+
onDestroy(() => {
63+
abortController.abort();
64+
ffmpegApi?.terminate();
65+
});
66+
67+
function downloadAudio() {
68+
if (!finalAudio) throw new Error('No audio to download');
69+
const url = URL.createObjectURL(finalAudio);
70+
try {
71+
const a = document.createElement('a');
72+
a.href = url;
73+
a.download = `${finalAudio.name}`;
74+
a.click();
75+
} finally {
76+
URL.revokeObjectURL(url);
77+
}
78+
}
2479
</script>
2580

2681
<div class="flex flex-col gap-4 items-center justify-center">
82+
{#if loading || !finalAudio}
83+
<Loading class="self-center justify-self-center size-16"/>
84+
{:else}
2785
<span class="inline-grid grid-cols-[auto_auto_1rem_auto_auto] gap-2 items-baseline">
28-
<Label class="justify-self-end">{$t`Length:`}</Label> <span>{$t`${formatedDuration}`}</span>
86+
<Label class="justify-self-end">{$t`Length:`}</Label> <span>{$t`${formattedDuration}`}</span>
2987
<span></span>
3088
<Label class="justify-self-end">{$t`Size:`}</Label> <span>{$t`${mb} MB`}</span>
31-
{#if name}
89+
{#if finalAudio.name}
3290
<Label class="justify-self-end">{$t`File name:`}</Label>
33-
<span class="col-span-4">{$t`${name}`}</span>
91+
<span class="col-span-4">{$t`${finalAudio.name}`}</span>
3492
{/if}
3593
<DevContent>
3694
<Label class="justify-self-end">{$t`Type:`}</Label>
37-
<span class="col-span-4">{$t`${audio.type}`}</span>
95+
<span class="col-span-4">{$t`${finalAudio.type}`}</span>
3896
</DevContent>
3997
</span>
40-
<!-- contain-inline-size prevents wavesurfer from freaking out inside a grid -->
41-
<!-- pb-8 ensures the timeline is in the bounds of the container -->
42-
<div class="w-full grow max-h-32 pb-3 contain-inline-size border-y">
43-
<Waveform {audio} bind:playing bind:audioApi bind:duration showTimeline autoplay class="size-full" />
44-
</div>
45-
<div class="flex gap-2">
46-
<Button variant="secondary" icon="i-mdi-close" onclick={onDiscard} disabled={!audio}>{$t`Discard`}</Button>
47-
<Button
48-
icon={playing ? 'i-mdi-pause' : 'i-mdi-play'}
49-
onclick={() => (playing ? audioApi?.pause() : audioApi?.play())}
50-
disabled={!audioApi}
51-
size="icon"
52-
/>
53-
</div>
98+
<!-- contain-size prevents wavesurfer from freaking out inside a grid
99+
contain-inline-size would improve the height reactivity of the waveform, but
100+
results in the waveform sometimes change its height unexpectedly -->
101+
<!-- pb-8 ensures the timeline is in the bounds of the container -->
102+
<div class="w-full grow max-h-32 pb-3 contain-size border-y">
103+
<Waveform audio={finalAudio} bind:playing bind:audioApi bind:duration showTimeline autoplay class="size-full"/>
104+
</div>
105+
<div class="flex gap-2">
106+
<Button onclick={() => downloadAudio()} disabled={!finalAudio} variant="secondary" icon="i-mdi-download">{$t`Download`}</Button>
107+
<Button variant="secondary" icon="i-mdi-close" onclick={onDiscard} disabled={!audio}>{$t`Discard`}</Button>
108+
<Button
109+
icon={playing ? 'i-mdi-pause' : 'i-mdi-play'}
110+
onclick={() => (playing ? audioApi?.pause() : audioApi?.play())}
111+
disabled={!audioApi}
112+
size="icon"
113+
/>
114+
</div>
115+
{/if}
116+
{#if error}
117+
<p class="text-destructive">{error}</p>
118+
{/if}
54119
</div>

0 commit comments

Comments
 (0)