diff --git a/README.md b/README.md index 284dee4c..d064d46a 100644 --- a/README.md +++ b/README.md @@ -330,7 +330,6 @@ void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, Rem With Platform Audio, the audio input and output are managed by the native ADM of WebRTC. This unlocks echo cancellation, noise suppression, auto gain control and hardware processing if available. There are some known issues with Platform Audio, that we are working on resolving: -- On iOS, disposing of Platform Audio object stops Unity audio output - On iOS and Unity 6, backgrounding the app breaks Platform Audio - On MacOS with bluetooth headset, unmuting can break audio output diff --git a/Runtime/Plugins/iOS/LiveKitAudioSession.mm b/Runtime/Plugins/iOS/LiveKitAudioSession.mm index f341fc56..74f63de5 100644 --- a/Runtime/Plugins/iOS/LiveKitAudioSession.mm +++ b/Runtime/Plugins/iOS/LiveKitAudioSession.mm @@ -16,54 +16,234 @@ #import +// This plugin coordinates the single shared AVAudioSession with WebRTC's iOS +// Audio Device Module (ADM). WebRTC ships an RTCAudioSession proxy that, left in +// its default "automatic" mode, reconfigures the category/route and *deactivates* +// the session whenever a call's playout/recording starts and stops. That fights +// with Unity/FMOD: on join the app's other audio (e.g. ambient music) is rerouted +// to the earpiece and attenuated, and on hang-up the session is deactivated out +// from under Unity so its audio dies. +// +// To make playout stable and keep Unity audio alive across call state, we put +// RTCAudioSession into MANUAL mode and have the app own the session: +// * We hold exactly one permanent activation (setActive:YES). Because +// RTCAudioSession ref-counts activation, WebRTC's per-call setActive:YES/NO +// only cycles the count and never actually deactivates the hardware session. +// * We set the category once (PlayAndRecord + VideoChat mode). VideoChat routes +// to the loudspeaker by default (while still honoring connected wired/Bluetooth +// headphones), so WebRTC re-applying its own config keeps output on the speaker +// instead of the earpiece. We deliberately do NOT force the speaker via +// overrideOutputAudioPort, which would override plugged-in headphones. +// * The VPIO voice-processing unit (hardware AEC/AGC/NS) is gated by +// isAudioEnabled. It defaults to YES so call audio works out of the box (the +// unit still only initializes once a call actually has an audio track, so +// pre-call audio is unaffected). Callers can toggle it via +// LiveKit_SetAudioEnabled -- e.g. OFF on hang-up so the unit stops between +// calls while our held activation keeps the session alive for Unity. +// +// RTCAudioSession lives inside the statically-linked liblivekit_ffi; we reach it +// dynamically via NSClassFromString + a protocol-typed id so this file never +// creates a link-time dependency on the class. If the class can't be found we +// fall back to configuring AVAudioSession directly (legacy behavior). + +/// Minimal subset of WebRTC's RTCAudioSession that we message dynamically. +@protocol LiveKitRTCAudioSession +@property(nonatomic, assign) BOOL useManualAudio; +@property(nonatomic, assign) BOOL isAudioEnabled; +@property(nonatomic, readonly) int activationCount; +- (void)lockForConfiguration; +- (void)unlockForConfiguration; +- (BOOL)setActive:(BOOL)active error:(NSError**)outError; +- (BOOL)setCategory:(AVAudioSessionCategory)category + mode:(AVAudioSessionMode)mode + options:(AVAudioSessionCategoryOptions)options + error:(NSError**)outError; +@end + +// Tracks whether *we* currently hold the one app-owned activation, so we add and +// release it exactly once regardless of how many times configure/restore run. +static BOOL s_liveKitHoldsActivation = NO; + +// Snapshot of the AVAudioSession configuration as it was the first time LiveKit +// touched the session (i.e. whatever Unity set up from its iOS Player Settings). +// Captured lazily in LiveKit_ConfigureAudioSessionForVoIP and re-applied by +// LiveKit_RestoreDefaultAudioSession when the last PlatformAudio is disposed. +static BOOL s_hasCachedState = NO; +static NSString* s_cachedCategory = nil; +static NSString* s_cachedMode = nil; +static AVAudioSessionCategoryOptions s_cachedCategoryOptions = 0; + +static const AVAudioSessionCategoryOptions kLiveKitCategoryOptions = + AVAudioSessionCategoryOptionDefaultToSpeaker | + AVAudioSessionCategoryOptionAllowBluetooth | + AVAudioSessionCategoryOptionAllowBluetoothA2DP; + +/// Returns WebRTC's shared RTCAudioSession if it's present in the linked binary, +/// or nil if the class can't be found (in which case callers use AVAudioSession). +static id LiveKit_RTCSession() { + Class cls = NSClassFromString(@"RTCAudioSession"); + if (!cls || ![cls respondsToSelector:@selector(sharedInstance)]) { + return nil; + } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warc-performSelector-leaks" + return (id)[cls performSelector:@selector(sharedInstance)]; +#pragma clang diagnostic pop +} + +/// Captures the current audio session category/mode/options exactly once, before +/// LiveKit reconfigures the session for VoIP. Subsequent calls are no-ops so the +/// snapshot always reflects the pristine, pre-LiveKit (Unity-configured) state. +static void LiveKit_CacheSessionStateIfNeeded() { + if (s_hasCachedState) { + return; + } + AVAudioSession* session = [AVAudioSession sharedInstance]; + // copy so the strings persist for the app lifetime regardless of ARC/MRC. + s_cachedCategory = [session.category copy]; + s_cachedMode = [session.mode copy]; + s_cachedCategoryOptions = session.categoryOptions; + s_hasCachedState = YES; + NSLog(@"LiveKit: cached audio session state: category=%@, mode=%@, options=%lu", + s_cachedCategory, s_cachedMode, (unsigned long)s_cachedCategoryOptions); +} + extern "C" { -/// Configures the iOS audio session for VoIP/WebRTC use. -/// This sets AVAudioSessionCategoryPlayAndRecord with VoiceChat mode, -/// which enables the VPIO (Voice Processing IO) AudioUnit for: -/// - Hardware echo cancellation (AEC) -/// - Automatic gain control (AGC) -/// - Noise suppression (NS) +/// Configures the iOS audio session for VoIP/WebRTC use and takes app ownership +/// of the shared AVAudioSession. +/// +/// This sets AVAudioSessionCategoryPlayAndRecord with VideoChat mode (which routes +/// to the loudspeaker by default and enables the VPIO Voice Processing IO unit for +/// hardware AEC/AGC/NS), puts RTCAudioSession into manual mode, and holds a single +/// permanent activation so WebRTC never deactivates the session on its own. /// -/// Call this before creating PlatformAudio to ensure WebRTC can -/// properly initialize the microphone and speaker. +/// Call this before creating PlatformAudio. Call audio is enabled by default, so +/// no further call is required for it to work; use LiveKit_SetAudioEnabled(false) +/// to stop the VPIO unit between calls (e.g. on hang-up). void LiveKit_ConfigureAudioSessionForVoIP() { - AVAudioSession* session = [AVAudioSession sharedInstance]; - NSError* error = nil; + // Snapshot the pristine (Unity Player Settings) session before we change it. + LiveKit_CacheSessionStateIfNeeded(); - // Configure for VoIP with echo cancellation - BOOL success = [session setCategory:AVAudioSessionCategoryPlayAndRecord - mode:AVAudioSessionModeVoiceChat - options:AVAudioSessionCategoryOptionDefaultToSpeaker | - AVAudioSessionCategoryOptionAllowBluetooth | - AVAudioSessionCategoryOptionAllowBluetoothA2DP - error:&error]; + id rtc = LiveKit_RTCSession(); - if (!success || error) { - NSLog(@"LiveKit: Failed to configure VoIP audio session: %@", error.localizedDescription); + if (rtc == nil) { + // RTCAudioSession unavailable: configure AVAudioSession directly (legacy). + AVAudioSession* session = [AVAudioSession sharedInstance]; + NSError* error = nil; + if (![session setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVideoChat + options:kLiveKitCategoryOptions + error:&error] || error) { + NSLog(@"LiveKit: Failed to configure audio session: %@", error.localizedDescription); + return; + } + if (![session setActive:YES error:&error] || error) { + NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); + return; + } + NSLog(@"LiveKit: Audio session configured (AVAudioSession fallback, VideoChat)"); return; } - // Activate the audio session - success = [session setActive:YES error:&error]; - if (!success || error) { - NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); - return; + // Manual mode: WebRTC won't activate/deactivate the session on its own, and + // won't initialize the VPIO unit until we grant permission via isAudioEnabled + // (set below). This is what lets us own activation and gate the unit. + rtc.useManualAudio = YES; + + [rtc lockForConfiguration]; + NSError* error = nil; + if (![rtc setCategory:AVAudioSessionCategoryPlayAndRecord + mode:AVAudioSessionModeVideoChat + options:kLiveKitCategoryOptions + error:&error] || error) { + NSLog(@"LiveKit: Failed to set audio category: %@", error.localizedDescription); + } + + // Hold exactly one app-owned activation. RTCAudioSession ref-counts activation, + // so WebRTC's balanced setActive:YES/NO during a call never drops the real + // session below active while we hold this. + if (!s_liveKitHoldsActivation) { + error = nil; + if ([rtc setActive:YES error:&error] && !error) { + s_liveKitHoldsActivation = YES; + } else { + NSLog(@"LiveKit: Failed to activate audio session: %@", error.localizedDescription); + } } - NSLog(@"LiveKit: Audio session configured for VoIP (PlayAndRecord + VoiceChat mode)"); + [rtc unlockForConfiguration]; + + // Grant WebRTC permission to initialize its audio unit by default so call audio + // works without an explicit LiveKit_SetAudioEnabled(true). The unit is only + // actually created once a call has an audio track, so pre-call audio is + // unaffected. Callers may still disable it (e.g. on hang-up) via + // LiveKit_SetAudioEnabled(false). + rtc.isAudioEnabled = YES; + + NSLog(@"LiveKit: Audio session configured for VoIP (PlayAndRecord + VideoChat, manual mode, activationCount=%d)", + rtc.activationCount); +} + +/// Enables or disables WebRTC's VPIO audio unit while the app keeps ownership of +/// the session. Pass true when a call connects and false when it ends. +/// +/// This is only effective in manual mode (set up by LiveKit_ConfigureAudioSessionForVoIP). +/// Disabling on hang-up stops incoming/outgoing call audio and the VPIO processing, +/// but leaves the session active (via the app's held activation), so Unity audio +/// keeps playing. +void LiveKit_SetAudioEnabled(bool enabled) { + id rtc = LiveKit_RTCSession(); + if (rtc == nil) { + return; + } + rtc.isAudioEnabled = enabled ? YES : NO; + NSLog(@"LiveKit: isAudioEnabled=%@ (activationCount=%d)", enabled ? @"YES" : @"NO", rtc.activationCount); } -/// Restores the audio session to the default ambient category. -/// Call this when PlatformAudio is disposed if you want to restore -/// the original audio behavior. +/// Restores the audio session Unity had before LiveKit touched it (or the ambient +/// category if LiveKit never configured it), relinquishes the app-owned activation +/// and manual mode, and reactivates the session so Unity audio output resumes. +/// Call this when the last PlatformAudio is disposed. void LiveKit_RestoreDefaultAudioSession() { + id rtc = LiveKit_RTCSession(); + + if (rtc != nil) { + // Stop the VPIO unit and release our activation before handing control back. + rtc.isAudioEnabled = NO; + if (s_liveKitHoldsActivation) { + NSError* error = nil; + if (![rtc setActive:NO error:&error] || error) { + NSLog(@"LiveKit: Failed to deactivate audio session: %@", error.localizedDescription); + } + s_liveKitHoldsActivation = NO; + } + rtc.useManualAudio = NO; + } + AVAudioSession* session = [AVAudioSession sharedInstance]; NSError* error = nil; + if (s_hasCachedState) { + if (![session setCategory:s_cachedCategory + mode:s_cachedMode + options:s_cachedCategoryOptions + error:&error] || error) { + NSLog(@"LiveKit: Failed to restore cached audio session (category=%@, mode=%@): %@", + s_cachedCategory, s_cachedMode, error.localizedDescription); + } + } else { + // Configure was never called, so we have nothing to restore to; fall back + // to the ambient category. + [session setCategory:AVAudioSessionCategoryAmbient error:&error]; + if (error) { + NSLog(@"LiveKit: Failed to restore default audio session: %@", error.localizedDescription); + } + } - [session setCategory:AVAudioSessionCategoryAmbient error:&error]; - if (error) { - NSLog(@"LiveKit: Failed to restore default audio session: %@", error.localizedDescription); + // Hand an active session back to Unity so its audio output resumes. + error = nil; + if (![session setActive:YES error:&error] || error) { + NSLog(@"LiveKit: Failed to reactivate audio session: %@", error.localizedDescription); } } diff --git a/Runtime/Scripts/Audio/PlatformAudio.cs b/Runtime/Scripts/Audio/PlatformAudio.cs index 7c113e20..cfe13639 100644 --- a/Runtime/Scripts/Audio/PlatformAudio.cs +++ b/Runtime/Scripts/Audio/PlatformAudio.cs @@ -27,10 +27,21 @@ internal static class IOSAudioSessionHelper internal static extern void LiveKit_ConfigureAudioSessionForVoIP(); /// - /// Restores the iOS audio session to ambient mode. + /// Restores the audio session Unity had before LiveKit configured it + /// (or the ambient category as a fallback) and reactivates it so Unity + /// audio output resumes. Called when the last PlatformAudio is disposed. /// [DllImport("__Internal")] internal static extern void LiveKit_RestoreDefaultAudioSession(); + + /// + /// Enables or disables WebRTC's VPIO audio unit while the app keeps + /// ownership of the audio session. Enable when a call connects, disable + /// when it ends. Disabling on hang-up stops call audio without + /// deactivating the session, so other app audio keeps playing. + /// + [DllImport("__Internal")] + internal static extern void LiveKit_SetAudioEnabled([MarshalAs(UnmanagedType.I1)] bool enabled); } #endif @@ -74,6 +85,11 @@ public sealed class PlatformAudio : IDisposable internal readonly FfiHandle Handle; private readonly PlatformAudioInfo _info; private bool _disposed = false; +#if UNITY_IOS && !UNITY_EDITOR + // Tracks live PlatformAudio instances so the iOS audio session is restored + // only when the last one is disposed (aligned with the native ADM ref-count). + private static int _instanceCount; +#endif /// /// Number of available recording (microphone) devices. @@ -92,7 +108,7 @@ public sealed class PlatformAudio : IDisposable /// to a room if you want automatic speaker playout for remote audio. /// /// On iOS, this automatically configures the audio session for VoIP mode - /// (PlayAndRecord category with VoiceChat mode) to enable hardware echo + /// (PlayAndRecord category with VideoChat mode) to enable hardware echo /// cancellation and microphone input. /// /// @@ -103,7 +119,7 @@ public PlatformAudio() { #if UNITY_IOS && !UNITY_EDITOR // Configure iOS audio session for VoIP before initializing WebRTC ADM. - // This sets PlayAndRecord category with VoiceChat mode for hardware AEC. + // This sets PlayAndRecord category with VideoChat mode for hardware AEC. IOSAudioSessionHelper.LiveKit_ConfigureAudioSessionForVoIP(); #endif @@ -119,6 +135,12 @@ public PlatformAudio() _info = platformAudio.Info; Utils.Debug($"PlatformAudio created: {RecordingDeviceCount} recording devices, {PlayoutDeviceCount} playout devices"); + +#if UNITY_IOS && !UNITY_EDITOR + // Count this instance only after successful construction so a failed + // ctor never leaves the counter stuck above zero. + System.Threading.Interlocked.Increment(ref _instanceCount); +#endif } /// @@ -354,6 +376,28 @@ public void StopRecording() Utils.Debug("PlatformAudio: stopped recording"); } + /// + /// Signals whether call audio should be active on the platform audio session. + /// + /// On iOS this gates WebRTC's VPIO audio unit while the app retains ownership + /// of the shared AVAudioSession. It is enabled by default when PlatformAudio is + /// created, so this only needs to be called to false when leaving a room + /// (and back to true when rejoining). Disabling stops the microphone/ + /// remote audio path and the hardware voice processing, but keeps the audio + /// session active so other Unity audio (e.g. background music) is not + /// interrupted — which is why Unity audio survives a hang-up. + /// + /// On other platforms this is a no-op: the OS/ADM manages the session directly. + /// + /// True while a call is active, false otherwise. + public void SetSessionAudioEnabled(bool enabled) + { +#if UNITY_IOS && !UNITY_EDITOR + IOSAudioSessionHelper.LiveKit_SetAudioEnabled(enabled); +#endif + Utils.Debug($"PlatformAudio: session audio enabled={enabled}"); + } + /// /// Releases the PlatformAudio resources. /// @@ -364,6 +408,17 @@ public void Dispose() { if (_disposed) return; Handle.Dispose(); + +#if UNITY_IOS && !UNITY_EDITOR + // Once the last instance is gone, relinquish the app-owned audio session: + // disable call audio, release our activation, leave manual mode, restore + // the session Unity had before LiveKit touched it, and reactivate it so + // Unity audio output resumes. Balances LiveKit_ConfigureAudioSessionForVoIP() + // in the constructor so the session isn't left stuck in PlayAndRecord. + if (System.Threading.Interlocked.Decrement(ref _instanceCount) == 0) + IOSAudioSessionHelper.LiveKit_RestoreDefaultAudioSession(); +#endif + _disposed = true; Utils.Debug("PlatformAudio disposed"); } diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 1dc27c71..c47c5538 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -168,6 +168,11 @@ private void OnEndCall() { if (_room == null) return; + // Disable call audio while keeping the app-owned audio session active, so + // Unity audio (e.g. background music) survives the hang-up on iOS. + if (usePlatformAudio) + _platformAudio?.SetSessionAudioEnabled(false); + _room.Disconnect(); CleanUpAllTracks(); _room = null; @@ -249,6 +254,13 @@ private IEnumerator ConnectToRoom() _localId = _room.LocalParticipant.Identity; buttonBar.SetConnected(true); + // Enable call audio now that we're in a room. On iOS this turns on WebRTC's + // VPIO unit while the app keeps ownership of the audio session; leaving the + // room disables it again (see OnEndCall / OnDisconnected) so other Unity + // audio keeps playing. + if (usePlatformAudio) + _platformAudio?.SetSessionAudioEnabled(true); + EnsureParticipantTile(_localId); foreach (var remote in _room.RemoteParticipants.Values) EnsureParticipantTile(remote.Identity); @@ -433,7 +445,14 @@ private void OnParticipantDisconnected(Participant participant, DisconnectReason } private void OnDisconnected(Room room) - => Debug.Log($"Disconnected from room: {room.DisconnectReason}"); + { + Debug.Log($"Disconnected from room: {room.DisconnectReason}"); + + // Covers server-initiated disconnects as well as OnEndCall; idempotent with + // the call already made there. Keeps the audio session active for Unity. + if (usePlatformAudio) + _platformAudio?.SetSessionAudioEnabled(false); + } private void OnTrackMuted(TrackPublication publication, Participant participant) {