From 000703336b7840c07c583d11e4e3164fb75e2486 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Mar 2026 16:30:58 +0800 Subject: [PATCH 1/6] improve audio/video streams --- Runtime/Scripts/AudioStream.cs | 54 ++++++-- Runtime/Scripts/Internal/AudioResampler.cs | 42 ++++++- Runtime/Scripts/VideoStream.cs | 136 ++++++++++++++++++--- Tests/EditMode/MediaStreamLifetimeTests.cs | 69 +++++++++++ 4 files changed, 269 insertions(+), 32 deletions(-) create mode 100644 Tests/EditMode/MediaStreamLifetimeTests.cs diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index b0d185b6..ccc028ae 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -15,12 +15,13 @@ public sealed class AudioStream : IDisposable { internal readonly FfiHandle Handle; private readonly AudioSource _audioSource; + private readonly AudioProbe _probe; private RingBuffer _buffer; private short[] _tempBuffer; private uint _numChannels; private uint _sampleRate; private AudioResampler _resampler = new AudioResampler(); - private object _lock = new object(); + private readonly object _lock = new object(); private bool _disposed = false; /// @@ -50,14 +51,20 @@ public AudioStream(RemoteAudioTrack audioTrack, AudioSource source) FfiClient.Instance.AudioStreamEventReceived += OnAudioStreamEvent; _audioSource = source; - var probe = _audioSource.gameObject.AddComponent(); - probe.AudioRead += OnAudioRead; + _probe = _audioSource.gameObject.AddComponent(); + _probe.AudioRead += OnAudioRead; _audioSource.Play(); } // Called on Unity audio thread private void OnAudioRead(float[] data, int channels, int sampleRate) { + if (_disposed) + { + Array.Clear(data, 0, data.Length); + return; + } + lock (_lock) { if (_buffer == null || channels != _numChannels || sampleRate != _sampleRate || data.Length != _tempBuffer.Length) @@ -90,13 +97,16 @@ static float S16ToFloat(short v) // Called on the MainThread (See FfiClient) private void OnAudioStreamEvent(AudioStreamEvent e) { + if (_disposed) + return; + if ((ulong)Handle.DangerousGetHandle() != e.StreamHandle) return; if (e.MessageCase != AudioStreamEvent.MessageOneofCase.FrameReceived) return; - var frame = new AudioFrame(e.FrameReceived.Frame); + using var frame = new AudioFrame(e.FrameReceived.Frame); lock (_lock) { @@ -105,7 +115,7 @@ private void OnAudioStreamEvent(AudioStreamEvent e) unsafe { - var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate); + using var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate); if (uFrame != null) { var data = new Span(uFrame.Data.ToPointer(), uFrame.Length); @@ -124,11 +134,39 @@ public void Dispose() private void Dispose(bool disposing) { - if (!_disposed && disposing) + if (_disposed) { - _audioSource.Stop(); - UnityEngine.Object.Destroy(_audioSource.GetComponent()); + return; } + + // Remove long-lived delegate references first so this instance can become collectible + // as soon as user code drops it. This also prevents late native callbacks from + // touching partially disposed state. + FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent; + + lock (_lock) + { + // Native resources can be released on both the explicit-dispose and finalizer + // paths. Unity objects are only touched when Dispose() is called explicitly on the + // main thread. + if (disposing) + { + _audioSource.Stop(); + if (_probe != null) + { + _probe.AudioRead -= OnAudioRead; + UnityEngine.Object.Destroy(_probe); + } + } + + _buffer?.Dispose(); + _buffer = null; + _tempBuffer = null; + _resampler?.Dispose(); + _resampler = null; + Handle.Dispose(); + } + _disposed = true; } diff --git a/Runtime/Scripts/Internal/AudioResampler.cs b/Runtime/Scripts/Internal/AudioResampler.cs index 47fa5c44..162d9228 100644 --- a/Runtime/Scripts/Internal/AudioResampler.cs +++ b/Runtime/Scripts/Internal/AudioResampler.cs @@ -1,27 +1,34 @@ +using System; using LiveKit.Internal.FFIClients.Requests; +using LiveKit.Internal; using LiveKit.Proto; -using UnityEngine; namespace LiveKit { - public class AudioResampler + public sealed class AudioResampler : IDisposable { - internal readonly OwnedAudioResampler resampler; + private readonly FfiHandle _handle; + private bool _disposed; public AudioResampler() { using var request = FFIBridge.Instance.NewRequest(); using var response = request.Send(); FfiResponse res = response; - resampler = res.NewAudioResampler.Resampler; + _handle = FfiHandle.FromOwnedHandle(res.NewAudioResampler.Resampler.Handle); } public AudioFrame RemixAndResample(AudioFrame frame, uint numChannels, uint sampleRate) { + if (_disposed) + { + throw new ObjectDisposedException(nameof(AudioResampler)); + } + using var request = FFIBridge.Instance.NewRequest(); using var audioFrameBufferInfo = request.TempResource(); var remix = request.request; - remix.ResamplerHandle = resampler.Handle.Id; + remix.ResamplerHandle = (ulong)_handle.DangerousGetHandle(); remix.Buffer = frame.Info; remix.NumChannels = numChannels; remix.SampleRate = sampleRate; @@ -35,5 +42,28 @@ public AudioFrame RemixAndResample(AudioFrame frame, uint numChannels, uint samp var newBuffer = res.RemixAndResample.Buffer; return new AudioFrame(newBuffer); } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _handle.Dispose(); + _disposed = true; + GC.SuppressFinalize(this); + } + + ~AudioResampler() + { + if (_disposed) + { + return; + } + + _handle.Dispose(); + _disposed = true; + } } -} \ No newline at end of file +} diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index eadd9ce8..8d2fa5e9 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -18,6 +18,14 @@ public class VideoStream private bool _disposed = false; private bool _dirty = false; private YuvToRgbConverter _converter; + private readonly object _frameLock = new object(); + // Separates frame production from consumption: + // - OnVideoStreamEvent produces the latest native frame coming from Rust + // - Update() consumes at most one frame per Unity tick for upload/render + // + // Keeping the "next frame" in a dedicated slot lets Unity coalesce bursts down to the + // newest pending frame without overwriting the frame currently being uploaded. + private VideoFrameBuffer _pendingBuffer; /// Called when we receive a new frame from the VideoTrack public event FrameReceiveDelegate FrameReceived; @@ -31,6 +39,9 @@ public class VideoStream /// The texture changes every time the video resolution changes. /// Can be null if UpdateRoutine isn't started public RenderTexture Texture { private set; get; } + // The frame currently owned by the Unity update/render path. Update() swaps the latest + // pending frame into this slot before converting/uploading it, so this represents the + // frame being actively consumed rather than the next frame arriving from Rust. public VideoFrameBuffer VideoBuffer { private set; get; } protected bool _playing = false; @@ -68,19 +79,38 @@ public void Dispose() private void Dispose(bool disposing) { - if (!_disposed) + if (_disposed) + return; + + // Remove the long-lived delegate reference first so this stream can be collected as + // soon as user code drops it, and so late native callbacks cannot mutate disposed + // frame/converter state. + FfiClient.Instance.VideoStreamEventReceived -= OnVideoStreamEvent; + + lock (_frameLock) { - if (disposing) - { - VideoBuffer?.Dispose(); - } - // Unity objects must be destroyed on main thread + // Native frame buffers are not Unity objects, so always release them even on the + // finalizer path. This keeps the stream from leaking native frame handles if user + // code forgets to call Dispose(). + VideoBuffer?.Dispose(); + VideoBuffer = null; + _pendingBuffer?.Dispose(); + _pendingBuffer = null; + } + + if (disposing) + { + // Unity objects must be destroyed on main thread, so only touch the converter and + // RenderTexture when Dispose() is called explicitly by user code. _converter?.Dispose(); _converter = null; - // Texture is owned and cleaned up by _converter. Set to null to avoid holding a reference to a disposed RenderTexture. + // Texture is owned and cleaned up by _converter. Set to null to avoid holding a + // reference to a disposed RenderTexture. Texture = null; - _disposed = true; } + + Handle.Dispose(); + _disposed = true; } public virtual void Start() @@ -92,6 +122,18 @@ public virtual void Start() public virtual void Stop() { _playing = false; + + // When the stream has no active consumer, do not keep the latest native frame alive. + // Rust may still be producing frames depending on its queue configuration, but Unity + // drops them immediately until Start() is called again. + lock (_frameLock) + { + _pendingBuffer?.Dispose(); + _pendingBuffer = null; + VideoBuffer?.Dispose(); + VideoBuffer = null; + _dirty = false; + } } public IEnumerator Update() @@ -103,10 +145,32 @@ public IEnumerator Update() if (_disposed) break; - if (VideoBuffer == null || !VideoBuffer.IsValid || !_dirty) + VideoFrameBuffer nextBuffer = null; + lock (_frameLock) + { + if (_dirty) + { + nextBuffer = _pendingBuffer; + _pendingBuffer = null; + _dirty = false; + } + } + + if (nextBuffer == null) continue; - _dirty = false; + // Latest-frame-wins: if Rust buffered multiple frames, the intake path keeps only + // the newest pending frame. Update() uploads at most one frame per Unity tick. + VideoBuffer?.Dispose(); + VideoBuffer = nextBuffer; + + if (!VideoBuffer.IsValid) + { + VideoBuffer.Dispose(); + VideoBuffer = null; + continue; + } + var rWidth = VideoBuffer.Width; var rHeight = VideoBuffer.Height; @@ -127,24 +191,60 @@ public IEnumerator Update() // Handle new video stream events private void OnVideoStreamEvent(VideoStreamEvent e) { + if (_disposed) + return; + if (e.StreamHandle != (ulong)Handle.DangerousGetHandle()) return; if (e.MessageCase != VideoStreamEvent.MessageOneofCase.FrameReceived) return; - + var newBuffer = e.FrameReceived.Buffer; var handle = new FfiHandle((IntPtr)newBuffer.Handle.Id); var frameInfo = newBuffer.Info; - - var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation); + // Create a managed wrapper around the native frame handle. This does not copy the + // underlying video payload; the wrapper simply owns the FFI handle until the frame is + // either uploaded or dropped. var buffer = VideoFrameBuffer.Create(handle, frameInfo); + if (buffer == null) + { + handle.Dispose(); + return; + } + + // If there is no active consumer, keep draining frames from Rust but drop them + // immediately on the Unity side to avoid growing native memory or preserving stale + // frames. The producer queue can be size 1, bounded N, or unbounded; this behavior is + // correct for all three because Unity only wants the most recent renderable frame. + if (!_playing) + { + buffer.Dispose(); + return; + } + + lock (_frameLock) + { + if (_disposed || !_playing) + { + buffer.Dispose(); + return; + } - VideoBuffer?.Dispose(); - VideoBuffer = buffer; - _dirty = true; + // Latest-frame-wins coalescing. If Rust delivers several frames before Update() + // runs again, replace the pending frame with the newest one and drop the older + // native buffer immediately. + _pendingBuffer?.Dispose(); + _pendingBuffer = buffer; + _dirty = true; + } - FrameReceived?.Invoke(frame); + // Avoid allocating VideoFrame objects when nobody is observing them. + if (FrameReceived != null) + { + var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation); + FrameReceived.Invoke(frame); + } } } -} \ No newline at end of file +} diff --git a/Tests/EditMode/MediaStreamLifetimeTests.cs b/Tests/EditMode/MediaStreamLifetimeTests.cs new file mode 100644 index 00000000..0340c323 --- /dev/null +++ b/Tests/EditMode/MediaStreamLifetimeTests.cs @@ -0,0 +1,69 @@ +using System.IO; +using NUnit.Framework; + +namespace LiveKit.EditModeTests +{ + public class MediaStreamLifetimeTests + { + private const string AudioStreamPath = "Runtime/Scripts/AudioStream.cs"; + private const string VideoStreamPath = "Runtime/Scripts/VideoStream.cs"; + private const string AudioResamplerPath = "Runtime/Scripts/Internal/AudioResampler.cs"; + + [Test] + public void AudioStream_Dispose_UnsubscribesAndReleasesOwnedResources() + { + var source = File.ReadAllText(AudioStreamPath); + + StringAssert.Contains("FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent;", source); + StringAssert.Contains("_probe.AudioRead -= OnAudioRead;", source); + StringAssert.Contains("_buffer?.Dispose();", source); + StringAssert.Contains("_resampler?.Dispose();", source); + StringAssert.Contains("Handle.Dispose();", source); + } + + [Test] + public void AudioStream_AudioFrames_AreDisposedAfterProcessing() + { + var source = File.ReadAllText(AudioStreamPath); + + // Both the inbound native frame and the remixed output frame should be scoped so their + // handles are released after each callback rather than accumulating over time. + StringAssert.Contains("using var frame = new AudioFrame(e.FrameReceived.Frame);", source); + StringAssert.Contains("using var uFrame = _resampler.RemixAndResample(frame, _numChannels, _sampleRate);", source); + } + + [Test] + public void AudioResampler_IsDisposable_AndReleasesNativeHandle() + { + var source = File.ReadAllText(AudioResamplerPath); + + StringAssert.Contains("public sealed class AudioResampler : IDisposable", source); + StringAssert.Contains("_handle.Dispose();", source); + } + + [Test] + public void VideoStream_Dispose_UnsubscribesAndReleasesOwnedResources() + { + var source = File.ReadAllText(VideoStreamPath); + + StringAssert.Contains("FfiClient.Instance.VideoStreamEventReceived -= OnVideoStreamEvent;", source); + StringAssert.Contains("VideoBuffer?.Dispose();", source); + StringAssert.Contains("_pendingBuffer?.Dispose();", source); + StringAssert.Contains("Handle.Dispose();", source); + } + + [Test] + public void VideoStream_UsesLatestFrameWinsCoalescing() + { + var source = File.ReadAllText(VideoStreamPath); + + // The intake path should maintain a dedicated pending slot and replace/drop superseded + // frames so Unity uploads at most the latest frame per tick. + StringAssert.Contains("private VideoFrameBuffer _pendingBuffer;", source); + StringAssert.Contains("_pendingBuffer?.Dispose();", source); + StringAssert.Contains("_pendingBuffer = buffer;", source); + StringAssert.Contains("nextBuffer = _pendingBuffer;", source); + StringAssert.Contains("VideoBuffer = nextBuffer;", source); + } + } +} From 16741dc956d47dd30a057815aad237fd5b582024 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Mar 2026 18:04:22 +0800 Subject: [PATCH 2/6] fix the build --- Tests/EditMode/MediaStreamLifetimeTests.cs | 44 ++++++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/Tests/EditMode/MediaStreamLifetimeTests.cs b/Tests/EditMode/MediaStreamLifetimeTests.cs index 0340c323..54849ad8 100644 --- a/Tests/EditMode/MediaStreamLifetimeTests.cs +++ b/Tests/EditMode/MediaStreamLifetimeTests.cs @@ -5,14 +5,42 @@ namespace LiveKit.EditModeTests { public class MediaStreamLifetimeTests { - private const string AudioStreamPath = "Runtime/Scripts/AudioStream.cs"; - private const string VideoStreamPath = "Runtime/Scripts/VideoStream.cs"; - private const string AudioResamplerPath = "Runtime/Scripts/Internal/AudioResampler.cs"; + private static readonly string[] AudioStreamPaths = + { + "Runtime/Scripts/AudioStream.cs", + "Assets/client-sdk-unity/Runtime/Scripts/AudioStream.cs", + }; + + private static readonly string[] VideoStreamPaths = + { + "Runtime/Scripts/VideoStream.cs", + "Assets/client-sdk-unity/Runtime/Scripts/VideoStream.cs", + }; + + private static readonly string[] AudioResamplerPaths = + { + "Runtime/Scripts/Internal/AudioResampler.cs", + "Assets/client-sdk-unity/Runtime/Scripts/Internal/AudioResampler.cs", + }; + + private static string ReadSource(params string[] candidates) + { + foreach (var candidate in candidates) + { + if (File.Exists(candidate)) + { + return File.ReadAllText(candidate); + } + } + + Assert.Fail($"Could not find source file. Tried: {string.Join(", ", candidates)}"); + return string.Empty; + } [Test] public void AudioStream_Dispose_UnsubscribesAndReleasesOwnedResources() { - var source = File.ReadAllText(AudioStreamPath); + var source = ReadSource(AudioStreamPaths); StringAssert.Contains("FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent;", source); StringAssert.Contains("_probe.AudioRead -= OnAudioRead;", source); @@ -24,7 +52,7 @@ public void AudioStream_Dispose_UnsubscribesAndReleasesOwnedResources() [Test] public void AudioStream_AudioFrames_AreDisposedAfterProcessing() { - var source = File.ReadAllText(AudioStreamPath); + var source = ReadSource(AudioStreamPaths); // Both the inbound native frame and the remixed output frame should be scoped so their // handles are released after each callback rather than accumulating over time. @@ -35,7 +63,7 @@ public void AudioStream_AudioFrames_AreDisposedAfterProcessing() [Test] public void AudioResampler_IsDisposable_AndReleasesNativeHandle() { - var source = File.ReadAllText(AudioResamplerPath); + var source = ReadSource(AudioResamplerPaths); StringAssert.Contains("public sealed class AudioResampler : IDisposable", source); StringAssert.Contains("_handle.Dispose();", source); @@ -44,7 +72,7 @@ public void AudioResampler_IsDisposable_AndReleasesNativeHandle() [Test] public void VideoStream_Dispose_UnsubscribesAndReleasesOwnedResources() { - var source = File.ReadAllText(VideoStreamPath); + var source = ReadSource(VideoStreamPaths); StringAssert.Contains("FfiClient.Instance.VideoStreamEventReceived -= OnVideoStreamEvent;", source); StringAssert.Contains("VideoBuffer?.Dispose();", source); @@ -55,7 +83,7 @@ public void VideoStream_Dispose_UnsubscribesAndReleasesOwnedResources() [Test] public void VideoStream_UsesLatestFrameWinsCoalescing() { - var source = File.ReadAllText(VideoStreamPath); + var source = ReadSource(VideoStreamPaths); // The intake path should maintain a dedicated pending slot and replace/drop superseded // frames so Unity uploads at most the latest frame per tick. From 11b6b40cd80b76a23c24abc8d6a11d81d32a7ea5 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Mar 2026 18:42:26 +0800 Subject: [PATCH 3/6] another try to fix the build --- Tests/EditMode/MediaStreamLifetimeTests.cs | 77 ++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/Tests/EditMode/MediaStreamLifetimeTests.cs b/Tests/EditMode/MediaStreamLifetimeTests.cs index 54849ad8..9681f2ab 100644 --- a/Tests/EditMode/MediaStreamLifetimeTests.cs +++ b/Tests/EditMode/MediaStreamLifetimeTests.cs @@ -1,5 +1,6 @@ using System.IO; using NUnit.Framework; +using UnityEngine; namespace LiveKit.EditModeTests { @@ -25,6 +26,53 @@ public class MediaStreamLifetimeTests private static string ReadSource(params string[] candidates) { + foreach (var root in SearchRoots()) + { + foreach (var candidate in candidates) + { + var combined = Path.GetFullPath(Path.Combine(root, candidate)); + if (File.Exists(combined)) + { + return File.ReadAllText(combined); + } + } + } + + foreach (var root in SearchRoots()) + { + foreach (var candidate in candidates) + { + var fileName = Path.GetFileName(candidate); + if (string.IsNullOrEmpty(fileName) || !Directory.Exists(root)) + { + continue; + } + + try + { + foreach (var match in Directory.EnumerateFiles(root, fileName, SearchOption.AllDirectories)) + { + // Keep the search specific to the intended suffix so a duplicate file + // name elsewhere in the repo does not satisfy the test accidentally. + var normalizedMatch = match.Replace('\\', '/'); + var normalizedCandidate = candidate.Replace('\\', '/'); + if (normalizedMatch.EndsWith(normalizedCandidate)) + { + return File.ReadAllText(match); + } + } + } + catch (IOException) + { + // Best-effort lookup for CI layout differences. + } + catch (UnauthorizedAccessException) + { + // Best-effort lookup for CI layout differences. + } + } + } + foreach (var candidate in candidates) { if (File.Exists(candidate)) @@ -37,6 +85,35 @@ private static string ReadSource(params string[] candidates) return string.Empty; } + private static string[] SearchRoots() + { + var roots = new System.Collections.Generic.List(); + + void AddWithParents(string path) + { + if (string.IsNullOrEmpty(path)) + { + return; + } + + var fullPath = Path.GetFullPath(path); + var dir = new DirectoryInfo(fullPath); + while (dir != null) + { + if (!roots.Contains(dir.FullName)) + { + roots.Add(dir.FullName); + } + dir = dir.Parent; + } + } + + AddWithParents(Directory.GetCurrentDirectory()); + AddWithParents(Application.dataPath); + + return roots.ToArray(); + } + [Test] public void AudioStream_Dispose_UnsubscribesAndReleasesOwnedResources() { From 376183c57c565a680670da997ef60700e3dc49b7 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Mar 2026 22:11:49 +0800 Subject: [PATCH 4/6] fix the build again --- Tests/EditMode/MediaStreamLifetimeTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/EditMode/MediaStreamLifetimeTests.cs b/Tests/EditMode/MediaStreamLifetimeTests.cs index 9681f2ab..425212d9 100644 --- a/Tests/EditMode/MediaStreamLifetimeTests.cs +++ b/Tests/EditMode/MediaStreamLifetimeTests.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using NUnit.Framework; using UnityEngine; From 5ec2c22fc6043cb12cd6b9760f447c2c2d9494fe Mon Sep 17 00:00:00 2001 From: shijing xian Date: Wed, 25 Mar 2026 22:54:29 -0700 Subject: [PATCH 5/6] keep audio frames off from unity main thread --- README.md | 8 +- Runtime/Scripts/AudioFrame.cs | 4 +- Runtime/Scripts/Internal/FFIClient.cs | 113 +++++++++------- Runtime/Scripts/Internal/RingBuffer.cs | 5 + Runtime/Scripts/MicrophoneSource.cs | 12 +- Runtime/Scripts/RtcAudioSource.cs | 179 ++++++++++++++++++++++--- 6 files changed, 250 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index dcdbb7f0..b89c8f19 100644 --- a/README.md +++ b/README.md @@ -97,9 +97,13 @@ add other linker flags to `UnityFramework`: `-ObjC` -Since `libiPhone-lib.a` has built-in old versions of `celt` and `libvpx` (This will cause the opus and vp8/vp9 codecs to not be called correctly and cause a crash.), so you need to adjust the link order to ensure that it is linked to `liblivekit_ffi.a` first. +Since `libiPhone-lib.a` has built-in old versions of `celt` and `libvpx` (This will cause the opus and vp8/vp9 codecs to not be called correctly and cause a crash.), you need to ensure that `liblivekit_ffi.a` is linked before `libiPhone-lib.a`. -The fix is ​​to remove and re-add `libiPhone-lib.a` from `Frameworks and Libraries`, making sure to link after `liblivekit_ffi.a`. +The package now applies an iOS post-build fix that rewrites the exported Xcode project so `libiPhone-lib.a` is moved after `liblivekit_ffi.a` in `UnityFramework -> Frameworks and Libraries`. + +It also strips the old CELT object cluster from the exported `Libraries/libiPhone-lib.a` so Xcode cannot resolve those codec symbols from Unity's archive. + +If your project disables package editor scripts or uses a custom Xcode export pipeline that overwrites `project.pbxproj` after LiveKit runs, you may still need to adjust the order manually by removing and re-adding `libiPhone-lib.a`. ## Examples diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs index 9be0a3c9..f8fc1cf3 100644 --- a/Runtime/Scripts/AudioFrame.cs +++ b/Runtime/Scripts/AudioFrame.cs @@ -57,8 +57,10 @@ protected virtual void Dispose(bool disposing) { _allocatedData.Dispose(); } + + _handle?.Dispose(); _disposed = true; } } } -} \ No newline at end of file +} diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index 5f83ee7b..b2a21b72 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -276,67 +276,80 @@ static unsafe void FFICallback(UIntPtr data, UIntPtr size) var respData = new Span(data.ToPointer()!, (int)size.ToUInt64()); var response = FfiEvent.Parser!.ParseFrom(respData); + // Keep remote audio delivery off the Unity main-thread dispatch queue. Audio playback + // has its own synchronization path and benefits from handling frames as soon as the + // native callback arrives. + if (response.MessageCase == FfiEvent.MessageOneofCase.AudioStreamEvent) + { + DispatchEvent(response); + return; + } + // Run on the main thread, the order of execution is guaranteed by Unity // It uses a Queue internally - Instance._context?.Post((resp) => + Instance._context?.Post(static (resp) => { var r = resp as FfiEvent; if (r == null) { return; } + DispatchEvent(r); + }, response); + } + + private static void DispatchEvent(FfiEvent ffiEvent) + { #if LK_VERBOSE - if (r.MessageCase != FfiEvent.MessageOneofCase.Logs) - Utils.Debug("Callback: " + r.MessageCase); + if (ffiEvent.MessageCase != FfiEvent.MessageOneofCase.Logs) + Utils.Debug("Callback: " + ffiEvent.MessageCase); #endif - var requestAsyncId = ExtractRequestAsyncId(r); - if (requestAsyncId.HasValue && Instance.TryDispatchPendingCallback(requestAsyncId.Value, r)) - { - // Async request/response callbacks are one-shot. Once matched, they should not - // also flow through the general event switch below. - return; - } + var requestAsyncId = ExtractRequestAsyncId(ffiEvent); + if (requestAsyncId.HasValue && Instance.TryDispatchPendingCallback(requestAsyncId.Value, ffiEvent)) + { + // Async request/response callbacks are one-shot. Once matched, they should not + // also flow through the general event switch below. + return; + } - switch (r.MessageCase) - { - case FfiEvent.MessageOneofCase.Logs: - Utils.HandleLogBatch(r.Logs); - break; - case FfiEvent.MessageOneofCase.PublishData: - break; - case FfiEvent.MessageOneofCase.RoomEvent: - Instance.RoomEventReceived?.Invoke(r.RoomEvent); - break; - case FfiEvent.MessageOneofCase.TrackEvent: - Instance.TrackEventReceived?.Invoke(r.TrackEvent!); - break; - case FfiEvent.MessageOneofCase.RpcMethodInvocation: - Instance.RpcMethodInvocationReceived?.Invoke(r.RpcMethodInvocation); - break; - case FfiEvent.MessageOneofCase.Disconnect: - Instance.DisconnectReceived?.Invoke(r.Disconnect!); - break; - case FfiEvent.MessageOneofCase.PublishTranscription: - break; - case FfiEvent.MessageOneofCase.VideoStreamEvent: - Instance.VideoStreamEventReceived?.Invoke(r.VideoStreamEvent!); - break; - case FfiEvent.MessageOneofCase.AudioStreamEvent: - Instance.AudioStreamEventReceived?.Invoke(r.AudioStreamEvent!); - break; - // Uses high-level data stream interface - case FfiEvent.MessageOneofCase.ByteStreamReaderEvent: - Instance.ByteStreamReaderEventReceived?.Invoke(r.ByteStreamReaderEvent!); - break; - case FfiEvent.MessageOneofCase.TextStreamReaderEvent: - Instance.TextStreamReaderEventReceived?.Invoke(r.TextStreamReaderEvent!); - break; - case FfiEvent.MessageOneofCase.Panic: - break; - default: - break; - } - }, response); + switch (ffiEvent.MessageCase) + { + case FfiEvent.MessageOneofCase.Logs: + Utils.HandleLogBatch(ffiEvent.Logs); + break; + case FfiEvent.MessageOneofCase.PublishData: + break; + case FfiEvent.MessageOneofCase.RoomEvent: + Instance.RoomEventReceived?.Invoke(ffiEvent.RoomEvent); + break; + case FfiEvent.MessageOneofCase.TrackEvent: + Instance.TrackEventReceived?.Invoke(ffiEvent.TrackEvent!); + break; + case FfiEvent.MessageOneofCase.RpcMethodInvocation: + Instance.RpcMethodInvocationReceived?.Invoke(ffiEvent.RpcMethodInvocation); + break; + case FfiEvent.MessageOneofCase.Disconnect: + Instance.DisconnectReceived?.Invoke(ffiEvent.Disconnect!); + break; + case FfiEvent.MessageOneofCase.PublishTranscription: + break; + case FfiEvent.MessageOneofCase.VideoStreamEvent: + Instance.VideoStreamEventReceived?.Invoke(ffiEvent.VideoStreamEvent!); + break; + case FfiEvent.MessageOneofCase.AudioStreamEvent: + Instance.AudioStreamEventReceived?.Invoke(ffiEvent.AudioStreamEvent!); + break; + case FfiEvent.MessageOneofCase.ByteStreamReaderEvent: + Instance.ByteStreamReaderEventReceived?.Invoke(ffiEvent.ByteStreamReaderEvent!); + break; + case FfiEvent.MessageOneofCase.TextStreamReaderEvent: + Instance.TextStreamReaderEventReceived?.Invoke(ffiEvent.TextStreamReaderEvent!); + break; + case FfiEvent.MessageOneofCase.Panic: + break; + default: + break; + } } private bool TryDispatchPendingCallback(ulong requestAsyncId, FfiEvent ffiEvent) diff --git a/Runtime/Scripts/Internal/RingBuffer.cs b/Runtime/Scripts/Internal/RingBuffer.cs index c5cf2203..28ecd292 100644 --- a/Runtime/Scripts/Internal/RingBuffer.cs +++ b/Runtime/Scripts/Internal/RingBuffer.cs @@ -58,6 +58,11 @@ public int Read(Span data) return readCount; } + public int SkipRead(int len) + { + return MoveReadPtr(len); + } + private int MoveReadPtr(int len) { int free = AvailableWrite(); diff --git a/Runtime/Scripts/MicrophoneSource.cs b/Runtime/Scripts/MicrophoneSource.cs index 923df981..9de42561 100644 --- a/Runtime/Scripts/MicrophoneSource.cs +++ b/Runtime/Scripts/MicrophoneSource.cs @@ -100,11 +100,17 @@ private IEnumerator StopMicrophone() Microphone.End(_deviceName); var probe = _sourceObject.GetComponent(); - probe.AudioRead -= OnAudioRead; - UnityEngine.Object.Destroy(probe); + if (probe != null) + { + probe.AudioRead -= OnAudioRead; + UnityEngine.Object.Destroy(probe); + } var source = _sourceObject.GetComponent(); - UnityEngine.Object.Destroy(source); + if (source != null) + UnityEngine.Object.Destroy(source); + + Utils.Debug($"MicrophoneSource device='{_deviceName}' stopped"); yield return null; } diff --git a/Runtime/Scripts/RtcAudioSource.cs b/Runtime/Scripts/RtcAudioSource.cs index eb4fcae8..0cb5f779 100644 --- a/Runtime/Scripts/RtcAudioSource.cs +++ b/Runtime/Scripts/RtcAudioSource.cs @@ -1,11 +1,13 @@ using System; using System.Collections; +using System.Collections.Generic; using LiveKit.Proto; using LiveKit.Internal; using LiveKit.Internal.FFIClients.Requests; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using System.Diagnostics; +using System.Threading; namespace LiveKit { @@ -23,6 +25,18 @@ public enum RtcAudioSourceType /// public abstract class RtcAudioSource : IRtcSource, IDisposable { + private sealed class PendingAudioFrame + { + public NativeArray FrameData; + public int FrameIndex; + public int SampleRate; + public int Channels; + public int SampleCount; + public long StartedTimestamp; + } + + private static int nextDebugId = 0; + /// /// Event triggered when audio samples are captured from the underlying source. /// Provides the audio data, channel count, and sample rate. @@ -45,24 +59,34 @@ public abstract class RtcAudioSource : IRtcSource, IDisposable private readonly RtcAudioSourceType _sourceType; public RtcAudioSourceType SourceType => _sourceType; + private readonly int _debugId = Interlocked.Increment(ref nextDebugId); + private readonly uint _expectedSampleRate; + private readonly uint _expectedChannels; internal readonly FfiHandle Handle; protected AudioSourceInfo _info; - /// - /// Temporary frame buffer for invoking the FFI capture method. - /// - private NativeArray _frameData; + // CaptureAudioFrame is asynchronous: the native side can continue reading from the PCM + // pointer after request.Send() returns and encode it later on another queue. Because of + // that, a single reusable NativeArray is unsafe here; the next AudioRead callback can + // overwrite it while Opus/WebRTC is still consuming the previous frame. + // + // Keep one NativeArray per in-flight request and release it only after the matching + // CaptureAudioFrame callback completes or is canceled. + private readonly Dictionary _pendingFrameData = new(); + private readonly object _pendingFrameDataLock = new object(); private bool _muted = false; public override bool Muted => _muted; private bool _started = false; private bool _disposed = false; + private int _audioReadCount = 0; protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType = RtcAudioSourceType.AudioSourceCustom) { _sourceType = audioSourceType; + _expectedChannels = (uint)channels; using var request = FFIBridge.Instance.NewRequest(); var newAudioSource = request.request; @@ -70,6 +94,7 @@ protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType = newAudioSource.NumChannels = (uint)channels; newAudioSource.SampleRate = _sourceType == RtcAudioSourceType.AudioSourceMicrophone ? DefaultMicrophoneSampleRate : DefaultSampleRate; + _expectedSampleRate = newAudioSource.SampleRate; UnityEngine.Debug.Log($"NewAudioSource: {newAudioSource.NumChannels} {newAudioSource.SampleRate}"); @@ -81,6 +106,7 @@ protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType = FfiResponse res = response; _info = res.NewAudioSource.Source.Info; Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); + Utils.Debug($"{DebugTag} created handle={Handle.DangerousGetHandle()} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); } /// @@ -91,6 +117,7 @@ public virtual void Start() if (_started) return; AudioRead += OnAudioRead; _started = true; + Utils.Debug($"{DebugTag} start"); } /// @@ -101,19 +128,46 @@ public virtual void Stop() if (!_started) return; AudioRead -= OnAudioRead; _started = false; + var pendingCount = PendingFrameCount(); + if (pendingCount > 0) + Utils.Warning($"{DebugTag} stop requested with {pendingCount} pending capture callbacks"); + else + Utils.Debug($"{DebugTag} stop"); } private void OnAudioRead(float[] data, int channels, int sampleRate) { if (_muted) return; + if (_disposed) return; - // The length of the data buffer corresponds to the DSP buffer size. - if (_frameData.Length != data.Length) + var frameIndex = Interlocked.Increment(ref _audioReadCount); + if (channels <= 0) { - if (_frameData.IsCreated) _frameData.Dispose(); - _frameData = new NativeArray(data.Length, Allocator.Persistent); + Utils.Warning($"{DebugTag} dropping audio frame #{frameIndex} because channels={channels}"); + return; } + if (data.Length == 0 || data.Length % channels != 0) + { + Utils.Warning($"{DebugTag} audio frame #{frameIndex} has invalid shape samples={data.Length} channels={channels}"); + return; + } + + if ((uint)sampleRate != _expectedSampleRate || (uint)channels != _expectedChannels) + { + Utils.Warning($"{DebugTag} audio frame #{frameIndex} metadata mismatch actualRate={sampleRate} actualChannels={channels} expectedRate={_expectedSampleRate} expectedChannels={_expectedChannels} sourceType={_sourceType}"); + } + + var pendingBeforeSend = PendingFrameCount(); + if (frameIndex <= 3 || frameIndex % 100 == 0 || pendingBeforeSend >= 3) + { + Utils.Debug($"{DebugTag} capture frame #{frameIndex} samples={data.Length} channels={channels} sampleRate={sampleRate} pendingBeforeSend={pendingBeforeSend} thread={Thread.CurrentThread.ManagedThreadId}"); + } + + // Each captured frame gets its own backing buffer so the native encoder can safely + // consume it asynchronously after request.Send() returns. + var frameData = new NativeArray(data.Length, Allocator.Persistent); + // Copy from the audio read buffer into the frame buffer, converting // each sample to a 16-bit signed integer. static short FloatToS16(float v) @@ -124,7 +178,7 @@ static short FloatToS16(float v) return (short)(v + Math.Sign(v) * 0.5f); } for (int i = 0; i < data.Length; i++) - _frameData[i] = FloatToS16(data[i]); + frameData[i] = FloatToS16(data[i]); // Capture the frame. using var request = FFIBridge.Instance.NewRequest(); @@ -136,7 +190,7 @@ static short FloatToS16(float v) unsafe { pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility - .GetUnsafePtr(_frameData); + .GetUnsafePtr(frameData); } pushFrame.Buffer.NumChannels = (uint)channels; pushFrame.Buffer.SampleRate = (uint)sampleRate; @@ -145,14 +199,63 @@ static short FloatToS16(float v) // Wait for async callback, log an error if the capture fails. The callback's AsyncId // echoes the RequestAsyncId that Unity wrote onto the request. var requestAsyncId = request.RequestAsyncId; + var pendingFrame = new PendingAudioFrame + { + FrameData = frameData, + FrameIndex = frameIndex, + SampleRate = sampleRate, + Channels = channels, + SampleCount = data.Length, + StartedTimestamp = Stopwatch.GetTimestamp(), + }; + lock (_pendingFrameDataLock) + { + _pendingFrameData[requestAsyncId] = pendingFrame; + } + void Callback(CaptureAudioFrameCallback callback) { if (callback.AsyncId != requestAsyncId) return; + var completedFrame = ReleasePendingFrameData(requestAsyncId); + if (completedFrame != null) + { + var elapsedMs = ElapsedMilliseconds(completedFrame.StartedTimestamp); + if (callback.HasError) + { + Utils.Error($"{DebugTag} capture callback failed asyncId={requestAsyncId} frame={completedFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()} error={callback.Error}"); + } + else if (completedFrame.FrameIndex <= 3 || completedFrame.FrameIndex % 100 == 0 || elapsedMs > 100) + { + Utils.Debug($"{DebugTag} capture callback asyncId={requestAsyncId} frame={completedFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()}"); + } + } if (callback.HasError) - Utils.Error($"Audio capture failed: {callback.Error}"); + Utils.Error($"{DebugTag} audio capture failed: {callback.Error}"); + } + void OnCanceled() + { + var canceledFrame = ReleasePendingFrameData(requestAsyncId); + if (canceledFrame != null) + { + var elapsedMs = ElapsedMilliseconds(canceledFrame.StartedTimestamp); + Utils.Warning($"{DebugTag} capture callback canceled asyncId={requestAsyncId} frame={canceledFrame.FrameIndex} elapsedMs={elapsedMs:F1} pendingAfter={PendingFrameCount()}"); + } + } + + FfiClient.Instance.RegisterPendingCallback(requestAsyncId, static e => e.CaptureAudioFrame, Callback, OnCanceled); + try + { + using var response = request.Send(); + } + catch + { + var failedFrame = ReleasePendingFrameData(requestAsyncId); + if (failedFrame != null) + { + Utils.Error($"{DebugTag} request send failed asyncId={requestAsyncId} frame={failedFrame.FrameIndex} pendingAfter={PendingFrameCount()}"); + } + throw; } - FfiClient.Instance.RegisterPendingCallback(requestAsyncId, static e => e.CaptureAudioFrame, Callback); - using var response = request.Send(); } /// @@ -174,9 +277,48 @@ public void Dispose() protected virtual void Dispose(bool disposing) { - if (!_disposed && disposing) Stop(); - if (_frameData.IsCreated) _frameData.Dispose(); + if (_disposed) return; + + if (disposing) Stop(); + + var pendingCount = PendingFrameCount(); + if (pendingCount > 0) + Utils.Warning($"{DebugTag} dispose(disposing={disposing}) with {pendingCount} pending capture callbacks"); + + lock (_pendingFrameDataLock) + { + foreach (var pendingFrame in _pendingFrameData.Values) + { + if (pendingFrame.FrameData.IsCreated) + pendingFrame.FrameData.Dispose(); + } + _pendingFrameData.Clear(); + } _disposed = true; + Utils.Debug($"{DebugTag} disposed"); + } + + private PendingAudioFrame ReleasePendingFrameData(ulong requestAsyncId) + { + PendingAudioFrame pendingFrame = null; + lock (_pendingFrameDataLock) + { + if (_pendingFrameData.TryGetValue(requestAsyncId, out pendingFrame)) + _pendingFrameData.Remove(requestAsyncId); + } + + if (pendingFrame != null && pendingFrame.FrameData.IsCreated) + pendingFrame.FrameData.Dispose(); + + return pendingFrame; + } + + private int PendingFrameCount() + { + lock (_pendingFrameDataLock) + { + return _pendingFrameData.Count; + } } ~RtcAudioSource() @@ -193,5 +335,12 @@ public IEnumerator PrepareAndStart() Start(); yield break; } + + private static double ElapsedMilliseconds(long startedTimestamp) + { + return (Stopwatch.GetTimestamp() - startedTimestamp) * 1000.0 / Stopwatch.Frequency; + } + + private string DebugTag => $"RtcAudioSource#{_debugId}"; } } From 7b94b74fe8ce89dd514e14eb91c244a109635237 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 27 Mar 2026 13:15:11 -0700 Subject: [PATCH 6/6] fix the latency / audio glitches problems --- Editor/IosLinkOrderDiagnostics.cs | 255 +++++++++++++++++++++++++ Editor/IosLinkOrderDiagnostics.cs.meta | 11 ++ Runtime/Scripts/AudioStream.cs | 92 ++++++++- Runtime/Scripts/Internal/RingBuffer.cs | 11 ++ Runtime/Scripts/MicrophoneSource.cs | 139 ++++++++++++-- 5 files changed, 485 insertions(+), 23 deletions(-) create mode 100644 Editor/IosLinkOrderDiagnostics.cs create mode 100644 Editor/IosLinkOrderDiagnostics.cs.meta diff --git a/Editor/IosLinkOrderDiagnostics.cs b/Editor/IosLinkOrderDiagnostics.cs new file mode 100644 index 00000000..edc8e8e0 --- /dev/null +++ b/Editor/IosLinkOrderDiagnostics.cs @@ -0,0 +1,255 @@ +#if UNITY_EDITOR +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using UnityEditor; +using UnityEditor.Callbacks; +using UnityEngine; +using Process = System.Diagnostics.Process; +using ProcessStartInfo = System.Diagnostics.ProcessStartInfo; + +public static class IosLinkOrderDiagnostics +{ + private const string Prefix = "LiveKit"; + private const string Libilivekit = "liblivekit_ffi.a in Frameworks"; + private const string Libiphone = "libiPhone-lib.a in Frameworks"; + private static readonly string[] ConflictingLibiPhoneMembers = + { + "bands.o", + "celt.o", + "cwrs.o", + "entcode.o", + "entdec.o", + "entenc.o", + "header.o", + "kiss_fft.o", + "laplace.o", + "mathops.o", + "mdct-acf77e498fcf88b2edc7736c8f447bee8d7b174d050f1e9bd161576066636768.o", + "modes.o", + "pitch.o", + "plc.o", + "quant_bands.o", + "rate.o", + "vq.o", + "fmod_codec_celt.o" + }; + + [PostProcessBuild(999)] + public static void OnPostProcessBuild(BuildTarget target, string pathToBuiltProject) + { + if (target != BuildTarget.iOS) + return; + + var projectPath = Path.Combine(pathToBuiltProject, "Unity-iPhone.xcodeproj", "project.pbxproj"); + if (!File.Exists(projectPath)) + { + Debug.LogWarning($"{Prefix}: iOS link-order diagnostic could not find {projectPath}"); + return; + } + + var projectText = File.ReadAllText(projectPath); + var fixedProjectText = EnsureSafeUnityFrameworkLinkOrder(projectText, out var wasModified); + if (wasModified) + { + File.WriteAllText(projectPath, fixedProjectText); + projectText = fixedProjectText; + Debug.Log($"{Prefix}: iOS link-order fix applied. Moved libiPhone-lib.a after liblivekit_ffi.a in UnityFramework -> Frameworks and Libraries."); + } + + StripConflictingCodecObjects(pathToBuiltProject); + + var frameworkSection = ExtractUnityFrameworkSection(projectText); + if (string.IsNullOrEmpty(frameworkSection)) + { + Debug.LogWarning($"{Prefix}: iOS link-order diagnostic could not locate the UnityFramework frameworks section in the exported Xcode project."); + return; + } + + var livekitIndex = frameworkSection.IndexOf(Libilivekit, StringComparison.Ordinal); + var iphoneIndex = frameworkSection.IndexOf(Libiphone, StringComparison.Ordinal); + if (livekitIndex < 0 || iphoneIndex < 0) + { + Debug.LogWarning($"{Prefix}: iOS link-order diagnostic could not locate both {Libilivekit} and {Libiphone} in UnityFramework -> Frameworks and Libraries."); + return; + } + + if (livekitIndex < iphoneIndex) + { + Debug.Log($"{Prefix}: iOS link-order diagnostic OK. UnityFramework links liblivekit_ffi.a before libiPhone-lib.a."); + return; + } + + Debug.LogWarning( + $"{Prefix}: iOS link-order diagnostic found a risky order in UnityFramework -> Frameworks and Libraries. " + + "libiPhone-lib.a appears before liblivekit_ffi.a. This repo documents that this can crash Opus/CELT on iOS. " + + "The post-process fix could not rewrite the exported project automatically." + ); + } + + private static string EnsureSafeUnityFrameworkLinkOrder(string projectText, out bool wasModified) + { + wasModified = false; + + if (!TryGetUnityFrameworkSectionBounds(projectText, out var sectionStart, out var sectionEnd)) + return projectText; + + var frameworkSection = projectText.Substring(sectionStart, sectionEnd - sectionStart); + var livekitIndex = frameworkSection.IndexOf(Libilivekit, StringComparison.Ordinal); + var iphoneIndex = frameworkSection.IndexOf(Libiphone, StringComparison.Ordinal); + if (livekitIndex < 0 || iphoneIndex < 0 || livekitIndex < iphoneIndex) + return projectText; + + var iphoneLineStart = frameworkSection.LastIndexOf('\n', iphoneIndex); + if (iphoneLineStart < 0) + iphoneLineStart = 0; + else + iphoneLineStart += 1; + + var iphoneLineEnd = frameworkSection.IndexOf('\n', iphoneIndex); + if (iphoneLineEnd < 0) + iphoneLineEnd = frameworkSection.Length; + else + iphoneLineEnd += 1; + + var iphoneLine = frameworkSection.Substring(iphoneLineStart, iphoneLineEnd - iphoneLineStart); + var sectionWithoutIphone = frameworkSection.Remove(iphoneLineStart, iphoneLineEnd - iphoneLineStart); + + var livekitIndexAfterRemoval = sectionWithoutIphone.IndexOf(Libilivekit, StringComparison.Ordinal); + if (livekitIndexAfterRemoval < 0) + return projectText; + + var insertIndex = sectionWithoutIphone.IndexOf('\n', livekitIndexAfterRemoval); + if (insertIndex < 0) + return projectText; + + insertIndex += 1; + var reorderedSection = sectionWithoutIphone.Insert(insertIndex, iphoneLine); + wasModified = !string.Equals(frameworkSection, reorderedSection, StringComparison.Ordinal); + if (!wasModified) + return projectText; + + return projectText.Substring(0, sectionStart) + reorderedSection + projectText.Substring(sectionEnd); + } + + private static string ExtractUnityFrameworkSection(string projectText) + { + if (!TryGetUnityFrameworkSectionBounds(projectText, out var sectionStart, out var sectionEnd)) + return string.Empty; + + return projectText.Substring(sectionStart, sectionEnd - sectionStart); + } + + private static bool TryGetUnityFrameworkSectionBounds(string projectText, out int sectionStart, out int sectionEnd) + { + sectionStart = -1; + sectionEnd = -1; + + const string marker = "/* UnityFramework */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;"; + var markerIndex = projectText.IndexOf(marker, StringComparison.Ordinal); + if (markerIndex < 0) + return false; + + var filesIndex = projectText.IndexOf("\t\t\tfiles = (", markerIndex, StringComparison.Ordinal); + if (filesIndex < 0) + return false; + + var endIndex = projectText.IndexOf("\t\t\t);", filesIndex, StringComparison.Ordinal); + if (endIndex < 0) + return false; + + sectionStart = filesIndex; + sectionEnd = endIndex; + return true; + } + + private static void StripConflictingCodecObjects(string pathToBuiltProject) + { + var archivePath = Path.Combine(pathToBuiltProject, "Libraries", "libiPhone-lib.a"); + if (!File.Exists(archivePath)) + { + Debug.Log($"{Prefix}: iOS archive fix skipped because {archivePath} was not found."); + return; + } + + if (!TryRunProcess("/usr/bin/ar", $" -t \"{archivePath}\"", out var memberOutput, out var listError)) + { + Debug.LogWarning($"{Prefix}: iOS archive fix could not inspect libiPhone-lib.a members. {listError}"); + return; + } + + var members = new HashSet( + memberOutput + .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .Select(line => line.Trim()), + StringComparer.Ordinal + ); + + var membersToRemove = ConflictingLibiPhoneMembers.Where(members.Contains).ToArray(); + if (membersToRemove.Length == 0) + { + Debug.Log($"{Prefix}: iOS archive fix found no conflicting CELT objects in libiPhone-lib.a."); + return; + } + + var deleteArguments = $" -d \"{archivePath}\" {string.Join(" ", membersToRemove.Select(QuoteArgument))}"; + if (!TryRunProcess("/usr/bin/ar", deleteArguments, out _, out var deleteError)) + { + Debug.LogWarning($"{Prefix}: iOS archive fix could not strip conflicting objects from libiPhone-lib.a. {deleteError}"); + return; + } + + if (!TryRunProcess("/usr/bin/ranlib", $" \"{archivePath}\"", out _, out var ranlibError)) + { + Debug.LogWarning($"{Prefix}: iOS archive fix stripped conflicting objects but ranlib failed. {ranlibError}"); + return; + } + + Debug.Log($"{Prefix}: iOS archive fix stripped {membersToRemove.Length} conflicting CELT objects from exported libiPhone-lib.a."); + } + + private static bool TryRunProcess(string fileName, string arguments, out string stdout, out string error) + { + stdout = string.Empty; + error = string.Empty; + + try + { + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + process.Start(); + stdout = process.StandardOutput.ReadToEnd(); + var stderr = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (process.ExitCode == 0) + return true; + + error = string.IsNullOrWhiteSpace(stderr) + ? $"{fileName} exited with code {process.ExitCode}." + : stderr.Trim(); + return false; + } + catch (Exception ex) + { + error = ex.Message; + return false; + } + } + + private static string QuoteArgument(string value) + { + return $"\"{value.Replace("\"", "\\\"")}\""; + } +} +#endif diff --git a/Editor/IosLinkOrderDiagnostics.cs.meta b/Editor/IosLinkOrderDiagnostics.cs.meta new file mode 100644 index 00000000..fe9b46b7 --- /dev/null +++ b/Editor/IosLinkOrderDiagnostics.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 342b5834197304d18a570f0a229755c3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/AudioStream.cs b/Runtime/Scripts/AudioStream.cs index ccc028ae..36c96ec4 100644 --- a/Runtime/Scripts/AudioStream.cs +++ b/Runtime/Scripts/AudioStream.cs @@ -24,6 +24,11 @@ public sealed class AudioStream : IDisposable private readonly object _lock = new object(); private bool _disposed = false; + // Pre-buffering state to prevent audio underruns + private bool _isPrimed = false; + private const float BufferSizeSeconds = 0.2f; // 200ms ring buffer for all platforms + private const float PrimingThresholdSeconds = 0.03f; // Wait for 30ms of data before playing + /// /// Creates a new audio stream from a remote audio track, attaching it to the /// given in the scene. @@ -54,6 +59,9 @@ public AudioStream(RemoteAudioTrack audioTrack, AudioSource source) _probe = _audioSource.gameObject.AddComponent(); _probe.AudioRead += OnAudioRead; _audioSource.Play(); + + // Subscribe to application pause events to handle background/foreground transitions + MonoBehaviourContext.OnApplicationPauseEvent += OnApplicationPause; } // Called on Unity audio thread @@ -67,14 +75,19 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) lock (_lock) { + // Initialize or reinitialize buffer if audio format changed if (_buffer == null || channels != _numChannels || sampleRate != _sampleRate || data.Length != _tempBuffer.Length) { - int size = (int)(channels * sampleRate * 0.2); + // Always use 200ms ring buffer for all platforms + int bufferSize = (int)(channels * sampleRate * BufferSizeSeconds); _buffer?.Dispose(); - _buffer = new RingBuffer(size * sizeof(short)); + _buffer = new RingBuffer(bufferSize * sizeof(short)); _tempBuffer = new short[data.Length]; _numChannels = (uint)channels; _sampleRate = (uint)sampleRate; + + // Buffer was recreated, need to re-prime + _isPrimed = false; } static float S16ToFloat(short v) @@ -82,18 +95,86 @@ static float S16ToFloat(short v) return v / 32768f; } - // "Send" the data to Unity + // Check if we have enough data in the buffer to start/continue playback. + // Pre-buffering strategy: wait for 30ms of data before playing to avoid underruns. + int primingThresholdBytes = (int)(channels * sampleRate * PrimingThresholdSeconds * sizeof(short)); + int availableBytes = _buffer.AvailableRead(); + + if (!_isPrimed) + { + // Not yet primed - check if we have enough data to start playing + if (availableBytes >= primingThresholdBytes) + { + _isPrimed = true; + Utils.Debug($"AudioStream primed with {availableBytes} bytes ({availableBytes / (channels * sampleRate * sizeof(short)) * 1000f:F1}ms)"); + } + else + { + // Not enough data yet, output silence and wait + Array.Clear(data, 0, data.Length); + return; + } + } + + // Try to read audio samples from the ring buffer into our temp buffer. + // The ring buffer acts as a jitter buffer between: + // - Rust FFI pushing frames (on main thread, with network timing) + // - Unity audio thread pulling samples (real-time, consistent timing) var temp = MemoryMarshal.Cast(_tempBuffer.AsSpan().Slice(0, data.Length)); - int read = _buffer.Read(temp); + int bytesRead = _buffer.Read(temp); + + // Calculate how many samples (shorts) were actually read from the ring buffer. + // If the buffer is empty or doesn't have enough data, bytesRead will be less than requested. + int samplesRead = bytesRead / sizeof(short); + // Underrun detection: If we couldn't read enough samples, immediately output silence + // and wait for the buffer to refill to 30ms before resuming playback. + // This prevents choppy audio from playing partial samples during underrun. + if (samplesRead < data.Length * 0.5f) // If we got less than 50% of requested samples + { + _isPrimed = false; + Utils.Debug($"AudioStream underrun detected, re-priming (got {samplesRead}/{data.Length} samples)"); + + // Output silence immediately instead of playing partial/choppy samples. + // On next frames, the !_isPrimed check above will ensure we wait for 30ms + // of data before resuming playback smoothly. + Array.Clear(data, 0, data.Length); + return; + } + + // Clear the entire output buffer to silence, then fill with the samples + // we successfully read from the ring buffer. Array.Clear(data, 0, data.Length); - for (int i = 0; i < data.Length; i++) + for (int i = 0; i < samplesRead; i++) { data[i] = S16ToFloat(_tempBuffer[i]); } } } + // Called when application goes to background or returns to foreground + private void OnApplicationPause(bool pause) + { + if (_disposed) + return; + + // When returning from background, clear the ring buffer and reset priming state. + // This ensures we don't play stale audio data and forces the stream to wait + // for fresh data (30ms) before resuming playback, preventing audio glitches. + if (!pause) // Returning to foreground + { + lock (_lock) + { + if (_buffer != null) + { + _buffer.Clear(); + _isPrimed = false; + Utils.Debug("AudioStream cleared buffer on app resume, waiting to re-prime"); + } + } + } + } + // Called on the MainThread (See FfiClient) private void OnAudioStreamEvent(AudioStreamEvent e) { @@ -143,6 +224,7 @@ private void Dispose(bool disposing) // as soon as user code drops it. This also prevents late native callbacks from // touching partially disposed state. FfiClient.Instance.AudioStreamEventReceived -= OnAudioStreamEvent; + MonoBehaviourContext.OnApplicationPauseEvent -= OnApplicationPause; lock (_lock) { diff --git a/Runtime/Scripts/Internal/RingBuffer.cs b/Runtime/Scripts/Internal/RingBuffer.cs index 28ecd292..2a07b1be 100644 --- a/Runtime/Scripts/Internal/RingBuffer.cs +++ b/Runtime/Scripts/Internal/RingBuffer.cs @@ -130,6 +130,17 @@ public int AvailableWrite() return _buffer.Length - AvailableRead(); } + /// + /// Clears all data from the ring buffer, resetting read and write positions. + /// Useful when resuming from background to discard stale audio data. + /// + public void Clear() + { + _writePos = 0; + _readPos = 0; + _sameWrap = true; + } + public void Dispose() { _buffer.Dispose(); diff --git a/Runtime/Scripts/MicrophoneSource.cs b/Runtime/Scripts/MicrophoneSource.cs index 9de42561..904b8da7 100644 --- a/Runtime/Scripts/MicrophoneSource.cs +++ b/Runtime/Scripts/MicrophoneSource.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using UnityEngine; +using LiveKit.Internal; namespace LiveKit { @@ -60,14 +61,55 @@ public override void Start() private IEnumerator StartMicrophone() { - var clip = Microphone.Start( - _deviceName, - loop: true, - lengthSec: 1, - frequency: (int)DefaultMicrophoneSampleRate - ); + // Validate that the GameObject is still valid before starting + if (_sourceObject == null) + { + Utils.Error("MicrophoneSource: GameObject is null, cannot start microphone"); + yield break; + } + + // Verify microphone is still authorized (could change during background) + if (!Application.HasUserAuthorization(UserAuthorization.Microphone)) + { + Utils.Error("MicrophoneSource: Microphone authorization lost"); + yield break; + } + + AudioClip clip = null; + try + { + clip = Microphone.Start( + _deviceName, + loop: true, + lengthSec: 1, + frequency: (int)DefaultMicrophoneSampleRate + ); + } + catch (Exception e) + { + Utils.Error($"MicrophoneSource: Exception starting microphone: {e.Message}"); + yield break; + } + if (clip == null) - throw new InvalidOperationException("Microphone start failed"); + { + Utils.Error("MicrophoneSource: Microphone.Start returned null, audio session may not be ready"); + yield break; + } + + // Ensure no duplicate components exist before adding new ones. + // This is important during app resume on iOS where components might not be + // fully destroyed yet due to Unity's deferred Destroy(). + var existingSource = _sourceObject.GetComponent(); + if (existingSource != null) + UnityEngine.Object.DestroyImmediate(existingSource); + + var existingProbe = _sourceObject.GetComponent(); + if (existingProbe != null) + { + existingProbe.AudioRead -= OnAudioRead; + UnityEngine.Object.DestroyImmediate(existingProbe); + } var source = _sourceObject.AddComponent(); source.clip = clip; @@ -78,9 +120,23 @@ private IEnumerator StartMicrophone() probe.ClearAfterInvocation(); probe.AudioRead += OnAudioRead; - var waitUntilReady = new WaitUntil(() => Microphone.GetPosition(_deviceName) > 0); - yield return waitUntilReady; + // Wait for microphone to actually start producing data with a timeout + const float timeout = 2f; + float elapsed = 0f; + while (Microphone.GetPosition(_deviceName) <= 0 && elapsed < timeout) + { + yield return new WaitForSeconds(0.05f); + elapsed += 0.05f; + } + + if (Microphone.GetPosition(_deviceName) <= 0) + { + Utils.Error($"MicrophoneSource: Microphone did not start producing data after {timeout}s"); + yield break; + } + source.Play(); + Utils.Debug($"MicrophoneSource device='{_deviceName}' started successfully"); } /// @@ -99,17 +155,21 @@ private IEnumerator StopMicrophone() if (Microphone.IsRecording(_deviceName)) Microphone.End(_deviceName); - var probe = _sourceObject.GetComponent(); - if (probe != null) + // Check if GameObject is still valid before trying to access components + if (_sourceObject != null) { - probe.AudioRead -= OnAudioRead; - UnityEngine.Object.Destroy(probe); + var probe = _sourceObject.GetComponent(); + if (probe != null) + { + probe.AudioRead -= OnAudioRead; + UnityEngine.Object.Destroy(probe); + } + + var source = _sourceObject.GetComponent(); + if (source != null) + UnityEngine.Object.Destroy(source); } - var source = _sourceObject.GetComponent(); - if (source != null) - UnityEngine.Object.Destroy(source); - Utils.Debug($"MicrophoneSource device='{_deviceName}' stopped"); yield return null; } @@ -121,16 +181,59 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) private void OnApplicationPause(bool pause) { - if (!pause && _started) + if (!_started) + return; + + if (pause) + { + // On iOS, when app goes to background, we should stop using audio resources + // to avoid AVAudioSession interruption errors (FigCaptureSourceRemote -17281) + MonoBehaviourContext.RunCoroutine(StopMicrophone()); + } + else + { + // When resuming, restart the microphone MonoBehaviourContext.RunCoroutine(RestartMicrophone()); + } } private IEnumerator RestartMicrophone() { yield return StopMicrophone(); + + // Wait for iOS audio session to be ready before attempting to restart. + // On iOS, after app resumes from background, the audio session needs time to + // recover from interruption. Poll for readiness instead of using arbitrary delay. + yield return WaitForMicrophoneReady(); + yield return StartMicrophone(); } + private IEnumerator WaitForMicrophoneReady() + { + // Wait for microphone devices to become available again after iOS audio session interruption. + // This is more reliable than a fixed delay because we wait for actual system readiness. + const float timeout = 2f; + float elapsed = 0f; + + // On iOS, Microphone.devices may be empty immediately after resume while + // AVAudioSession is recovering from interruption. Wait until devices are available. + while (Microphone.devices.Length == 0 && elapsed < timeout) + { + yield return new WaitForSeconds(0.05f); + elapsed += 0.05f; + } + + if (Microphone.devices.Length == 0) + { + Utils.Error($"MicrophoneSource: Microphone devices not available after {timeout}s timeout"); + yield break; + } + + // Extra frame to ensure audio session is fully ready + yield return null; + } + protected override void Dispose(bool disposing) { if (!_disposed && disposing) Stop();