From dca568d6fde55d08ca7457d69b035a4b5b738dc6 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:26:05 +0100 Subject: [PATCH 1/6] Working latency tests for connection and audio --- Tests/PlayMode/LatencyTests.cs | 366 +++++++++++++++++++ Tests/PlayMode/LatencyTests.cs.meta | 11 + Tests/PlayMode/Utils/EnergyDetector.cs | 33 ++ Tests/PlayMode/Utils/EnergyDetector.cs.meta | 11 + Tests/PlayMode/Utils/TestAudioSource.cs | 25 ++ Tests/PlayMode/Utils/TestAudioSource.cs.meta | 11 + Tests/PlayMode/Utils/TestRoomContext.cs | 3 +- 7 files changed, 459 insertions(+), 1 deletion(-) create mode 100644 Tests/PlayMode/LatencyTests.cs create mode 100644 Tests/PlayMode/LatencyTests.cs.meta create mode 100644 Tests/PlayMode/Utils/EnergyDetector.cs create mode 100644 Tests/PlayMode/Utils/EnergyDetector.cs.meta create mode 100644 Tests/PlayMode/Utils/TestAudioSource.cs create mode 100644 Tests/PlayMode/Utils/TestAudioSource.cs.meta diff --git a/Tests/PlayMode/LatencyTests.cs b/Tests/PlayMode/LatencyTests.cs new file mode 100644 index 00000000..018acb25 --- /dev/null +++ b/Tests/PlayMode/LatencyTests.cs @@ -0,0 +1,366 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.TestTools; +using LiveKit.Proto; +using LiveKit.PlayModeTests.Utils; +using Debug = UnityEngine.Debug; + +namespace LiveKit.PlayModeTests +{ + public class LatencyTests + { + // Audio configuration (matching C++ test) + const int kAudioSampleRate = 48000; + const int kAudioChannels = 1; + const int kAudioFrameDurationMs = 10; + const int kSamplesPerFrame = kAudioSampleRate * kAudioFrameDurationMs / 1000; // 480 + + // Energy detection + const double kHighEnergyThreshold = 0.3; + const int kHighEnergyFramesPerPulse = 5; + + // Test parameters + const int kTotalPulses = 10; + const int kFramesBetweenPulses = 100; // ~1 second at 10ms/frame + const float kEchoTimeoutSeconds = 2f; + const int kConnectionTestIterations = 5; + + // ===================================================================== + // Test 1: Connection Time Measurement + // ===================================================================== + + [UnityTest, Category("E2E")] + public IEnumerator ConnectionTime() + { + Debug.Log("\n=== Connection Time Measurement Test ==="); + Debug.Log($"Iterations: {kConnectionTestIterations}"); + + var stats = new LatencyStats(); + + for (int i = 0; i < kConnectionTestIterations; i++) + { + using var context = new TestRoomContext(); + var sw = Stopwatch.StartNew(); + yield return context.ConnectAll(); + sw.Stop(); + + if (context.ConnectionError != null) + { + Debug.Log($" Iteration {i + 1}: FAILED to connect - {context.ConnectionError}"); + continue; + } + + double latencyMs = sw.Elapsed.TotalMilliseconds; + stats.AddMeasurement(latencyMs); + Debug.Log($" Iteration {i + 1}: {latencyMs:F2} ms"); + + // Small delay between iterations + yield return new WaitForSeconds(0.5f); + } + + stats.PrintStats("Connection Time Statistics"); + Assert.Greater(stats.Count, 0, "At least one connection should succeed"); + } + + // ===================================================================== + // Test 2: Audio Latency Measurement using Energy Detection + // ===================================================================== + + [UnityTest, Category("E2E")] + public IEnumerator AudioLatency() + { + Debug.Log("\n=== Audio Latency Measurement Test ==="); + Debug.Log("Using energy detection to measure audio round-trip latency"); + + // --- Connect two participants --- + var sender = TestRoomContext.ConnectionOptions.Default; + sender.Identity = "sender"; + var receiver = TestRoomContext.ConnectionOptions.Default; + receiver.Identity = "receiver"; + + using var context = new TestRoomContext(new[] { receiver, sender }); + yield return context.ConnectAll(); + if (context.ConnectionError != null) + Assert.Fail(context.ConnectionError); + + var receiverRoom = context.Rooms[0]; + var senderRoom = context.Rooms[1]; + Debug.Log($"Receiver connected as: {receiverRoom.LocalParticipant.Identity}"); + Debug.Log($"Sender connected as: {senderRoom.LocalParticipant.Identity}"); + + // --- Wait for sender to be visible to receiver --- + var participantExpectation = new Expectation(timeoutSeconds: 10f); + if (receiverRoom.RemoteParticipants.ContainsKey(sender.Identity)) + { + participantExpectation.Fulfill(); + } + else + { + receiverRoom.ParticipantConnected += (p) => + { + if (p.Identity == sender.Identity) + participantExpectation.Fulfill(); + }; + } + yield return participantExpectation.Wait(); + if (participantExpectation.Error != null) + Assert.Fail($"Sender not visible to receiver: {participantExpectation.Error}"); + + // --- Create and publish audio track from sender --- + var audioSource = new TestAudioSource(channels: kAudioChannels); + audioSource.Start(); + + var audioTrack = LocalAudioTrack.CreateAudioTrack("latency-test", audioSource, senderRoom); + var publishOptions = new TrackPublishOptions(); + var publish = senderRoom.LocalParticipant.PublishTrack(audioTrack, publishOptions); + yield return publish; + if (publish.IsError) + Assert.Fail("Failed to publish audio track"); + + Debug.Log("Audio track published, waiting for subscription..."); + + // --- Wait for receiver to subscribe to the audio track --- + RemoteAudioTrack subscribedTrack = null; + var trackExpectation = new Expectation(timeoutSeconds: 10f); + receiverRoom.TrackSubscribed += (track, publication, participant) => + { + if (track is RemoteAudioTrack rat && participant.Identity == sender.Identity) + { + subscribedTrack = rat; + trackExpectation.Fulfill(); + } + }; + yield return trackExpectation.Wait(); + if (trackExpectation.Error != null) + Assert.Fail($"Receiver did not subscribe to audio track: {trackExpectation.Error}"); + + Debug.Log("Audio track subscribed, creating audio stream..."); + + // --- Set up receiver audio stream + energy detector --- + var listenerGO = new GameObject("LatencyTestAudioListener"); + listenerGO.AddComponent(); // Required for OnAudioFilterRead to fire + + var receiverGO = new GameObject("LatencyTestReceiver"); + var unityAudioSource = receiverGO.AddComponent(); + unityAudioSource.spatialBlend = 0f; // 2D audio + unityAudioSource.volume = 1f; + + // AudioStream constructor adds AudioProbe and starts playback + var audioStream = new AudioStream(subscribedTrack, unityAudioSource); + + // Add EnergyDetector AFTER AudioStream so OnAudioFilterRead order is correct + var energyDetector = receiverGO.AddComponent(); + + // --- Shared state for latency measurement --- + var stats = new LatencyStats(); + var lockObj = new object(); + long lastHighEnergySendTimeTicks = 0; + bool waitingForEcho = false; + int missedPulses = 0; + + // Energy detector callback (runs on audio thread) + energyDetector.EnergyDetected += (energy) => + { + lock (lockObj) + { + if (!waitingForEcho || energy <= kHighEnergyThreshold) + return; + + long receiveTimeTicks = Stopwatch.GetTimestamp(); + long sendTimeTicks = lastHighEnergySendTimeTicks; + + if (sendTimeTicks > 0) + { + double latencyMs = (receiveTimeTicks - sendTimeTicks) + * 1000.0 / Stopwatch.Frequency; + + if (latencyMs > 0 && latencyMs < 5000) + { + stats.AddMeasurement(latencyMs); + Debug.Log($"Audio latency: {latencyMs:F2} ms (energy: {energy:F3})"); + } + waitingForEcho = false; + } + } + }; + + // --- Send audio frames in real-time --- + Debug.Log("Starting audio pulse transmission..."); + + int frameCount = 0; + int pulsesSent = 0; + int highEnergyFramesRemaining = 0; + var frameDuration = TimeSpan.FromMilliseconds(kAudioFrameDurationMs); + var nextFrameTime = Stopwatch.GetTimestamp(); + + while (pulsesSent < kTotalPulses) + { + // Wait until it's time to send the next frame + long now = Stopwatch.GetTimestamp(); + double waitMs = (nextFrameTime - now) * 1000.0 / Stopwatch.Frequency; + if (waitMs > 1.0) + { + yield return null; // yield to Unity, come back next frame + continue; + } + nextFrameTime += (long)(frameDuration.TotalSeconds * Stopwatch.Frequency); + + float[] frameData; + + // Check for echo timeout + lock (lockObj) + { + if (waitingForEcho && lastHighEnergySendTimeTicks > 0) + { + double elapsedMs = (Stopwatch.GetTimestamp() - lastHighEnergySendTimeTicks) + * 1000.0 / Stopwatch.Frequency; + if (elapsedMs > kEchoTimeoutSeconds * 1000) + { + Debug.Log($" Echo timeout for pulse {pulsesSent}, moving on..."); + waitingForEcho = false; + missedPulses++; + } + } + } + + if (highEnergyFramesRemaining > 0) + { + // Continue sending high-energy frames for current pulse + frameData = GenerateHighEnergyFrame(kSamplesPerFrame); + highEnergyFramesRemaining--; + } + else + { + bool shouldPulse; + lock (lockObj) { shouldPulse = !waitingForEcho; } + + if (frameCount % kFramesBetweenPulses == 0 && shouldPulse) + { + // Start a new pulse + frameData = GenerateHighEnergyFrame(kSamplesPerFrame); + highEnergyFramesRemaining = kHighEnergyFramesPerPulse - 1; + + lock (lockObj) + { + lastHighEnergySendTimeTicks = Stopwatch.GetTimestamp(); + waitingForEcho = true; + } + pulsesSent++; + Debug.Log($"Sent pulse {pulsesSent}/{kTotalPulses} ({kHighEnergyFramesPerPulse} frames)"); + } + else + { + frameData = GenerateSilentFrame(kSamplesPerFrame); + } + } + + audioSource.PushFrame(frameData, kAudioChannels, kAudioSampleRate); + frameCount++; + } + + // Wait for last echo + yield return new WaitForSeconds(kEchoTimeoutSeconds); + + // --- Print results --- + stats.PrintStats("Audio Latency Statistics"); + if (missedPulses > 0) + Debug.Log($"Missed pulses (timeout): {missedPulses}"); + + // --- Cleanup --- + audioStream.Dispose(); + UnityEngine.Object.Destroy(receiverGO); + UnityEngine.Object.Destroy(listenerGO); + audioSource.Stop(); + audioSource.Dispose(); + + Assert.Greater(stats.Count, 0, "At least one audio latency measurement should be recorded"); + } + + // ===================================================================== + // Audio Generation Helpers + // ===================================================================== + + static float[] GenerateHighEnergyFrame(int samplesPerChannel) + { + var data = new float[samplesPerChannel * kAudioChannels]; + const double frequency = 1000.0; // 1kHz sine wave + const double amplitude = 0.9; + + for (int i = 0; i < samplesPerChannel; i++) + { + double t = (double)i / kAudioSampleRate; + float sample = (float)(amplitude * Math.Sin(2.0 * Math.PI * frequency * t)); + for (int ch = 0; ch < kAudioChannels; ch++) + { + data[i * kAudioChannels + ch] = sample; + } + } + return data; + } + + static float[] GenerateSilentFrame(int samplesPerChannel) + { + return new float[samplesPerChannel * kAudioChannels]; + } + + // ===================================================================== + // Latency Stats Helper + // ===================================================================== + + class LatencyStats + { + readonly List _measurements = new(); + readonly object _lock = new(); + + public int Count + { + get { lock (_lock) return _measurements.Count; } + } + + public void AddMeasurement(double ms) + { + lock (_lock) _measurements.Add(ms); + } + + public void PrintStats(string title) + { + lock (_lock) + { + Debug.Log($"\n--- {title} ---"); + if (_measurements.Count == 0) + { + Debug.Log(" No measurements recorded"); + return; + } + + double min = double.MaxValue, max = double.MinValue, sum = 0; + foreach (var m in _measurements) + { + if (m < min) min = m; + if (m > max) max = m; + sum += m; + } + double avg = sum / _measurements.Count; + + double sumSqDiff = 0; + foreach (var m in _measurements) + { + double diff = m - avg; + sumSqDiff += diff * diff; + } + double stddev = Math.Sqrt(sumSqDiff / _measurements.Count); + + Debug.Log($" Count: {_measurements.Count}"); + Debug.Log($" Min: {min:F2} ms"); + Debug.Log($" Max: {max:F2} ms"); + Debug.Log($" Avg: {avg:F2} ms"); + Debug.Log($" StdDev: {stddev:F2} ms"); + } + } + } + } +} diff --git a/Tests/PlayMode/LatencyTests.cs.meta b/Tests/PlayMode/LatencyTests.cs.meta new file mode 100644 index 00000000..93bb7a7c --- /dev/null +++ b/Tests/PlayMode/LatencyTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b9c51a221ae8a4c10bf1c744b06b7fd0 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/EnergyDetector.cs b/Tests/PlayMode/Utils/EnergyDetector.cs new file mode 100644 index 00000000..aad4f8f6 --- /dev/null +++ b/Tests/PlayMode/Utils/EnergyDetector.cs @@ -0,0 +1,33 @@ +using System; +using UnityEngine; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// MonoBehaviour that detects audio energy via OnAudioFilterRead. + /// Attach to the same GameObject as an AudioSource after AudioStream + /// has added its AudioProbe, so this filter runs second and sees + /// the filled audio data. + /// + public class EnergyDetector : MonoBehaviour + { + /// + /// Fired on the audio thread when non-trivial energy is detected. + /// The double parameter is the RMS energy of the frame. + /// + public event Action EnergyDetected; + + void OnAudioFilterRead(float[] data, int channels) + { + double sumSquared = 0.0; + for (int i = 0; i < data.Length; i++) + { + sumSquared += data[i] * data[i]; + } + double rms = Math.Sqrt(sumSquared / data.Length); + + if (rms > 0.01) + EnergyDetected?.Invoke(rms); + } + } +} diff --git a/Tests/PlayMode/Utils/EnergyDetector.cs.meta b/Tests/PlayMode/Utils/EnergyDetector.cs.meta new file mode 100644 index 00000000..0eb5cde0 --- /dev/null +++ b/Tests/PlayMode/Utils/EnergyDetector.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0bb4d155287b94d16954c12e27c58937 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/TestAudioSource.cs b/Tests/PlayMode/Utils/TestAudioSource.cs new file mode 100644 index 00000000..a11abef6 --- /dev/null +++ b/Tests/PlayMode/Utils/TestAudioSource.cs @@ -0,0 +1,25 @@ +using System; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// A programmatic audio source for testing. Allows pushing audio frames + /// directly without requiring a Unity AudioSource or microphone. + /// + public class TestAudioSource : RtcAudioSource + { + public override event Action AudioRead; + + public TestAudioSource(int channels = 1) + : base(channels, RtcAudioSourceType.AudioSourceCustom) { } + + /// + /// Push an audio frame into the FFI capture pipeline. + /// Must call Start() first so the base class is subscribed to AudioRead. + /// + public void PushFrame(float[] data, int channels, int sampleRate) + { + AudioRead?.Invoke(data, channels, sampleRate); + } + } +} diff --git a/Tests/PlayMode/Utils/TestAudioSource.cs.meta b/Tests/PlayMode/Utils/TestAudioSource.cs.meta new file mode 100644 index 00000000..6fff68ea --- /dev/null +++ b/Tests/PlayMode/Utils/TestAudioSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: eceb212ad967a4943bb7c60bcd12a038 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/TestRoomContext.cs b/Tests/PlayMode/Utils/TestRoomContext.cs index e3227230..e116d393 100644 --- a/Tests/PlayMode/Utils/TestRoomContext.cs +++ b/Tests/PlayMode/Utils/TestRoomContext.cs @@ -7,7 +7,8 @@ namespace LiveKit.PlayModeTests.Utils { public class TestRoomContext : IDisposable { - public readonly string RoomName = $"unity-test-{Guid.NewGuid()}"; + // public readonly string RoomName = $"unity-test-{Guid.NewGuid()}"; + public readonly string RoomName = $"unity-test-max"; public List Rooms { get; private set; } = new List(); public string? ConnectionError { get; private set; } From 95afbb4029d0f8e5bb86c989005b45f362732df5 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:57:54 +0100 Subject: [PATCH 2/6] Encoding the id of the pulse in the frequency --- Tests/PlayMode/LatencyTests.cs | 126 +++++++++--------- Tests/PlayMode/Utils/EnergyDetector.cs | 33 ----- Tests/PlayMode/Utils/PulseDetector.cs | 89 +++++++++++++ ...Detector.cs.meta => PulseDetector.cs.meta} | 2 +- 4 files changed, 150 insertions(+), 100 deletions(-) delete mode 100644 Tests/PlayMode/Utils/EnergyDetector.cs create mode 100644 Tests/PlayMode/Utils/PulseDetector.cs rename Tests/PlayMode/Utils/{EnergyDetector.cs.meta => PulseDetector.cs.meta} (83%) diff --git a/Tests/PlayMode/LatencyTests.cs b/Tests/PlayMode/LatencyTests.cs index 018acb25..a4d1a34a 100644 --- a/Tests/PlayMode/LatencyTests.cs +++ b/Tests/PlayMode/LatencyTests.cs @@ -19,8 +19,10 @@ public class LatencyTests const int kAudioFrameDurationMs = 10; const int kSamplesPerFrame = kAudioSampleRate * kAudioFrameDurationMs / 1000; // 480 - // Energy detection - const double kHighEnergyThreshold = 0.3; + // Pulse frequency tagging + const double kBaseFrequency = 500.0; // Pulse 0 = 500 Hz + const double kFrequencyStep = 100.0; // Pulse i = 500 + i*100 Hz + const double kMagnitudeThreshold = 0.15; const int kHighEnergyFramesPerPulse = 5; // Test parameters @@ -67,14 +69,14 @@ public IEnumerator ConnectionTime() } // ===================================================================== - // Test 2: Audio Latency Measurement using Energy Detection + // Test 2: Audio Latency Measurement using Frequency-Tagged Pulses // ===================================================================== [UnityTest, Category("E2E")] public IEnumerator AudioLatency() { Debug.Log("\n=== Audio Latency Measurement Test ==="); - Debug.Log("Using energy detection to measure audio round-trip latency"); + Debug.Log("Using frequency-tagged pulses to measure audio round-trip latency"); // --- Connect two participants --- var sender = TestRoomContext.ConnectionOptions.Default; @@ -140,50 +142,54 @@ public IEnumerator AudioLatency() Debug.Log("Audio track subscribed, creating audio stream..."); - // --- Set up receiver audio stream + energy detector --- + // --- Set up receiver audio stream + pulse detector --- var listenerGO = new GameObject("LatencyTestAudioListener"); - listenerGO.AddComponent(); // Required for OnAudioFilterRead to fire + listenerGO.AddComponent(); var receiverGO = new GameObject("LatencyTestReceiver"); var unityAudioSource = receiverGO.AddComponent(); - unityAudioSource.spatialBlend = 0f; // 2D audio + unityAudioSource.spatialBlend = 0f; unityAudioSource.volume = 1f; // AudioStream constructor adds AudioProbe and starts playback var audioStream = new AudioStream(subscribedTrack, unityAudioSource); - // Add EnergyDetector AFTER AudioStream so OnAudioFilterRead order is correct - var energyDetector = receiverGO.AddComponent(); + // Add PulseDetector AFTER AudioStream so OnAudioFilterRead order is correct + var pulseDetector = receiverGO.AddComponent(); + pulseDetector.TotalPulses = kTotalPulses; + pulseDetector.BaseFrequency = kBaseFrequency; + pulseDetector.FrequencyStep = kFrequencyStep; + pulseDetector.MagnitudeThreshold = kMagnitudeThreshold; // --- Shared state for latency measurement --- var stats = new LatencyStats(); var lockObj = new object(); - long lastHighEnergySendTimeTicks = 0; - bool waitingForEcho = false; + var sendTimestamps = new long[kTotalPulses]; // per-pulse send timestamps + var pulseReceived = new bool[kTotalPulses]; // track which pulses have been matched int missedPulses = 0; - // Energy detector callback (runs on audio thread) - energyDetector.EnergyDetected += (energy) => + // Pulse detector callback (runs on audio thread) + pulseDetector.PulseReceived += (pulseIndex, magnitude) => { lock (lockObj) { - if (!waitingForEcho || energy <= kHighEnergyThreshold) + if (pulseIndex < 0 || pulseIndex >= kTotalPulses) return; + if (pulseReceived[pulseIndex]) + return; // already matched this pulse + if (sendTimestamps[pulseIndex] == 0) + return; // not sent yet long receiveTimeTicks = Stopwatch.GetTimestamp(); - long sendTimeTicks = lastHighEnergySendTimeTicks; + double latencyMs = (receiveTimeTicks - sendTimestamps[pulseIndex]) + * 1000.0 / Stopwatch.Frequency; - if (sendTimeTicks > 0) + if (latencyMs > 0 && latencyMs < 5000) { - double latencyMs = (receiveTimeTicks - sendTimeTicks) - * 1000.0 / Stopwatch.Frequency; - - if (latencyMs > 0 && latencyMs < 5000) - { - stats.AddMeasurement(latencyMs); - Debug.Log($"Audio latency: {latencyMs:F2} ms (energy: {energy:F3})"); - } - waitingForEcho = false; + pulseReceived[pulseIndex] = true; + stats.AddMeasurement(latencyMs); + Debug.Log($" Pulse {pulseIndex}: latency {latencyMs:F2} ms " + + $"(freq: {kBaseFrequency + pulseIndex * kFrequencyStep} Hz, mag: {magnitude:F3})"); } } }; @@ -194,6 +200,7 @@ public IEnumerator AudioLatency() int frameCount = 0; int pulsesSent = 0; int highEnergyFramesRemaining = 0; + double currentPulseFrequency = 0; var frameDuration = TimeSpan.FromMilliseconds(kAudioFrameDurationMs); var nextFrameTime = Stopwatch.GetTimestamp(); @@ -204,67 +211,55 @@ public IEnumerator AudioLatency() double waitMs = (nextFrameTime - now) * 1000.0 / Stopwatch.Frequency; if (waitMs > 1.0) { - yield return null; // yield to Unity, come back next frame + yield return null; continue; } nextFrameTime += (long)(frameDuration.TotalSeconds * Stopwatch.Frequency); float[] frameData; - // Check for echo timeout - lock (lockObj) - { - if (waitingForEcho && lastHighEnergySendTimeTicks > 0) - { - double elapsedMs = (Stopwatch.GetTimestamp() - lastHighEnergySendTimeTicks) - * 1000.0 / Stopwatch.Frequency; - if (elapsedMs > kEchoTimeoutSeconds * 1000) - { - Debug.Log($" Echo timeout for pulse {pulsesSent}, moving on..."); - waitingForEcho = false; - missedPulses++; - } - } - } - if (highEnergyFramesRemaining > 0) { - // Continue sending high-energy frames for current pulse - frameData = GenerateHighEnergyFrame(kSamplesPerFrame); + frameData = GenerateToneFrame(kSamplesPerFrame, currentPulseFrequency); highEnergyFramesRemaining--; } - else + else if (frameCount % kFramesBetweenPulses == 0) { - bool shouldPulse; - lock (lockObj) { shouldPulse = !waitingForEcho; } + // Start a new pulse with unique frequency + currentPulseFrequency = kBaseFrequency + pulsesSent * kFrequencyStep; + frameData = GenerateToneFrame(kSamplesPerFrame, currentPulseFrequency); + highEnergyFramesRemaining = kHighEnergyFramesPerPulse - 1; - if (frameCount % kFramesBetweenPulses == 0 && shouldPulse) - { - // Start a new pulse - frameData = GenerateHighEnergyFrame(kSamplesPerFrame); - highEnergyFramesRemaining = kHighEnergyFramesPerPulse - 1; - - lock (lockObj) - { - lastHighEnergySendTimeTicks = Stopwatch.GetTimestamp(); - waitingForEcho = true; - } - pulsesSent++; - Debug.Log($"Sent pulse {pulsesSent}/{kTotalPulses} ({kHighEnergyFramesPerPulse} frames)"); - } - else + lock (lockObj) { - frameData = GenerateSilentFrame(kSamplesPerFrame); + sendTimestamps[pulsesSent] = Stopwatch.GetTimestamp(); } + Debug.Log($"Sent pulse {pulsesSent}/{kTotalPulses} " + + $"(freq: {currentPulseFrequency} Hz, {kHighEnergyFramesPerPulse} frames)"); + pulsesSent++; + } + else + { + frameData = GenerateSilentFrame(kSamplesPerFrame); } audioSource.PushFrame(frameData, kAudioChannels, kAudioSampleRate); frameCount++; } - // Wait for last echo + // Wait for last echoes to arrive yield return new WaitForSeconds(kEchoTimeoutSeconds); + // Count missed pulses + lock (lockObj) + { + for (int i = 0; i < kTotalPulses; i++) + { + if (sendTimestamps[i] > 0 && !pulseReceived[i]) + missedPulses++; + } + } + // --- Print results --- stats.PrintStats("Audio Latency Statistics"); if (missedPulses > 0) @@ -284,10 +279,9 @@ public IEnumerator AudioLatency() // Audio Generation Helpers // ===================================================================== - static float[] GenerateHighEnergyFrame(int samplesPerChannel) + static float[] GenerateToneFrame(int samplesPerChannel, double frequency) { var data = new float[samplesPerChannel * kAudioChannels]; - const double frequency = 1000.0; // 1kHz sine wave const double amplitude = 0.9; for (int i = 0; i < samplesPerChannel; i++) diff --git a/Tests/PlayMode/Utils/EnergyDetector.cs b/Tests/PlayMode/Utils/EnergyDetector.cs deleted file mode 100644 index aad4f8f6..00000000 --- a/Tests/PlayMode/Utils/EnergyDetector.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using UnityEngine; - -namespace LiveKit.PlayModeTests.Utils -{ - /// - /// MonoBehaviour that detects audio energy via OnAudioFilterRead. - /// Attach to the same GameObject as an AudioSource after AudioStream - /// has added its AudioProbe, so this filter runs second and sees - /// the filled audio data. - /// - public class EnergyDetector : MonoBehaviour - { - /// - /// Fired on the audio thread when non-trivial energy is detected. - /// The double parameter is the RMS energy of the frame. - /// - public event Action EnergyDetected; - - void OnAudioFilterRead(float[] data, int channels) - { - double sumSquared = 0.0; - for (int i = 0; i < data.Length; i++) - { - sumSquared += data[i] * data[i]; - } - double rms = Math.Sqrt(sumSquared / data.Length); - - if (rms > 0.01) - EnergyDetected?.Invoke(rms); - } - } -} diff --git a/Tests/PlayMode/Utils/PulseDetector.cs b/Tests/PlayMode/Utils/PulseDetector.cs new file mode 100644 index 00000000..658bf529 --- /dev/null +++ b/Tests/PlayMode/Utils/PulseDetector.cs @@ -0,0 +1,89 @@ +using System; +using UnityEngine; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// MonoBehaviour that identifies audio pulses by frequency using the Goertzel algorithm. + /// Attach to the same GameObject as an AudioSource after AudioStream has added its + /// AudioProbe, so this filter runs second and sees the filled audio data. + /// + public class PulseDetector : MonoBehaviour + { + public int TotalPulses; + public double BaseFrequency; + public double FrequencyStep; + public double MagnitudeThreshold; + + /// + /// Fired on the audio thread when a pulse is detected. + /// Parameters: (pulseIndex, magnitude). + /// + public event Action PulseReceived; + + private int _sampleRate; + + void OnEnable() + { + _sampleRate = AudioSettings.outputSampleRate; + AudioSettings.OnAudioConfigurationChanged += OnAudioConfigChanged; + } + + void OnDisable() + { + AudioSettings.OnAudioConfigurationChanged -= OnAudioConfigChanged; + } + + void OnAudioConfigChanged(bool deviceWasChanged) + { + _sampleRate = AudioSettings.outputSampleRate; + } + + void OnAudioFilterRead(float[] data, int channels) + { + int sampleRate = _sampleRate; + int samples = data.Length / channels; + if (samples == 0) return; + + int bestPulse = -1; + double bestMag = 0; + + for (int p = 0; p < TotalPulses; p++) + { + double freq = BaseFrequency + p * FrequencyStep; + double mag = Goertzel(data, channels, samples, sampleRate, freq); + if (mag > bestMag) + { + bestMag = mag; + bestPulse = p; + } + } + + // Debug.Log($"bestPulse: {bestPulse} | bestMag: {bestMag}"); + if (bestPulse >= 0 && bestMag > MagnitudeThreshold) + PulseReceived?.Invoke(bestPulse, bestMag); + } + + /// + /// Goertzel algorithm — computes the magnitude of a single frequency bin. + /// O(N) per frequency, much cheaper than a full FFT. + /// + static double Goertzel(float[] data, int channels, int N, int sampleRate, double freq) + { + double k = 0.5 + (double)N * freq / sampleRate; + double w = 2.0 * Math.PI * k / N; + double coeff = 2.0 * Math.Cos(w); + double s0 = 0, s1 = 0, s2 = 0; + + for (int i = 0; i < N; i++) + { + s0 = data[i * channels] + coeff * s1 - s2; + s2 = s1; + s1 = s0; + } + + double power = s1 * s1 + s2 * s2 - coeff * s1 * s2; + return Math.Sqrt(Math.Abs(power)) / N; + } + } +} diff --git a/Tests/PlayMode/Utils/EnergyDetector.cs.meta b/Tests/PlayMode/Utils/PulseDetector.cs.meta similarity index 83% rename from Tests/PlayMode/Utils/EnergyDetector.cs.meta rename to Tests/PlayMode/Utils/PulseDetector.cs.meta index 0eb5cde0..1a23d69d 100644 --- a/Tests/PlayMode/Utils/EnergyDetector.cs.meta +++ b/Tests/PlayMode/Utils/PulseDetector.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 0bb4d155287b94d16954c12e27c58937 +guid: 075f9b67150684b77a332c0b45514421 MonoImporter: externalObjects: {} serializedVersion: 2 From 158d0fa24bf266f6972b54fc28d7fc7f3b4afd47 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:40:01 +0100 Subject: [PATCH 3/6] Simple (non tagged) video test works --- Runtime/Scripts/RtcVideoSource.cs | 2 +- Tests/PlayMode/LatencyTests.cs | 190 ++++++++++++++++++ Tests/PlayMode/LiveKit.PlayModeTests.asmdef | 3 +- Tests/PlayMode/Utils/TestCoroutineRunner.cs | 47 +++++ .../Utils/TestCoroutineRunner.cs.meta | 11 + Tests/PlayMode/Utils/TestVideoSource.cs | 64 ++++++ Tests/PlayMode/Utils/TestVideoSource.cs.meta | 11 + 7 files changed, 326 insertions(+), 2 deletions(-) create mode 100644 Tests/PlayMode/Utils/TestCoroutineRunner.cs create mode 100644 Tests/PlayMode/Utils/TestCoroutineRunner.cs.meta create mode 100644 Tests/PlayMode/Utils/TestVideoSource.cs create mode 100644 Tests/PlayMode/Utils/TestVideoSource.cs.meta diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs index fc9b7677..24e15e13 100644 --- a/Runtime/Scripts/RtcVideoSource.cs +++ b/Runtime/Scripts/RtcVideoSource.cs @@ -42,7 +42,7 @@ public enum VideoStreamSource private bool _muted = false; public override bool Muted => _muted; - internal RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType) + protected RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType) { _sourceType = sourceType; _bufferType = bufferType; diff --git a/Tests/PlayMode/LatencyTests.cs b/Tests/PlayMode/LatencyTests.cs index a4d1a34a..ad7b02ca 100644 --- a/Tests/PlayMode/LatencyTests.cs +++ b/Tests/PlayMode/LatencyTests.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.InteropServices; using NUnit.Framework; using UnityEngine; using UnityEngine.TestTools; @@ -275,6 +276,195 @@ public IEnumerator AudioLatency() Assert.Greater(stats.Count, 0, "At least one audio latency measurement should be recorded"); } + // ===================================================================== + // Test 3: Video Latency Measurement using Bright Pulses + // ===================================================================== + + // Video configuration + const int kVideoWidth = 64; + const int kVideoHeight = 64; + const int kVideoBrightnessThreshold = 50; + const float kVideoPulseDurationSeconds = 0.5f; + const float kVideoPulseIntervalSeconds = 2f; + const float kVideoEchoTimeoutSeconds = 3f; + const float kVideoWarmupSeconds = 3f; + + [UnityTest, Category("E2E")] + public IEnumerator VideoLatency() + { + Debug.Log("\n=== Video Latency Measurement Test ==="); + Debug.Log("Using bright pulses to measure video round-trip latency"); + + // --- Connect two participants --- + var sender = TestRoomContext.ConnectionOptions.Default; + sender.Identity = "video-sender"; + var receiver = TestRoomContext.ConnectionOptions.Default; + receiver.Identity = "video-receiver"; + + using var context = new TestRoomContext(new[] { receiver, sender }); + yield return context.ConnectAll(); + if (context.ConnectionError != null) + Assert.Fail(context.ConnectionError); + + var receiverRoom = context.Rooms[0]; + var senderRoom = context.Rooms[1]; + Debug.Log($"Receiver connected as: {receiverRoom.LocalParticipant.Identity}"); + Debug.Log($"Sender connected as: {senderRoom.LocalParticipant.Identity}"); + + // --- Wait for sender to be visible to receiver --- + var participantExpectation = new Expectation(timeoutSeconds: 10f); + if (receiverRoom.RemoteParticipants.ContainsKey(sender.Identity)) + { + participantExpectation.Fulfill(); + } + else + { + receiverRoom.ParticipantConnected += (p) => + { + if (p.Identity == sender.Identity) + participantExpectation.Fulfill(); + }; + } + yield return participantExpectation.Wait(); + if (participantExpectation.Error != null) + Assert.Fail($"Sender not visible to receiver: {participantExpectation.Error}"); + + // --- Create and publish video track from sender --- + var videoSource = new TestVideoSource(kVideoWidth, kVideoHeight); + videoSource.Start(); + + var videoTrack = LocalVideoTrack.CreateVideoTrack("video-latency-test", videoSource, senderRoom); + var publishOptions = new TrackPublishOptions(); + var publish = senderRoom.LocalParticipant.PublishTrack(videoTrack, publishOptions); + yield return publish; + if (publish.IsError) + Assert.Fail("Failed to publish video track"); + + // Start the video source update coroutine (drives ReadBuffer + SendFrame each frame) + var sourceUpdateCoroutine = TestCoroutineRunner.Start(videoSource.Update()); + + Debug.Log("Video track published, waiting for subscription..."); + + // --- Wait for receiver to subscribe to the video track --- + RemoteVideoTrack subscribedTrack = null; + var trackExpectation = new Expectation(timeoutSeconds: 10f); + receiverRoom.TrackSubscribed += (track, publication, participant) => + { + if (track is RemoteVideoTrack rvt && participant.Identity == sender.Identity) + { + subscribedTrack = rvt; + trackExpectation.Fulfill(); + } + }; + yield return trackExpectation.Wait(); + if (trackExpectation.Error != null) + Assert.Fail($"Receiver did not subscribe to video track: {trackExpectation.Error}"); + + Debug.Log("Video track subscribed, creating video stream..."); + + // --- Set up receiver video stream --- + var videoStream = new VideoStream(subscribedTrack); + videoStream.Start(); + var streamUpdateCoroutine = TestCoroutineRunner.Start(videoStream.Update()); + + // --- Shared state for latency measurement --- + var stats = new LatencyStats(); + long lastSendTimeTicks = 0; + bool waitingForEcho = false; + int pulsesSent = 0; + int missedPulses = 0; + + // Subscribe to received video frames (fires on main thread) + videoStream.FrameReceived += (frame) => + { + var buffer = videoStream.VideoBuffer; + if (buffer == null || !buffer.IsValid) + return; + + if (buffer.Info.Components.Count == 0) + return; + + var yComponent = buffer.Info.Components[0]; + if (!yComponent.HasDataPtr) + return; + + // Sample Y plane to get average brightness + var yPtr = (IntPtr)yComponent.DataPtr; + int sampleCount = Math.Min(16, (int)(buffer.Width * buffer.Height)); + int ySum = 0; + for (int i = 0; i < sampleCount; i++) + { + ySum += Marshal.ReadByte(yPtr, i); + } + int avgY = ySum / sampleCount; + + if (!waitingForEcho || avgY <= kVideoBrightnessThreshold) + return; + + long receiveTimeTicks = Stopwatch.GetTimestamp(); + double latencyMs = (receiveTimeTicks - lastSendTimeTicks) + * 1000.0 / Stopwatch.Frequency; + + if (latencyMs > 0 && latencyMs < 5000) + { + waitingForEcho = false; + stats.AddMeasurement(latencyMs); + Debug.Log($" Pulse {pulsesSent + 1}: latency {latencyMs:F2} ms (Y: {avgY})"); + } + }; + + // --- Warm up: send black frames so the encoder stabilizes --- + Debug.Log($"Warming up encoder for {kVideoWarmupSeconds}s..."); + yield return new WaitForSeconds(kVideoWarmupSeconds); + + // --- Send video pulses --- + Debug.Log("Starting video pulse transmission..."); + + for (pulsesSent = 0; pulsesSent < kTotalPulses; pulsesSent++) + { + // Send bright pulse + videoSource.SetBright(true); + lastSendTimeTicks = Stopwatch.GetTimestamp(); + waitingForEcho = true; + Debug.Log($"Sent pulse {pulsesSent + 1}/{kTotalPulses}"); + + // Hold pulse for kVideoPulseDurationSeconds (time-based, not frame-count) + yield return new WaitForSeconds(kVideoPulseDurationSeconds); + + // Reset to black between pulses + videoSource.SetBright(false); + + // Wait before next pulse + yield return new WaitForSeconds(kVideoPulseIntervalSeconds); + + // Check for timeout + if (waitingForEcho) + { + Debug.Log($" Echo timeout for pulse {pulsesSent + 1}, moving on..."); + waitingForEcho = false; + missedPulses++; + } + } + + // Wait for last echo + yield return new WaitForSeconds(kVideoEchoTimeoutSeconds); + + // --- Print results --- + stats.PrintStats("Video Latency Statistics"); + if (missedPulses > 0) + Debug.Log($"Missed pulses (timeout): {missedPulses}"); + + // --- Cleanup --- + videoStream.Stop(); + videoStream.Dispose(); + TestCoroutineRunner.Stop(streamUpdateCoroutine); + videoSource.Stop(); + videoSource.Dispose(); + TestCoroutineRunner.Stop(sourceUpdateCoroutine); + + Assert.Greater(stats.Count, 0, "At least one video latency measurement should be recorded"); + } + // ===================================================================== // Audio Generation Helpers // ===================================================================== diff --git a/Tests/PlayMode/LiveKit.PlayModeTests.asmdef b/Tests/PlayMode/LiveKit.PlayModeTests.asmdef index f239d10c..381e0b2f 100644 --- a/Tests/PlayMode/LiveKit.PlayModeTests.asmdef +++ b/Tests/PlayMode/LiveKit.PlayModeTests.asmdef @@ -11,7 +11,8 @@ "allowUnsafeCode": false, "overrideReferences": true, "precompiledReferences": [ - "nunit.framework.dll" + "nunit.framework.dll", + "Google.Protobuf.dll" ], "autoReferenced": false, "defineConstraints": [ diff --git a/Tests/PlayMode/Utils/TestCoroutineRunner.cs b/Tests/PlayMode/Utils/TestCoroutineRunner.cs new file mode 100644 index 00000000..6966c478 --- /dev/null +++ b/Tests/PlayMode/Utils/TestCoroutineRunner.cs @@ -0,0 +1,47 @@ +using System.Collections; +using UnityEngine; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Helper for running coroutines from non-MonoBehaviour test code. + /// Creates a temporary GameObject with a MonoBehaviour to host coroutines. + /// + public class TestCoroutineRunner : MonoBehaviour + { + private static TestCoroutineRunner _instance; + + private static TestCoroutineRunner Instance + { + get + { + if (_instance == null) + { + var go = new GameObject("TestCoroutineRunner"); + _instance = go.AddComponent(); + } + return _instance; + } + } + + public static Coroutine Start(IEnumerator routine) + { + return Instance.StartCoroutine(routine); + } + + public static void Stop(Coroutine coroutine) + { + if (_instance != null && coroutine != null) + _instance.StopCoroutine(coroutine); + } + + public static void Cleanup() + { + if (_instance != null) + { + Destroy(_instance.gameObject); + _instance = null; + } + } + } +} diff --git a/Tests/PlayMode/Utils/TestCoroutineRunner.cs.meta b/Tests/PlayMode/Utils/TestCoroutineRunner.cs.meta new file mode 100644 index 00000000..3c588640 --- /dev/null +++ b/Tests/PlayMode/Utils/TestCoroutineRunner.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 23454041a003e4c17b9bee9247b88bf3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/TestVideoSource.cs b/Tests/PlayMode/Utils/TestVideoSource.cs new file mode 100644 index 00000000..20bda64a --- /dev/null +++ b/Tests/PlayMode/Utils/TestVideoSource.cs @@ -0,0 +1,64 @@ +using LiveKit.Proto; +using Unity.Collections; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// A programmatic video source for testing. Generates solid white or black + /// RGBA frames without requiring GPU textures. + /// + public class TestVideoSource : RtcVideoSource + { + private readonly int _width; + private readonly int _height; + private bool _bright = false; + + public TestVideoSource(int width = 64, int height = 64) + : base(VideoStreamSource.Texture, VideoBufferType.Rgba) + { + _width = width; + _height = height; + base.Init(); + } + + public override int GetWidth() => _width; + public override int GetHeight() => _height; + protected override VideoRotation GetVideoRotation() => VideoRotation._0; + + /// + /// Set whether the frame is bright (white) or dark (black). + /// + public void SetBright(bool bright) + { + _bright = bright; + } + + protected override bool ReadBuffer() + { + int size = _width * _height * 4; + if (!_captureBuffer.IsCreated || _captureBuffer.Length != size) + { + if (_captureBuffer.IsCreated) _captureBuffer.Dispose(); + _captureBuffer = new NativeArray(size, Allocator.Persistent); + } + + byte value = _bright ? (byte)255 : (byte)0; + + for (int i = 0; i < size; i += 4) + { + _captureBuffer[i] = value; // R + _captureBuffer[i + 1] = value; // G + _captureBuffer[i + 2] = value; // B + _captureBuffer[i + 3] = 255; // A + } + + _requestPending = true; + return false; + } + + ~TestVideoSource() + { + Dispose(false); + } + } +} diff --git a/Tests/PlayMode/Utils/TestVideoSource.cs.meta b/Tests/PlayMode/Utils/TestVideoSource.cs.meta new file mode 100644 index 00000000..398f5484 --- /dev/null +++ b/Tests/PlayMode/Utils/TestVideoSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 23da0c66171a74041a0a96b75a00207e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 47b6cd112654a6d1fe65ba322e1aa839481023ab Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:46:08 +0100 Subject: [PATCH 4/6] Coded video frames work, coding works with bar code schema --- Tests/PlayMode/LatencyTests.cs | 87 +++++++++++++++---------- Tests/PlayMode/Utils/TestVideoSource.cs | 42 ++++++++---- 2 files changed, 84 insertions(+), 45 deletions(-) diff --git a/Tests/PlayMode/LatencyTests.cs b/Tests/PlayMode/LatencyTests.cs index ad7b02ca..1a820c0f 100644 --- a/Tests/PlayMode/LatencyTests.cs +++ b/Tests/PlayMode/LatencyTests.cs @@ -277,13 +277,13 @@ public IEnumerator AudioLatency() } // ===================================================================== - // Test 3: Video Latency Measurement using Bright Pulses + // Test 3: Video Latency Measurement using Spatial Binary Encoding // ===================================================================== // Video configuration const int kVideoWidth = 64; const int kVideoHeight = 64; - const int kVideoBrightnessThreshold = 50; + const int kVideoStripThreshold = 128; const float kVideoPulseDurationSeconds = 0.5f; const float kVideoPulseIntervalSeconds = 2f; const float kVideoEchoTimeoutSeconds = 3f; @@ -293,7 +293,7 @@ public IEnumerator AudioLatency() public IEnumerator VideoLatency() { Debug.Log("\n=== Video Latency Measurement Test ==="); - Debug.Log("Using bright pulses to measure video round-trip latency"); + Debug.Log("Using spatial binary encoding to measure video round-trip latency"); // --- Connect two participants --- var sender = TestRoomContext.ConnectionOptions.Default; @@ -369,9 +369,8 @@ public IEnumerator VideoLatency() // --- Shared state for latency measurement --- var stats = new LatencyStats(); - long lastSendTimeTicks = 0; - bool waitingForEcho = false; - int pulsesSent = 0; + var sendTimestamps = new long[kTotalPulses]; + var pulseReceived = new bool[kTotalPulses]; int missedPulses = 0; // Subscribe to received video frames (fires on main thread) @@ -388,28 +387,51 @@ public IEnumerator VideoLatency() if (!yComponent.HasDataPtr) return; - // Sample Y plane to get average brightness + // Decode spatial binary pattern from Y plane + // Sample the center of each vertical strip at 3 rows to get reliable values var yPtr = (IntPtr)yComponent.DataPtr; - int sampleCount = Math.Min(16, (int)(buffer.Width * buffer.Height)); - int ySum = 0; - for (int i = 0; i < sampleCount; i++) + int width = (int)buffer.Width; + int stripWidth = width / TestVideoSource.NumStrips; + int[] sampleRows = { (int)buffer.Height / 4, (int)buffer.Height / 2, (int)(buffer.Height * 3 / 4) }; + + int decodedIndex = 0; + for (int strip = 0; strip < TestVideoSource.NumStrips; strip++) { - ySum += Marshal.ReadByte(yPtr, i); + int centerX = strip * stripWidth + stripWidth / 2; + int ySum = 0; + foreach (int row in sampleRows) + { + int offset = row * width + centerX; + ySum += Marshal.ReadByte(yPtr, offset); + } + int avgY = ySum / sampleRows.Length; + + if (avgY > kVideoStripThreshold) + decodedIndex |= (1 << strip); } - int avgY = ySum / sampleCount; - if (!waitingForEcho || avgY <= kVideoBrightnessThreshold) + // Index 0 = all black = no pulse + if (decodedIndex == 0) + return; + + // Pulse indices are 1-based in encoding (pulse 0 sends index 1, etc.) + int pulseIndex = decodedIndex - 1; + if (pulseIndex < 0 || pulseIndex >= kTotalPulses) + return; + if (pulseReceived[pulseIndex]) + return; + if (sendTimestamps[pulseIndex] == 0) return; long receiveTimeTicks = Stopwatch.GetTimestamp(); - double latencyMs = (receiveTimeTicks - lastSendTimeTicks) + double latencyMs = (receiveTimeTicks - sendTimestamps[pulseIndex]) * 1000.0 / Stopwatch.Frequency; if (latencyMs > 0 && latencyMs < 5000) { - waitingForEcho = false; + pulseReceived[pulseIndex] = true; stats.AddMeasurement(latencyMs); - Debug.Log($" Pulse {pulsesSent + 1}: latency {latencyMs:F2} ms (Y: {avgY})"); + Debug.Log($" Pulse {pulseIndex}: latency {latencyMs:F2} ms (decoded: {decodedIndex})"); } }; @@ -420,35 +442,34 @@ public IEnumerator VideoLatency() // --- Send video pulses --- Debug.Log("Starting video pulse transmission..."); - for (pulsesSent = 0; pulsesSent < kTotalPulses; pulsesSent++) + // Encode pulse index as 1-based so that index 0 (all black) means "no pulse" + for (int pulsesSent = 0; pulsesSent < kTotalPulses; pulsesSent++) { - // Send bright pulse - videoSource.SetBright(true); - lastSendTimeTicks = Stopwatch.GetTimestamp(); - waitingForEcho = true; - Debug.Log($"Sent pulse {pulsesSent + 1}/{kTotalPulses}"); + int encodedIndex = pulsesSent + 1; + videoSource.SetPulseIndex(encodedIndex); + sendTimestamps[pulsesSent] = Stopwatch.GetTimestamp(); + Debug.Log($"Sent pulse {pulsesSent + 1}/{kTotalPulses} (pattern: {Convert.ToString(encodedIndex, 2).PadLeft(4, '0')})"); - // Hold pulse for kVideoPulseDurationSeconds (time-based, not frame-count) + // Hold pulse so encoder has time to process yield return new WaitForSeconds(kVideoPulseDurationSeconds); // Reset to black between pulses - videoSource.SetBright(false); + videoSource.SetPulseIndex(-1); // Wait before next pulse yield return new WaitForSeconds(kVideoPulseIntervalSeconds); - - // Check for timeout - if (waitingForEcho) - { - Debug.Log($" Echo timeout for pulse {pulsesSent + 1}, moving on..."); - waitingForEcho = false; - missedPulses++; - } } - // Wait for last echo + // Wait for last echoes to arrive yield return new WaitForSeconds(kVideoEchoTimeoutSeconds); + // Count missed pulses + for (int i = 0; i < kTotalPulses; i++) + { + if (sendTimestamps[i] > 0 && !pulseReceived[i]) + missedPulses++; + } + // --- Print results --- stats.PrintStats("Video Latency Statistics"); if (missedPulses > 0) diff --git a/Tests/PlayMode/Utils/TestVideoSource.cs b/Tests/PlayMode/Utils/TestVideoSource.cs index 20bda64a..91eed475 100644 --- a/Tests/PlayMode/Utils/TestVideoSource.cs +++ b/Tests/PlayMode/Utils/TestVideoSource.cs @@ -4,14 +4,23 @@ namespace LiveKit.PlayModeTests.Utils { /// - /// A programmatic video source for testing. Generates solid white or black - /// RGBA frames without requiring GPU textures. + /// A programmatic video source for testing. Encodes a pulse index as a + /// spatial binary pattern: the frame is divided into 4 vertical strips, + /// each either black or white, representing 4 bits of the index. + /// This survives WebRTC lossy compression because detection only needs + /// to distinguish black vs white in large uniform regions. /// public class TestVideoSource : RtcVideoSource { private readonly int _width; private readonly int _height; - private bool _bright = false; + private int _pulseIndex = -1; + + /// + /// Number of vertical strips used for binary encoding. + /// 4 strips = 4 bits = values 0–15. + /// + public const int NumStrips = 4; public TestVideoSource(int width = 64, int height = 64) : base(VideoStreamSource.Texture, VideoBufferType.Rgba) @@ -26,11 +35,12 @@ public TestVideoSource(int width = 64, int height = 64) protected override VideoRotation GetVideoRotation() => VideoRotation._0; /// - /// Set whether the frame is bright (white) or dark (black). + /// Set the pulse index to encode. -1 = all black (between pulses). + /// Index is encoded as 4-bit binary across vertical strips. /// - public void SetBright(bool bright) + public void SetPulseIndex(int index) { - _bright = bright; + _pulseIndex = index; } protected override bool ReadBuffer() @@ -42,14 +52,22 @@ protected override bool ReadBuffer() _captureBuffer = new NativeArray(size, Allocator.Persistent); } - byte value = _bright ? (byte)255 : (byte)0; + int stripWidth = _width / NumStrips; - for (int i = 0; i < size; i += 4) + for (int y = 0; y < _height; y++) { - _captureBuffer[i] = value; // R - _captureBuffer[i + 1] = value; // G - _captureBuffer[i + 2] = value; // B - _captureBuffer[i + 3] = 255; // A + for (int x = 0; x < _width; x++) + { + int strip = x / stripWidth; + bool bright = _pulseIndex >= 0 && ((_pulseIndex >> strip) & 1) == 1; + byte value = bright ? (byte)255 : (byte)0; + + int offset = (y * _width + x) * 4; + _captureBuffer[offset] = value; // R + _captureBuffer[offset + 1] = value; // G + _captureBuffer[offset + 2] = value; // B + _captureBuffer[offset + 3] = 255; // A + } } _requestPending = true; From 1d45381f774a84eb99eb642627c4ca7c53692528 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:02:08 +0100 Subject: [PATCH 5/6] Cleaned up the video pulse codec --- Tests/PlayMode/LatencyTests.cs | 55 ++------------- ...PulseDetector.cs => AudioPulseDetector.cs} | 2 +- ...tor.cs.meta => AudioPulseDetector.cs.meta} | 0 Tests/PlayMode/Utils/VideoPulseCodec.cs | 67 +++++++++++++++++++ Tests/PlayMode/Utils/VideoPulseCodec.cs.meta | 11 +++ 5 files changed, 86 insertions(+), 49 deletions(-) rename Tests/PlayMode/Utils/{PulseDetector.cs => AudioPulseDetector.cs} (98%) rename Tests/PlayMode/Utils/{PulseDetector.cs.meta => AudioPulseDetector.cs.meta} (100%) create mode 100644 Tests/PlayMode/Utils/VideoPulseCodec.cs create mode 100644 Tests/PlayMode/Utils/VideoPulseCodec.cs.meta diff --git a/Tests/PlayMode/LatencyTests.cs b/Tests/PlayMode/LatencyTests.cs index 1a820c0f..a65aefbd 100644 --- a/Tests/PlayMode/LatencyTests.cs +++ b/Tests/PlayMode/LatencyTests.cs @@ -2,7 +2,6 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; -using System.Runtime.InteropServices; using NUnit.Framework; using UnityEngine; using UnityEngine.TestTools; @@ -156,7 +155,7 @@ public IEnumerator AudioLatency() var audioStream = new AudioStream(subscribedTrack, unityAudioSource); // Add PulseDetector AFTER AudioStream so OnAudioFilterRead order is correct - var pulseDetector = receiverGO.AddComponent(); + var pulseDetector = receiverGO.AddComponent(); pulseDetector.TotalPulses = kTotalPulses; pulseDetector.BaseFrequency = kBaseFrequency; pulseDetector.FrequencyStep = kFrequencyStep; @@ -376,46 +375,7 @@ public IEnumerator VideoLatency() // Subscribe to received video frames (fires on main thread) videoStream.FrameReceived += (frame) => { - var buffer = videoStream.VideoBuffer; - if (buffer == null || !buffer.IsValid) - return; - - if (buffer.Info.Components.Count == 0) - return; - - var yComponent = buffer.Info.Components[0]; - if (!yComponent.HasDataPtr) - return; - - // Decode spatial binary pattern from Y plane - // Sample the center of each vertical strip at 3 rows to get reliable values - var yPtr = (IntPtr)yComponent.DataPtr; - int width = (int)buffer.Width; - int stripWidth = width / TestVideoSource.NumStrips; - int[] sampleRows = { (int)buffer.Height / 4, (int)buffer.Height / 2, (int)(buffer.Height * 3 / 4) }; - - int decodedIndex = 0; - for (int strip = 0; strip < TestVideoSource.NumStrips; strip++) - { - int centerX = strip * stripWidth + stripWidth / 2; - int ySum = 0; - foreach (int row in sampleRows) - { - int offset = row * width + centerX; - ySum += Marshal.ReadByte(yPtr, offset); - } - int avgY = ySum / sampleRows.Length; - - if (avgY > kVideoStripThreshold) - decodedIndex |= (1 << strip); - } - - // Index 0 = all black = no pulse - if (decodedIndex == 0) - return; - - // Pulse indices are 1-based in encoding (pulse 0 sends index 1, etc.) - int pulseIndex = decodedIndex - 1; + int pulseIndex = VideoPulseCodec.Decode(videoStream.VideoBuffer, kVideoStripThreshold); if (pulseIndex < 0 || pulseIndex >= kTotalPulses) return; if (pulseReceived[pulseIndex]) @@ -431,7 +391,7 @@ public IEnumerator VideoLatency() { pulseReceived[pulseIndex] = true; stats.AddMeasurement(latencyMs); - Debug.Log($" Pulse {pulseIndex}: latency {latencyMs:F2} ms (decoded: {decodedIndex})"); + Debug.Log($" Pulse {pulseIndex}: latency {latencyMs:F2} ms"); } }; @@ -442,19 +402,18 @@ public IEnumerator VideoLatency() // --- Send video pulses --- Debug.Log("Starting video pulse transmission..."); - // Encode pulse index as 1-based so that index 0 (all black) means "no pulse" for (int pulsesSent = 0; pulsesSent < kTotalPulses; pulsesSent++) { - int encodedIndex = pulsesSent + 1; - videoSource.SetPulseIndex(encodedIndex); + int encoded = VideoPulseCodec.Encode(pulsesSent); + videoSource.SetPulseIndex(encoded); sendTimestamps[pulsesSent] = Stopwatch.GetTimestamp(); - Debug.Log($"Sent pulse {pulsesSent + 1}/{kTotalPulses} (pattern: {Convert.ToString(encodedIndex, 2).PadLeft(4, '0')})"); + Debug.Log($"Sent pulse {pulsesSent + 1}/{kTotalPulses} (pattern: {Convert.ToString(encoded, 2).PadLeft(4, '0')})"); // Hold pulse so encoder has time to process yield return new WaitForSeconds(kVideoPulseDurationSeconds); // Reset to black between pulses - videoSource.SetPulseIndex(-1); + videoSource.SetPulseIndex(VideoPulseCodec.Encode(-1)); // Wait before next pulse yield return new WaitForSeconds(kVideoPulseIntervalSeconds); diff --git a/Tests/PlayMode/Utils/PulseDetector.cs b/Tests/PlayMode/Utils/AudioPulseDetector.cs similarity index 98% rename from Tests/PlayMode/Utils/PulseDetector.cs rename to Tests/PlayMode/Utils/AudioPulseDetector.cs index 658bf529..e17702ad 100644 --- a/Tests/PlayMode/Utils/PulseDetector.cs +++ b/Tests/PlayMode/Utils/AudioPulseDetector.cs @@ -8,7 +8,7 @@ namespace LiveKit.PlayModeTests.Utils /// Attach to the same GameObject as an AudioSource after AudioStream has added its /// AudioProbe, so this filter runs second and sees the filled audio data. /// - public class PulseDetector : MonoBehaviour + public class AudioPulseDetector : MonoBehaviour { public int TotalPulses; public double BaseFrequency; diff --git a/Tests/PlayMode/Utils/PulseDetector.cs.meta b/Tests/PlayMode/Utils/AudioPulseDetector.cs.meta similarity index 100% rename from Tests/PlayMode/Utils/PulseDetector.cs.meta rename to Tests/PlayMode/Utils/AudioPulseDetector.cs.meta diff --git a/Tests/PlayMode/Utils/VideoPulseCodec.cs b/Tests/PlayMode/Utils/VideoPulseCodec.cs new file mode 100644 index 00000000..5c9faaee --- /dev/null +++ b/Tests/PlayMode/Utils/VideoPulseCodec.cs @@ -0,0 +1,67 @@ +using System; +using System.Runtime.InteropServices; +using LiveKit; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Encodes and decodes pulse indices for video latency measurement using + /// spatial binary encoding. The frame is divided into vertical strips, + /// each either black or white, representing bits of the pulse index. + /// + public static class VideoPulseCodec + { + /// + /// Encode a pulse index for use with . + /// Returns a 1-based encoded index so that 0 (all black) means "no pulse". + /// Pass -1 to clear (send black frames between pulses). + /// + public static int Encode(int pulseIndex) + { + return pulseIndex < 0 ? -1 : pulseIndex + 1; + } + + /// + /// Decode a pulse index from a received I420 video frame buffer by sampling + /// the Y plane at the center of each vertical strip. + /// Returns the 0-based pulse index, or -1 if no pulse is detected. + /// + public static int Decode(VideoFrameBuffer buffer, int stripThreshold = 128) + { + if (buffer == null || !buffer.IsValid) + return -1; + + if (buffer.Info.Components.Count == 0) + return -1; + + var yComponent = buffer.Info.Components[0]; + if (!yComponent.HasDataPtr) + return -1; + + var yPtr = (IntPtr)yComponent.DataPtr; + int width = (int)buffer.Width; + int height = (int)buffer.Height; + int stripWidth = width / TestVideoSource.NumStrips; + int[] sampleRows = { height / 4, height / 2, height * 3 / 4 }; + + int decodedIndex = 0; + for (int strip = 0; strip < TestVideoSource.NumStrips; strip++) + { + int centerX = strip * stripWidth + stripWidth / 2; + int ySum = 0; + foreach (int row in sampleRows) + { + int offset = row * width + centerX; + ySum += Marshal.ReadByte(yPtr, offset); + } + int avgY = ySum / sampleRows.Length; + + if (avgY > stripThreshold) + decodedIndex |= (1 << strip); + } + + // 0 = all black = no pulse; otherwise subtract 1 to get 0-based pulse index + return decodedIndex == 0 ? -1 : decodedIndex - 1; + } + } +} diff --git a/Tests/PlayMode/Utils/VideoPulseCodec.cs.meta b/Tests/PlayMode/Utils/VideoPulseCodec.cs.meta new file mode 100644 index 00000000..f2a1a9f5 --- /dev/null +++ b/Tests/PlayMode/Utils/VideoPulseCodec.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8d0341f1305a54e608a89f8a2b68175f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 43a8e41ee0e00e4ff3435ec53a6cffed45900266 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:07:41 +0100 Subject: [PATCH 6/6] Desync test added --- Tests/PlayMode/LatencyTests.cs | 299 +++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) diff --git a/Tests/PlayMode/LatencyTests.cs b/Tests/PlayMode/LatencyTests.cs index a65aefbd..14f79148 100644 --- a/Tests/PlayMode/LatencyTests.cs +++ b/Tests/PlayMode/LatencyTests.cs @@ -445,6 +445,305 @@ public IEnumerator VideoLatency() Assert.Greater(stats.Count, 0, "At least one video latency measurement should be recorded"); } + // ===================================================================== + // Test 4: A/V Sync Measurement + // ===================================================================== + + // A/V sync configuration + const float kAVSyncPulseDurationSeconds = 0.5f; + const float kAVSyncPulseIntervalSeconds = 2f; + const float kAVSyncEchoTimeoutSeconds = 3f; + const float kAVSyncWarmupSeconds = 3f; + + [UnityTest, Category("E2E")] + public IEnumerator AVSync() + { + Debug.Log("\n=== A/V Sync Measurement Test ==="); + Debug.Log("Measuring audio-video desync by sending simultaneous tagged pulses"); + + // --- Connect two participants --- + var sender = TestRoomContext.ConnectionOptions.Default; + sender.Identity = "avsync-sender"; + var receiver = TestRoomContext.ConnectionOptions.Default; + receiver.Identity = "avsync-receiver"; + + using var context = new TestRoomContext(new[] { receiver, sender }); + yield return context.ConnectAll(); + if (context.ConnectionError != null) + Assert.Fail(context.ConnectionError); + + var receiverRoom = context.Rooms[0]; + var senderRoom = context.Rooms[1]; + + // --- Wait for sender to be visible to receiver --- + var participantExpectation = new Expectation(timeoutSeconds: 10f); + if (receiverRoom.RemoteParticipants.ContainsKey(sender.Identity)) + { + participantExpectation.Fulfill(); + } + else + { + receiverRoom.ParticipantConnected += (p) => + { + if (p.Identity == sender.Identity) + participantExpectation.Fulfill(); + }; + } + yield return participantExpectation.Wait(); + if (participantExpectation.Error != null) + Assert.Fail($"Sender not visible to receiver: {participantExpectation.Error}"); + + // --- Publish audio track --- + var audioSource = new TestAudioSource(channels: kAudioChannels); + audioSource.Start(); + var audioTrack = LocalAudioTrack.CreateAudioTrack("avsync-audio", audioSource, senderRoom); + var audioPublish = senderRoom.LocalParticipant.PublishTrack(audioTrack, new TrackPublishOptions()); + yield return audioPublish; + if (audioPublish.IsError) + Assert.Fail("Failed to publish audio track"); + + // --- Publish video track --- + var videoSource = new TestVideoSource(kVideoWidth, kVideoHeight); + videoSource.Start(); + var videoTrack = LocalVideoTrack.CreateVideoTrack("avsync-video", videoSource, senderRoom); + var videoPublish = senderRoom.LocalParticipant.PublishTrack(videoTrack, new TrackPublishOptions()); + yield return videoPublish; + if (videoPublish.IsError) + Assert.Fail("Failed to publish video track"); + + var videoSourceCoroutine = TestCoroutineRunner.Start(videoSource.Update()); + + Debug.Log("Both tracks published, waiting for subscriptions..."); + + // --- Wait for receiver to subscribe to both tracks --- + RemoteAudioTrack subscribedAudioTrack = null; + RemoteVideoTrack subscribedVideoTrack = null; + var bothTracksExpectation = new Expectation( + predicate: () => subscribedAudioTrack != null && subscribedVideoTrack != null, + timeoutSeconds: 15f); + + receiverRoom.TrackSubscribed += (track, publication, participant) => + { + if (participant.Identity != sender.Identity) return; + if (track is RemoteAudioTrack rat) subscribedAudioTrack = rat; + if (track is RemoteVideoTrack rvt) subscribedVideoTrack = rvt; + }; + yield return bothTracksExpectation.Wait(); + if (bothTracksExpectation.Error != null) + Assert.Fail($"Failed to subscribe to both tracks: {bothTracksExpectation.Error}"); + + Debug.Log("Both tracks subscribed, setting up receivers..."); + + // --- Set up audio receiver --- + var listenerGO = new GameObject("AVSyncAudioListener"); + listenerGO.AddComponent(); + + var audioReceiverGO = new GameObject("AVSyncAudioReceiver"); + var unityAudioSource = audioReceiverGO.AddComponent(); + unityAudioSource.spatialBlend = 0f; + unityAudioSource.volume = 1f; + + var audioStream = new AudioStream(subscribedAudioTrack, unityAudioSource); + + var audioPulseDetector = audioReceiverGO.AddComponent(); + audioPulseDetector.TotalPulses = kTotalPulses; + audioPulseDetector.BaseFrequency = kBaseFrequency; + audioPulseDetector.FrequencyStep = kFrequencyStep; + audioPulseDetector.MagnitudeThreshold = kMagnitudeThreshold; + + // --- Set up video receiver --- + var videoStream = new VideoStream(subscribedVideoTrack); + videoStream.Start(); + var videoStreamCoroutine = TestCoroutineRunner.Start(videoStream.Update()); + + // --- Shared state --- + var lockObj = new object(); + var sendTimestamps = new long[kTotalPulses]; + var audioReceiveTicks = new long[kTotalPulses]; + var videoReceiveTicks = new long[kTotalPulses]; + + // Audio detection (runs on audio thread) + audioPulseDetector.PulseReceived += (pulseIndex, magnitude) => + { + lock (lockObj) + { + if (pulseIndex < 0 || pulseIndex >= kTotalPulses) return; + if (audioReceiveTicks[pulseIndex] != 0) return; + if (sendTimestamps[pulseIndex] == 0) return; + + audioReceiveTicks[pulseIndex] = Stopwatch.GetTimestamp(); + Debug.Log($" [Audio] Pulse {pulseIndex} received " + + $"(freq: {kBaseFrequency + pulseIndex * kFrequencyStep} Hz)"); + } + }; + + // Video detection (runs on main thread) + videoStream.FrameReceived += (frame) => + { + int pulseIndex = VideoPulseCodec.Decode(videoStream.VideoBuffer, kVideoStripThreshold); + if (pulseIndex < 0 || pulseIndex >= kTotalPulses) return; + if (videoReceiveTicks[pulseIndex] != 0) return; + if (sendTimestamps[pulseIndex] == 0) return; + + videoReceiveTicks[pulseIndex] = Stopwatch.GetTimestamp(); + Debug.Log($" [Video] Pulse {pulseIndex} received"); + }; + + // --- Audio frame pusher (runs concurrently) --- + int currentAudioPulseIndex = -1; + bool audioRunning = true; + + IEnumerator PushAudioFrames() + { + var frameDuration = TimeSpan.FromMilliseconds(kAudioFrameDurationMs); + var nextFrameTime = Stopwatch.GetTimestamp(); + int highEnergyFramesRemaining = 0; + double currentFrequency = 0; + int lastPulseIndex = -1; + + while (audioRunning) + { + long now = Stopwatch.GetTimestamp(); + double waitMs = (nextFrameTime - now) * 1000.0 / Stopwatch.Frequency; + if (waitMs > 1.0) + { + yield return null; + continue; + } + nextFrameTime += (long)(frameDuration.TotalSeconds * Stopwatch.Frequency); + + int pulseIdx = currentAudioPulseIndex; + float[] frameData; + + if (highEnergyFramesRemaining > 0) + { + frameData = GenerateToneFrame(kSamplesPerFrame, currentFrequency); + highEnergyFramesRemaining--; + } + else if (pulseIdx >= 0 && pulseIdx != lastPulseIndex) + { + // New pulse started + currentFrequency = kBaseFrequency + pulseIdx * kFrequencyStep; + frameData = GenerateToneFrame(kSamplesPerFrame, currentFrequency); + highEnergyFramesRemaining = kHighEnergyFramesPerPulse - 1; + lastPulseIndex = pulseIdx; + } + else + { + frameData = GenerateSilentFrame(kSamplesPerFrame); + } + + audioSource.PushFrame(frameData, kAudioChannels, kAudioSampleRate); + } + } + + var audioFrameCoroutine = TestCoroutineRunner.Start(PushAudioFrames()); + + // --- Warm up --- + Debug.Log($"Warming up for {kAVSyncWarmupSeconds}s..."); + yield return new WaitForSeconds(kAVSyncWarmupSeconds); + + // --- Send simultaneous A/V pulses --- + Debug.Log("Starting A/V pulse transmission..."); + + for (int pulsesSent = 0; pulsesSent < kTotalPulses; pulsesSent++) + { + // Start both audio and video pulse simultaneously + int videoEncoded = VideoPulseCodec.Encode(pulsesSent); + videoSource.SetPulseIndex(videoEncoded); + lock (lockObj) + { + sendTimestamps[pulsesSent] = Stopwatch.GetTimestamp(); + } + currentAudioPulseIndex = pulsesSent; + + Debug.Log($"Sent A/V pulse {pulsesSent + 1}/{kTotalPulses}"); + + // Hold pulse + yield return new WaitForSeconds(kAVSyncPulseDurationSeconds); + + // Reset both to idle + videoSource.SetPulseIndex(VideoPulseCodec.Encode(-1)); + currentAudioPulseIndex = -1; + + // Wait before next pulse + yield return new WaitForSeconds(kAVSyncPulseIntervalSeconds); + } + + // Wait for last echoes + yield return new WaitForSeconds(kAVSyncEchoTimeoutSeconds); + + // Stop audio pusher + audioRunning = false; + yield return null; + + // --- Compute results --- + var audioStats = new LatencyStats(); + var videoStats = new LatencyStats(); + var desyncStats = new LatencyStats(); + int audioMissed = 0, videoMissed = 0, bothReceived = 0; + + lock (lockObj) + { + for (int i = 0; i < kTotalPulses; i++) + { + if (sendTimestamps[i] == 0) continue; + + bool hasAudio = audioReceiveTicks[i] != 0; + bool hasVideo = videoReceiveTicks[i] != 0; + + if (!hasAudio) audioMissed++; + if (!hasVideo) videoMissed++; + + double audioLatencyMs = hasAudio + ? (audioReceiveTicks[i] - sendTimestamps[i]) * 1000.0 / Stopwatch.Frequency + : -1; + double videoLatencyMs = hasVideo + ? (videoReceiveTicks[i] - sendTimestamps[i]) * 1000.0 / Stopwatch.Frequency + : -1; + + if (hasAudio && audioLatencyMs > 0 && audioLatencyMs < 5000) + audioStats.AddMeasurement(audioLatencyMs); + if (hasVideo && videoLatencyMs > 0 && videoLatencyMs < 5000) + videoStats.AddMeasurement(videoLatencyMs); + + if (hasAudio && hasVideo && audioLatencyMs > 0 && videoLatencyMs > 0) + { + double desyncMs = videoLatencyMs - audioLatencyMs; + desyncStats.AddMeasurement(desyncMs); + bothReceived++; + Debug.Log($" Pulse {i}: audio={audioLatencyMs:F2}ms video={videoLatencyMs:F2}ms desync={desyncMs:F2}ms"); + } + } + } + + // --- Print results --- + audioStats.PrintStats("A/V Sync - Audio Latency"); + videoStats.PrintStats("A/V Sync - Video Latency"); + desyncStats.PrintStats("A/V Sync - Desync (positive = video lags audio)"); + + Debug.Log($"Pulses with both A+V received: {bothReceived}/{kTotalPulses}"); + if (audioMissed > 0) Debug.Log($"Audio missed: {audioMissed}"); + if (videoMissed > 0) Debug.Log($"Video missed: {videoMissed}"); + + // --- Cleanup --- + TestCoroutineRunner.Stop(audioFrameCoroutine); + audioStream.Dispose(); + UnityEngine.Object.Destroy(audioReceiverGO); + UnityEngine.Object.Destroy(listenerGO); + audioSource.Stop(); + audioSource.Dispose(); + + videoStream.Stop(); + videoStream.Dispose(); + TestCoroutineRunner.Stop(videoStreamCoroutine); + videoSource.Stop(); + videoSource.Dispose(); + TestCoroutineRunner.Stop(videoSourceCoroutine); + + Assert.Greater(bothReceived, 0, "At least one pulse should be received by both audio and video"); + } + // ===================================================================== // Audio Generation Helpers // =====================================================================