Skip to content

Commit 0d755e5

Browse files
chenosauruschenosaurus
authored
Use shader for YUV to RGB conversion on received frames (livekit#168)
Improve performance on video render pipeline by moving YUV-> RGB conversion into a shader
1 parent fe49cfb commit 0d755e5

8 files changed

Lines changed: 274 additions & 17 deletions

File tree

‎LICENSE.txt.meta‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Scripts/Video.meta‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Scripts/Video/YuvToRgbConverter.cs‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
using System;
2+
using UnityEngine;
3+
4+
namespace LiveKit
5+
{
6+
// Converts I420 YUV frames to RGBA into an output RenderTexture, via GPU shader or CPU fallback.
7+
internal sealed class YuvToRgbConverter : IDisposable
8+
{
9+
public bool UseGpuShader { get; set; } = true;
10+
public RenderTexture Output { get; private set; }
11+
12+
private Material _yuvToRgbMaterial;
13+
private Texture2D _planeY;
14+
private Texture2D _planeU;
15+
private Texture2D _planeV;
16+
17+
// Ensure Output exists and matches the given size; returns true if created or resized.
18+
public bool EnsureOutput(int width, int height)
19+
{
20+
var changed = false;
21+
if (Output == null || Output.width != width || Output.height != height)
22+
{
23+
if (Output != null)
24+
{
25+
Output.Release();
26+
UnityEngine.Object.Destroy(Output);
27+
}
28+
Output = new RenderTexture(width, height, 0, RenderTextureFormat.ARGB32);
29+
Output.Create();
30+
changed = true;
31+
}
32+
return changed;
33+
}
34+
35+
// Convert the given buffer to RGBA and write into Output.
36+
public void Convert(VideoFrameBuffer buffer)
37+
{
38+
if (buffer == null || !buffer.IsValid)
39+
return;
40+
41+
int width = (int)buffer.Width;
42+
int height = (int)buffer.Height;
43+
44+
EnsureOutput(width, height);
45+
46+
if (UseGpuShader)
47+
{
48+
EnsureGpuMaterial();
49+
EnsureYuvPlaneTextures(width, height);
50+
UploadYuvPlanes(buffer);
51+
52+
if (_yuvToRgbMaterial != null)
53+
{
54+
GpuConvertToRenderTarget();
55+
return;
56+
}
57+
// fall through to CPU if shader missing
58+
}
59+
60+
CpuConvertToRenderTarget(buffer, width, height);
61+
}
62+
63+
// Release all Unity resources (RT, material, textures).
64+
public void Dispose()
65+
{
66+
if (_planeY != null) UnityEngine.Object.Destroy(_planeY);
67+
if (_planeU != null) UnityEngine.Object.Destroy(_planeU);
68+
if (_planeV != null) UnityEngine.Object.Destroy(_planeV);
69+
if (Output != null)
70+
{
71+
Output.Release();
72+
UnityEngine.Object.Destroy(Output);
73+
}
74+
if (_yuvToRgbMaterial != null) UnityEngine.Object.Destroy(_yuvToRgbMaterial);
75+
}
76+
77+
// Ensure the GPU YUV->RGB material exists.
78+
private void EnsureGpuMaterial()
79+
{
80+
if (_yuvToRgbMaterial == null)
81+
{
82+
var shader = Shader.Find("Hidden/LiveKit/YUV2RGB");
83+
if (shader != null)
84+
_yuvToRgbMaterial = new Material(shader);
85+
}
86+
}
87+
88+
// Ensure or recreate a plane texture with given format and filter settings.
89+
private static void EnsurePlaneTexture(ref Texture2D tex, int width, int height, TextureFormat format, FilterMode filterMode)
90+
{
91+
if (tex == null || tex.width != width || tex.height != height)
92+
{
93+
if (tex != null) UnityEngine.Object.Destroy(tex);
94+
tex = new Texture2D(width, height, format, false, true);
95+
tex.filterMode = filterMode;
96+
tex.wrapMode = TextureWrapMode.Clamp;
97+
}
98+
}
99+
100+
// Ensure Y, U, V plane textures exist with correct dimensions.
101+
private void EnsureYuvPlaneTextures(int width, int height)
102+
{
103+
EnsurePlaneTexture(ref _planeY, width, height, TextureFormat.R8, FilterMode.Bilinear);
104+
var chromaW = width / 2;
105+
var chromaH = height / 2;
106+
EnsurePlaneTexture(ref _planeU, chromaW, chromaH, TextureFormat.R8, FilterMode.Bilinear);
107+
EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Bilinear);
108+
}
109+
110+
// Upload raw Y, U, V plane bytes from buffer to textures.
111+
private void UploadYuvPlanes(VideoFrameBuffer buffer)
112+
{
113+
var info = buffer.Info;
114+
if (info.Components.Count < 3) return;
115+
var yComp = info.Components[0];
116+
var uComp = info.Components[1];
117+
var vComp = info.Components[2];
118+
119+
_planeY.LoadRawTextureData((IntPtr)yComp.DataPtr, (int)yComp.Size);
120+
_planeY.Apply(false, false);
121+
_planeU.LoadRawTextureData((IntPtr)uComp.DataPtr, (int)uComp.Size);
122+
_planeU.Apply(false, false);
123+
_planeV.LoadRawTextureData((IntPtr)vComp.DataPtr, (int)vComp.Size);
124+
_planeV.Apply(false, false);
125+
}
126+
127+
// CPU-side conversion to RGBA and blit to the output render target.
128+
private void CpuConvertToRenderTarget(VideoFrameBuffer buffer, int width, int height)
129+
{
130+
var rgba = buffer.ToRGBA();
131+
var tempTex = new Texture2D(width, height, TextureFormat.RGBA32, false);
132+
try
133+
{
134+
tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize());
135+
tempTex.Apply();
136+
Graphics.Blit(tempTex, Output);
137+
}
138+
finally
139+
{
140+
UnityEngine.Object.Destroy(tempTex);
141+
rgba.Dispose();
142+
}
143+
}
144+
145+
// GPU-side YUV->RGB conversion using shader material.
146+
private void GpuConvertToRenderTarget()
147+
{
148+
_yuvToRgbMaterial.SetTexture("_TexY", _planeY);
149+
_yuvToRgbMaterial.SetTexture("_TexU", _planeU);
150+
_yuvToRgbMaterial.SetTexture("_TexV", _planeV);
151+
Graphics.Blit(Texture2D.blackTexture, Output, _yuvToRgbMaterial);
152+
}
153+
}
154+
}
155+

‎Runtime/Scripts/Video/YuvToRgbConverter.cs.meta‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Scripts/VideoStream.cs‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,14 @@ namespace LiveKit
1010
public class VideoStream
1111
{
1212
public delegate void FrameReceiveDelegate(VideoFrame frame);
13-
public delegate void TextureReceiveDelegate(Texture2D tex2d);
13+
public delegate void TextureReceiveDelegate(Texture tex);
1414
public delegate void TextureUploadDelegate();
1515

1616
internal readonly FfiHandle Handle;
1717
private VideoStreamInfo _info;
1818
private bool _disposed = false;
1919
private bool _dirty = false;
20+
private YuvToRgbConverter _converter;
2021

2122
/// Called when we receive a new frame from the VideoTrack
2223
public event FrameReceiveDelegate FrameReceived;
@@ -29,7 +30,7 @@ public class VideoStream
2930

3031
/// The texture changes every time the video resolution changes.
3132
/// Can be null if UpdateRoutine isn't started
32-
public Texture2D Texture { private set; get; }
33+
public RenderTexture Texture { private set; get; }
3334
public VideoFrameBuffer VideoBuffer { private set; get; }
3435

3536
protected bool _playing = false;
@@ -70,8 +71,14 @@ private void Dispose(bool disposing)
7071
if (!_disposed)
7172
{
7273
if (disposing)
74+
{
7375
VideoBuffer?.Dispose();
74-
if (Texture != null) UnityEngine.Object.Destroy(Texture);
76+
}
77+
// Unity objects must be destroyed on main thread
78+
_converter?.Dispose();
79+
_converter = null;
80+
// Texture is owned and cleaned up by _converter. Set to null to avoid holding a reference to a disposed RenderTexture.
81+
Texture = null;
7582
_disposed = true;
7683
}
7784
}
@@ -103,30 +110,21 @@ public IEnumerator Update()
103110
var rWidth = VideoBuffer.Width;
104111
var rHeight = VideoBuffer.Height;
105112

106-
var textureChanged = false;
107-
if (Texture == null || Texture.width != rWidth || Texture.height != rHeight)
108-
{
109-
if (Texture != null) UnityEngine.Object.Destroy(Texture);
110-
Texture = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false);
111-
Texture.ignoreMipmapLimit = false;
112-
textureChanged = true;
113-
}
114-
var rgba = VideoBuffer.ToRGBA();
115-
{
116-
Texture.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize());
117-
}
118-
Texture.Apply();
113+
if (_converter == null) _converter = new YuvToRgbConverter();
114+
var textureChanged = _converter.EnsureOutput((int)rWidth, (int)rHeight);
115+
_converter.Convert(VideoBuffer);
116+
if (textureChanged) Texture = _converter.Output;
119117

120118
if (textureChanged)
121119
TextureReceived?.Invoke(Texture);
122120

123121
TextureUploaded?.Invoke();
124-
rgba.Dispose();
125122
}
126123

127124
yield break;
128125
}
129126

127+
// Handle new video stream events
130128
private void OnVideoStreamEvent(VideoStreamEvent e)
131129
{
132130
if (e.StreamHandle != (ulong)Handle.DangerousGetHandle())

‎Runtime/Shaders.meta‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Runtime/Shaders/YuvToRgb.shader‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
Shader "Hidden/LiveKit/YUV2RGB"
2+
{
3+
SubShader
4+
{
5+
Tags { "RenderType" = "Opaque" "Queue" = "Geometry" }
6+
Pass
7+
{
8+
ZTest Always Cull Off ZWrite Off
9+
10+
HLSLPROGRAM
11+
#pragma vertex vert
12+
#pragma fragment frag
13+
#include "UnityCG.cginc"
14+
15+
sampler2D _TexY;
16+
sampler2D _TexU;
17+
sampler2D _TexV;
18+
19+
struct appdata
20+
{
21+
float4 vertex : POSITION;
22+
float2 uv : TEXCOORD0;
23+
};
24+
25+
struct v2f
26+
{
27+
float4 pos : SV_POSITION;
28+
half2 uv : TEXCOORD0;
29+
};
30+
31+
v2f vert(appdata v)
32+
{
33+
v2f o;
34+
o.pos = UnityObjectToClipPos(v.vertex);
35+
o.uv = half2(v.uv);
36+
return o;
37+
}
38+
39+
inline half3 yuvToRgb709Limited(half y, half u, half v)
40+
{
41+
// BT.709 limited range
42+
half c = y - half(16.0 / 255.0);
43+
half d = u - half(128.0 / 255.0);
44+
half e = v - half(128.0 / 255.0);
45+
46+
half Y = half(1.16438356) * c;
47+
48+
half3 rgb;
49+
rgb.r = Y + half(1.79274107) * e;
50+
rgb.g = Y - half(0.21324861) * d - half(0.53290933) * e;
51+
rgb.b = Y + half(2.11240179) * d;
52+
return saturate(rgb);
53+
}
54+
55+
half4 frag(v2f i) : SV_Target
56+
{
57+
// Flip horizontally to match Unity's texture orientation with incoming YUV data
58+
half2 uv = half2(1.0h - i.uv.x, i.uv.y);
59+
60+
half y = tex2D(_TexY, uv).r;
61+
half u = tex2D(_TexU, uv).r;
62+
half v = tex2D(_TexV, uv).r;
63+
return half4(yuvToRgb709Limited(y, u, v), 1.0h);
64+
}
65+
ENDHLSL
66+
}
67+
}
68+
}
69+
70+

‎Runtime/Shaders/YuvToRgb.shader.meta‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)