From 4fca4ad2764d2c1564fd48d36af17b467b01cc1a Mon Sep 17 00:00:00 2001 From: Daniil Gusev Date: Mon, 18 May 2026 13:58:15 +0200 Subject: [PATCH] CPU optimizations for Opus encoder to reduce consumption, especially for idle publishers like agents - Default channel layout: "stereo" -> "mono" - Default bitrate: 96kbps -> 64kbps - Default Opus compression level: "10" -> "7" --- getstream/video/rtc/encoders_patches.py | 48 ++++++++++++++--- getstream/video/rtc/pc.py | 4 +- tests/rtc/test_encoders_patches.py | 72 +++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/getstream/video/rtc/encoders_patches.py b/getstream/video/rtc/encoders_patches.py index 7c93445f..de3d7a6b 100644 --- a/getstream/video/rtc/encoders_patches.py +++ b/getstream/video/rtc/encoders_patches.py @@ -7,8 +7,12 @@ from aiortc import RTCRtpCodecParameters from aiortc.codecs.h264 import MAX_FRAME_RATE as H264_MAX_FRAME_RATE from aiortc.codecs.h264 import H264Encoder +from aiortc.codecs.opus import SAMPLE_RATE as OPUS_SAMPLE_RATE +from aiortc.codecs.opus import SAMPLES_PER_FRAME as OPUS_SAMPLES_PER_FRAME +from aiortc.codecs.opus import OpusEncoder from aiortc.codecs.vpx import Vp8Encoder from aiortc.rtcrtpsender import RTCEncodedFrame, RTCRtpSender +from av import AudioResampler logger = logging.getLogger(__name__) @@ -18,6 +22,13 @@ STREAM_VIDEO_MIN_BITRATE = 1_500_000 # 1.5 Mbps STREAM_VIDEO_MAX_BITRATE = 3_000_000 # 3 Mbps +# Override hardcoded Opus defaults to reduce CPU usage. Lower than aiortc's 96 kbps stereo default; tuned for +# voice. Both are configurable per StreamOpusEncoder() instance. +STREAM_OPUS_DEFAULT_BITRATE = 64_000 # 96kbps -> 64kbps +STREAM_OPUS_DEFAULT_LAYOUT = "mono" # stereo -> mono +# Compression level <= 7 reduces the CPU work for encoder with minimal quality impact at 64kbps +STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL = 7 # 10 -> 7 + # Check if the Stream bitrate patching is disabled via environment variable BITRATE_PATCH_DISABLED = os.getenv( @@ -36,6 +47,7 @@ try: + # TODO: Implement a way to configure encoders per track. # Verify the name-mangled attributes we depend on still exist. assert hasattr(Vp8Encoder(), "_Vp8Encoder__target_bitrate") assert hasattr(H264Encoder(), "_H264Encoder__target_bitrate") @@ -124,16 +136,38 @@ def _encode_frame(self, frame, force_keyframe): StreamH264Encoder = None # type: ignore[assignment, misc] +class StreamOpusEncoder(OpusEncoder): + """OpusEncoder subclass with configurable bitrate and channel layout.""" + + def __init__( + self, + bitrate: int = STREAM_OPUS_DEFAULT_BITRATE, + layout: str = STREAM_OPUS_DEFAULT_LAYOUT, + compression_level: int = STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL, + ) -> None: + super().__init__() + self.codec.bit_rate = bitrate + self.codec.layout = layout + self.codec.options = { + "application": "voip", + "compression_level": str(compression_level), + } + # Resampler layout must match codec layout; parent hard-codes stereo. + self.resampler = AudioResampler( + format="s16", + layout=layout, + rate=OPUS_SAMPLE_RATE, + frame_size=OPUS_SAMPLES_PER_FRAME, + ) + + def patch_sender_encoder(sender: RTCRtpSender) -> None: """Patch a sender to use Stream's tuned encoders for the negotiated codec. - Works for video (VP8/H264) senders. If anything + Works for video (VP8/H264) and audio (Opus) senders. If anything goes wrong (e.g. aiortc internals changed), the sender is left untouched and will use the stock encoder via get_encoder(). """ - if StreamVp8Encoder is None or StreamH264Encoder is None: - return - try: _orig_next = sender._next_encoded_frame @@ -142,10 +176,12 @@ async def _next_with_stream_encoder( ) -> Optional[RTCEncodedFrame]: if sender._RTCRtpSender__encoder is None: # type: ignore[attr-defined] mime = codec.mimeType.lower() - if mime == "video/vp8": + if mime == "video/vp8" and StreamVp8Encoder is not None: sender._RTCRtpSender__encoder = StreamVp8Encoder() # type: ignore[attr-defined] - elif mime == "video/h264": + elif mime == "video/h264" and StreamH264Encoder is not None: sender._RTCRtpSender__encoder = StreamH264Encoder() # type: ignore[attr-defined] + elif mime == "audio/opus": + sender._RTCRtpSender__encoder = StreamOpusEncoder() # type: ignore[attr-defined] return await _orig_next(codec) sender._next_encoded_frame = _next_with_stream_encoder # type: ignore[method-assign] diff --git a/getstream/video/rtc/pc.py b/getstream/video/rtc/pc.py index 5b221b59..86605408 100644 --- a/getstream/video/rtc/pc.py +++ b/getstream/video/rtc/pc.py @@ -80,8 +80,8 @@ def addTrack(self, track: MediaStreamTrack) -> RTCRtpSender: if transceiver.sender is sender: transceiver.setCodecPreferences(publish_codec_preferences()) break - if not BITRATE_PATCH_DISABLED: - patch_sender_encoder(sender) + if not BITRATE_PATCH_DISABLED: + patch_sender_encoder(sender) return sender async def handle_answer(self, response): diff --git a/tests/rtc/test_encoders_patches.py b/tests/rtc/test_encoders_patches.py index 80807f3b..7e57f4d6 100644 --- a/tests/rtc/test_encoders_patches.py +++ b/tests/rtc/test_encoders_patches.py @@ -3,13 +3,18 @@ import pytest from aiortc.codecs.h264 import H264Encoder +from aiortc.codecs.opus import OpusEncoder from aiortc.codecs.vpx import Vp8Encoder from getstream.video.rtc.encoders_patches import ( + STREAM_OPUS_DEFAULT_BITRATE, + STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL, + STREAM_OPUS_DEFAULT_LAYOUT, STREAM_VIDEO_DEFAULT_BITRATE, STREAM_VIDEO_MAX_BITRATE, STREAM_VIDEO_MIN_BITRATE, StreamH264Encoder, + StreamOpusEncoder, StreamVp8Encoder, patch_sender_encoder, ) @@ -68,6 +73,31 @@ def test_accepts_in_range(self): assert enc.target_bitrate == 2_000_000 +class TestStreamOpusEncoder: + def test_defaults(self): + enc = StreamOpusEncoder() + assert enc.codec.bit_rate == STREAM_OPUS_DEFAULT_BITRATE + assert enc.codec.layout.name == STREAM_OPUS_DEFAULT_LAYOUT + assert enc.codec.options["compression_level"] == str( + STREAM_OPUS_DEFAULT_COMPRESSION_LEVEL + ) + assert enc.codec.options["application"] == "voip" + + def test_custom_bitrate(self): + enc = StreamOpusEncoder(bitrate=32_000) + assert enc.codec.bit_rate == 32_000 + + def test_custom_layout_updates_resampler(self): + """Resampler layout must track codec layout — encode() asserts on it.""" + enc = StreamOpusEncoder(layout="stereo") + assert enc.codec.layout.name == "stereo" + assert enc.resampler.layout.name == "stereo" + + def test_custom_compression_level(self): + enc = StreamOpusEncoder(compression_level=10) + assert enc.codec.options["compression_level"] == "10" + + class TestBitratePatchDisabled: @pytest.mark.parametrize("env_val", [None, "", "1", "true", "yes", "on"]) def test_enabled_by_default(self, monkeypatch, env_val): @@ -165,6 +195,25 @@ async def _orig_coro(codec): assert isinstance(sender._RTCRtpSender__encoder, H264Cls) + @pytest.mark.asyncio + async def test_installs_opus_encoder(self): + sender = MagicMock() + sender._RTCRtpSender__encoder = None + + async def _orig_coro(codec): + return None + + sender._next_encoded_frame = _orig_coro + patch_sender_encoder(sender) + + codec = MagicMock() + codec.mimeType = "audio/opus" + await sender._next_encoded_frame(codec) + + from getstream.video.rtc.encoders_patches import StreamOpusEncoder as OpusCls + + assert isinstance(sender._RTCRtpSender__encoder, OpusCls) + @pytest.mark.asyncio async def test_does_not_replace_existing_encoder(self): """Already-set encoder is not replaced.""" @@ -183,15 +232,25 @@ async def _orig_coro(codec): await sender._next_encoded_frame(codec) assert sender._RTCRtpSender__encoder is existing_encoder - def test_noop_when_encoders_none(self, monkeypatch): - """patch_sender_encoder is a no-op when encoder classes failed to load.""" + @pytest.mark.asyncio + async def test_skips_mime_when_encoder_class_is_none(self, monkeypatch): + """If a Stream* class failed to load, that mime branch leaves the encoder unset.""" import getstream.video.rtc.encoders_patches as mod monkeypatch.setattr(mod, "StreamVp8Encoder", None) sender = MagicMock() - orig = sender._next_encoded_frame + sender._RTCRtpSender__encoder = None + + async def _orig_coro(codec): + return None + + sender._next_encoded_frame = _orig_coro mod.patch_sender_encoder(sender) - assert sender._next_encoded_frame is orig + + codec = MagicMock() + codec.mimeType = "video/VP8" + await sender._next_encoded_frame(codec) + assert sender._RTCRtpSender__encoder is None class TestUpstreamAssumptions: @@ -216,6 +275,11 @@ def test_upstream_assumptions_hold(self): "H264Encoder name-mangled __target_bitrate changed" ) + # OpusEncoder still exposes the public attrs StreamOpusEncoder overrides. + opus = OpusEncoder() + assert hasattr(opus, "codec"), "OpusEncoder lost codec" + assert hasattr(opus, "resampler"), "OpusEncoder lost resampler" + # RTCRtpSender has _next_encoded_frame and uses __encoder import inspect