diff --git a/.changes/priority-control b/.changes/priority-control new file mode 100644 index 000000000..141c3e4c0 --- /dev/null +++ b/.changes/priority-control @@ -0,0 +1 @@ +patch type="added" "Bitrate priority control APIs" diff --git a/lib/livekit_client.dart b/lib/livekit_client.dart index a0f6b21c6..27276733b 100644 --- a/lib/livekit_client.dart +++ b/lib/livekit_client.dart @@ -59,9 +59,11 @@ export 'src/track/remote/video.dart'; export 'src/track/track.dart'; export 'src/json/agent_attributes.dart'; export 'src/types/data_stream.dart'; +export 'src/types/audio_encoding.dart'; export 'src/types/other.dart'; export 'src/types/participant_permissions.dart'; export 'src/types/participant_state.dart'; +export 'src/types/priority.dart'; export 'src/types/rpc.dart'; export 'src/types/transcription_segment.dart'; export 'src/types/video_dimensions.dart'; diff --git a/lib/src/options.dart b/lib/src/options.dart index 765b4e4bb..af31ce5fd 100644 --- a/lib/src/options.dart +++ b/lib/src/options.dart @@ -17,6 +17,7 @@ import 'e2ee/options.dart'; import 'track/local/audio.dart'; import 'track/local/video.dart'; import 'track/options.dart'; +import 'types/audio_encoding.dart'; import 'types/other.dart'; import 'types/video_encoding.dart'; import 'types/video_parameters.dart'; @@ -301,19 +302,14 @@ class VideoPublishOptions extends PublishOptions { String toString() => '${runtimeType}(videoEncoding: ${videoEncoding}, simulcast: ${simulcast})'; } -class AudioPreset { - static const telephone = 12000; - static const speech = 24000; - static const music = 48000; - static const musicStereo = 64000; - static const musicHighQuality = 96000; - static const musicHighQualityStereo = 128000; -} - /// Options used when publishing audio. class AudioPublishOptions extends PublishOptions { static const defaultMicrophoneName = 'microphone'; + /// Preferred encoding parameters. + /// Defaults to [AudioEncoding.presetMusic] when not set. + final AudioEncoding? encoding; + /// Whether to enable DTX (Discontinuous Transmission) or not. /// https://en.wikipedia.org/wiki/Discontinuous_transmission /// Defaults to true. @@ -322,9 +318,6 @@ class AudioPublishOptions extends PublishOptions { /// red (Redundant Audio Data) final bool? red; - /// max audio bitrate - final int audioBitrate; - /// Mark this audio as originating from a pre-connect buffer. /// Used to populate protobuf audioFeatures (TF_PRECONNECT_BUFFER). final bool preConnect; @@ -332,23 +325,23 @@ class AudioPublishOptions extends PublishOptions { const AudioPublishOptions({ super.name, super.stream, + this.encoding, this.dtx = true, this.red = true, - this.audioBitrate = AudioPreset.music, this.preConnect = false, }); AudioPublishOptions copyWith({ + AudioEncoding? encoding, bool? dtx, - int? audioBitrate, String? name, String? stream, bool? red, bool? preConnect, }) => AudioPublishOptions( + encoding: encoding ?? this.encoding, dtx: dtx ?? this.dtx, - audioBitrate: audioBitrate ?? this.audioBitrate, name: name ?? this.name, stream: stream ?? this.stream, red: red ?? this.red, @@ -356,8 +349,7 @@ class AudioPublishOptions extends PublishOptions { ); @override - String toString() => - '${runtimeType}(dtx: ${dtx}, audioBitrate: ${audioBitrate}, red: ${red}, preConnect: ${preConnect})'; + String toString() => '${runtimeType}(encoding: ${encoding}, dtx: ${dtx}, red: ${red}, preConnect: ${preConnect})'; } final backupCodecs = ['vp8', 'h264']; diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 47e623df5..dc0f39948 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -48,6 +48,7 @@ import '../track/local/audio.dart'; import '../track/local/local.dart'; import '../track/local/video.dart'; import '../track/options.dart'; +import '../types/audio_encoding.dart'; import '../types/data_stream.dart'; import '../types/other.dart'; import '../types/participant_permissions.dart'; @@ -123,11 +124,8 @@ class LocalParticipant extends Participant { // Use defaultPublishOptions if options is null publishOptions ??= track.lastPublishOptions ?? room.roomOptions.defaultAudioPublishOptions; - final List encodings = [ - rtc.RTCRtpEncoding( - maxBitrate: publishOptions.audioBitrate, - ) - ]; + final audioEncoding = publishOptions.encoding ?? AudioEncoding.presetMusic; + final List encodings = [audioEncoding.toRTCRtpEncoding()]; final req = lk_rtc.AddTrackRequest( cid: track.getCid(), @@ -162,9 +160,7 @@ class LocalParticipant extends Participant { final transceiverInit = rtc.RTCRtpTransceiverInit( direction: rtc.TransceiverDirection.SendOnly, - sendEncodings: [ - if (publishOptions.audioBitrate > 0) rtc.RTCRtpEncoding(maxBitrate: publishOptions.audioBitrate), - ], + sendEncodings: encodings, ); // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web) track.transceiver = await room.engine.publisher?.pc.addTransceiver( diff --git a/lib/src/track/web/_audio_analyser.dart b/lib/src/track/web/_audio_analyser.dart index e3c25c343..087b1bbc3 100644 --- a/lib/src/track/web/_audio_analyser.dart +++ b/lib/src/track/web/_audio_analyser.dart @@ -2,8 +2,7 @@ import 'dart:js_interop'; import 'dart:js_interop_unsafe'; import 'dart:math' as math; -import 'package:dart_webrtc/dart_webrtc.dart' show MediaStreamTrackWeb; -import 'package:dart_webrtc/dart_webrtc.dart' show MediaStreamWeb; +import 'package:dart_webrtc/dart_webrtc.dart' show MediaStreamTrackWeb, MediaStreamWeb; import 'package:web/web.dart' as web; import '../../track/local/local.dart' show AudioTrack; diff --git a/lib/src/types/audio_encoding.dart b/lib/src/types/audio_encoding.dart new file mode 100644 index 000000000..ee277299a --- /dev/null +++ b/lib/src/types/audio_encoding.dart @@ -0,0 +1,88 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:meta/meta.dart'; + +import 'priority.dart'; + +/// A type that represents audio encoding information. +@immutable +class AudioEncoding { + /// Maximum bitrate for the audio track. + final int maxBitrate; + + /// Priority for bandwidth allocation. + final Priority? bitratePriority; + + /// Priority for DSCP marking. Requires `RTCConfiguration.isDscpEnabled` to be true. + final Priority? networkPriority; + + const AudioEncoding({ + required this.maxBitrate, + this.bitratePriority, + this.networkPriority, + }); + + AudioEncoding copyWith({ + int? maxBitrate, + Priority? bitratePriority, + Priority? networkPriority, + }) => + AudioEncoding( + maxBitrate: maxBitrate ?? this.maxBitrate, + bitratePriority: bitratePriority ?? this.bitratePriority, + networkPriority: networkPriority ?? this.networkPriority, + ); + + @override + String toString() => + '${runtimeType}(maxBitrate: ${maxBitrate}, bitratePriority: ${bitratePriority}, networkPriority: ${networkPriority})'; + + @override + bool operator ==(Object other) => + identical(this, other) || + other is AudioEncoding && + maxBitrate == other.maxBitrate && + bitratePriority == other.bitratePriority && + networkPriority == other.networkPriority; + + @override + int get hashCode => Object.hash(maxBitrate, bitratePriority, networkPriority); + + static const presetTelephone = AudioEncoding(maxBitrate: 12000); + static const presetSpeech = AudioEncoding(maxBitrate: 24000); + static const presetMusic = AudioEncoding(maxBitrate: 48000); + static const presetMusicStereo = AudioEncoding(maxBitrate: 64000); + static const presetMusicHighQuality = AudioEncoding(maxBitrate: 96000); + static const presetMusicHighQualityStereo = AudioEncoding(maxBitrate: 128000); + + static const presets = [ + presetTelephone, + presetSpeech, + presetMusic, + presetMusicStereo, + presetMusicHighQuality, + presetMusicHighQualityStereo, + ]; +} + +/// Convenience extension for [AudioEncoding]. +extension AudioEncodingExt on AudioEncoding { + rtc.RTCRtpEncoding toRTCRtpEncoding() => rtc.RTCRtpEncoding( + maxBitrate: maxBitrate, + priority: bitratePriority?.toRtcpPriorityType() ?? rtc.RTCPriorityType.low, + networkPriority: networkPriority?.toRtcpPriorityType(), + ); +} diff --git a/lib/src/types/other.dart b/lib/src/types/other.dart index f5c1dd7bc..46f39b07b 100644 --- a/lib/src/types/other.dart +++ b/lib/src/types/other.dart @@ -141,11 +141,21 @@ class RTCConfiguration { final RTCIceTransportPolicy? iceTransportPolicy; final bool? encodedInsertableStreams; + /// Allows DSCP (Differentiated Services Code Point) codes to be set on + /// outgoing packets for network level QoS. + /// + /// This is a best effort hint and network routers may ignore DSCP markings. + /// Required for `networkPriority` to take effect. + /// + /// Ignored on web platforms. + final bool? isDscpEnabled; + const RTCConfiguration({ this.iceCandidatePoolSize, this.iceServers, this.iceTransportPolicy, this.encodedInsertableStreams, + this.isDscpEnabled, }); Map toMap() { @@ -158,6 +168,7 @@ class RTCConfiguration { // only supports unified plan 'sdpSemantics': 'unified-plan', if (encodedInsertableStreams != null) 'encodedInsertableStreams': encodedInsertableStreams, + if (isDscpEnabled != null) 'enableDscp': isDscpEnabled, if (iceServersMap.isNotEmpty) 'iceServers': iceServersMap, if (iceCandidatePoolSize != null) 'iceCandidatePoolSize': iceCandidatePoolSize, if (iceTransportPolicy != null) 'iceTransportPolicy': iceTransportPolicy!.toStringValue(), @@ -170,12 +181,14 @@ class RTCConfiguration { List? iceServers, RTCIceTransportPolicy? iceTransportPolicy, bool? encodedInsertableStreams, + bool? isDscpEnabled, }) => RTCConfiguration( iceCandidatePoolSize: iceCandidatePoolSize ?? this.iceCandidatePoolSize, iceServers: iceServers ?? this.iceServers, iceTransportPolicy: iceTransportPolicy ?? this.iceTransportPolicy, encodedInsertableStreams: encodedInsertableStreams ?? this.encodedInsertableStreams, + isDscpEnabled: isDscpEnabled ?? this.isDscpEnabled, ); } diff --git a/lib/src/types/priority.dart b/lib/src/types/priority.dart new file mode 100644 index 000000000..63dc9a260 --- /dev/null +++ b/lib/src/types/priority.dart @@ -0,0 +1,41 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; + +/// Priority levels for RTP encoding parameters. +/// +/// `bitratePriority` controls WebRTC internal bandwidth allocation between streams. +/// `networkPriority` controls DSCP marking for network-level QoS. +enum Priority { + veryLow, + low, + medium, + high, +} + +extension PriorityExt on Priority { + rtc.RTCPriorityType toRtcpPriorityType() { + switch (this) { + case Priority.veryLow: + return rtc.RTCPriorityType.veryLow; + case Priority.low: + return rtc.RTCPriorityType.low; + case Priority.medium: + return rtc.RTCPriorityType.medium; + case Priority.high: + return rtc.RTCPriorityType.high; + } + } +} diff --git a/lib/src/types/video_encoding.dart b/lib/src/types/video_encoding.dart index 72c309fa9..4f24694bb 100644 --- a/lib/src/types/video_encoding.dart +++ b/lib/src/types/video_encoding.dart @@ -15,28 +15,46 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; +import 'priority.dart'; + /// A type that represents video encoding information. @immutable class VideoEncoding implements Comparable { + /// Maximum framerate for the video track. final int maxFramerate; + + /// Maximum bitrate for the video track. final int maxBitrate; + /// Priority for bandwidth allocation. + final Priority? bitratePriority; + + /// Priority for DSCP marking. Requires `RTCConfiguration.isDscpEnabled` to be true. + final Priority? networkPriority; + const VideoEncoding({ required this.maxFramerate, required this.maxBitrate, + this.bitratePriority, + this.networkPriority, }); VideoEncoding copyWith({ int? maxFramerate, int? maxBitrate, + Priority? bitratePriority, + Priority? networkPriority, }) => VideoEncoding( maxFramerate: maxFramerate ?? this.maxFramerate, maxBitrate: maxBitrate ?? this.maxBitrate, + bitratePriority: bitratePriority ?? this.bitratePriority, + networkPriority: networkPriority ?? this.networkPriority, ); @override - String toString() => '${runtimeType}(maxFramerate: ${maxFramerate}, maxBitrate: ${maxBitrate})'; + String toString() => + '${runtimeType}(maxFramerate: ${maxFramerate}, maxBitrate: ${maxBitrate}, bitratePriority: ${bitratePriority}, networkPriority: ${networkPriority})'; // ---------------------------------------------------------------------- // equality @@ -44,10 +62,14 @@ class VideoEncoding implements Comparable { @override bool operator ==(Object other) => identical(this, other) || - other is VideoEncoding && maxFramerate == other.maxFramerate && maxBitrate == other.maxBitrate; + other is VideoEncoding && + maxFramerate == other.maxFramerate && + maxBitrate == other.maxBitrate && + bitratePriority == other.bitratePriority && + networkPriority == other.networkPriority; @override - int get hashCode => Object.hash(maxFramerate, maxBitrate); + int get hashCode => Object.hash(maxFramerate, maxBitrate, bitratePriority, networkPriority); // ---------------------------------------------------------------------- // Comparable @@ -55,13 +77,18 @@ class VideoEncoding implements Comparable { @override int compareTo(VideoEncoding other) { // compare bitrates - final result = maxBitrate.compareTo(other.maxBitrate); - // if bitrates are the same, compare by fps - if (result == 0) { - return maxFramerate.compareTo(other.maxFramerate); - } + var result = maxBitrate.compareTo(other.maxBitrate); + if (result != 0) return result; + + // compare by fps + result = maxFramerate.compareTo(other.maxFramerate); + if (result != 0) return result; + + // compare by priority fields for consistency with == and hashCode + result = (bitratePriority?.index ?? -1).compareTo(other.bitratePriority?.index ?? -1); + if (result != 0) return result; - return result; + return (networkPriority?.index ?? -1).compareTo(other.networkPriority?.index ?? -1); } } @@ -78,5 +105,7 @@ extension VideoEncodingExt on VideoEncoding { maxFramerate: maxFramerate, maxBitrate: maxBitrate, numTemporalLayers: numTemporalLayers, + priority: bitratePriority?.toRtcpPriorityType() ?? rtc.RTCPriorityType.low, + networkPriority: networkPriority?.toRtcpPriorityType(), ); } diff --git a/lib/src/utils.dart b/lib/src/utils.dart index d2390de9e..d242e6d90 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -33,6 +33,7 @@ import 'options.dart'; import 'support/platform.dart'; import 'track/local/video.dart'; import 'types/other.dart'; +import 'types/priority.dart'; import 'types/video_dimensions.dart'; import 'types/video_encoding.dart'; import 'types/video_parameters.dart'; @@ -428,6 +429,8 @@ class Utils { rid: videoRids[2 - i], maxBitrate: videoEncoding.maxBitrate ~/ math.pow(3, i), maxFramerate: original.encoding!.maxFramerate, + priority: videoEncoding.bitratePriority?.toRtcpPriorityType() ?? rtc.RTCPriorityType.low, + networkPriority: videoEncoding.networkPriority?.toRtcpPriorityType(), )); } } else { diff --git a/pubspec.lock b/pubspec.lock index 8d498a6df..d5b313878 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -205,10 +205,10 @@ packages: dependency: "direct main" description: name: dart_webrtc - sha256: "51bcda4ba5d7dd9e65a309244ce3ac0b58025e6e1f6d7442cee4cd02134ef65f" + sha256: "4ed7b9fa9924e5a81eb39271e2c2356739dd1039d60a13b86ba6c5f448625086" url: "https://pub.dev" source: hosted - version: "1.6.0" + version: "1.7.0" dbus: dependency: transitive description: @@ -292,10 +292,10 @@ packages: dependency: "direct main" description: name: flutter_webrtc - sha256: "71a38363a5b50603e405c275f30de2eb90f980b0cc94b0e1e9d8b9d6a6b03bf0" + sha256: "0f86b518e9349e71a136a96e0ea11294cad8a8531b2bc9ae99e69df332ac898a" url: "https://pub.dev" source: hosted - version: "1.2.1" + version: "1.3.0" frontend_server_client: dependency: transitive description: @@ -809,10 +809,10 @@ packages: dependency: transitive description: name: webrtc_interface - sha256: "2e604a31703ad26781782fb14fa8a4ee621154ee2c513d2b9938e486fa695233" + sha256: ad0e5786b2acd3be72a3219ef1dde9e1cac071cf4604c685f11b61d63cdd6eb3 url: "https://pub.dev" source: hosted - version: "1.3.0" + version: "1.4.0" win32: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 27150709b..a5d8a6fbf 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -46,8 +46,8 @@ dependencies: json_annotation: ^4.9.0 # Fix version to avoid version conflicts between WebRTC-SDK pods, which both this package and flutter_webrtc depend on. - flutter_webrtc: 1.2.1 - dart_webrtc: ^1.6.0 + flutter_webrtc: 1.3.0 + dart_webrtc: ^1.7.0 dev_dependencies: flutter_test: