From f90275ffa4e3ff52fa65bd1ecce9a9bc0202f835 Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Wed, 5 Feb 2025 10:26:57 +0800 Subject: [PATCH] fix: Improve speaker switching logic for iOS. (#692) * fix: Improve speaker switching logic for iOS. * update. * fix. --- example/lib/widgets/controls.dart | 5 +-- lib/src/core/engine.dart | 8 ++-- lib/src/core/room.dart | 15 ++++--- lib/src/core/transport.dart | 12 ++--- lib/src/hardware/hardware.dart | 51 +++++++++++++-------- lib/src/support/native.dart | 5 --- lib/src/support/native_audio.dart | 55 +++++++++++++++++++---- lib/src/track/audio_management.dart | 54 +++++++--------------- lib/src/track/track.dart | 2 + lib/src/widgets/video_track_renderer.dart | 16 ++++--- shared_swift/LiveKitPlugin.swift | 4 ++ 11 files changed, 131 insertions(+), 96 deletions(-) diff --git a/example/lib/widgets/controls.dart b/example/lib/widgets/controls.dart index 977ab156f..79f6d6c7e 100644 --- a/example/lib/widgets/controls.dart +++ b/example/lib/widgets/controls.dart @@ -107,14 +107,13 @@ class _ControlsWidgetState extends State { setState(() {}); } - void _setSpeakerphoneOn() { + void _setSpeakerphoneOn() async { _speakerphoneOn = !_speakerphoneOn; - Hardware.instance.setSpeakerphoneOn(_speakerphoneOn); + await widget.room.setSpeakerOn(_speakerphoneOn, forceSpeakerOutput: false); setState(() {}); } void _toggleCamera() async { - // final track = participant.videoTrackPublications.firstOrNull?.track; if (track == null) return; diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 9c73ab50b..413e65ccd 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -541,8 +541,8 @@ class Engine extends Disposable with EventsEmittable { type: Reliability.lossy, ))); // _onDCStateUpdated(Reliability.lossy, state) - } catch (_) { - logger.severe('[$objectId] createDataChannel() did throw $_'); + } catch (err) { + logger.severe('[$objectId] createDataChannel() did throw $err'); } try { @@ -558,8 +558,8 @@ class Engine extends Disposable with EventsEmittable { state: state, type: Reliability.reliable, ))); - } catch (_) { - logger.severe('[$objectId] createDataChannel() did throw $_'); + } catch (err) { + logger.severe('[$objectId] createDataChannel() did throw $err'); } } diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index bdcdb1145..bff4b68dd 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -1066,9 +1066,15 @@ extension RoomHardwareManagementMethods on Room { ); } - Future setSpeakerOn(bool speakerOn) async { - if (lkPlatformIs(PlatformType.iOS) || lkPlatformIs(PlatformType.android)) { - await Hardware.instance.setSpeakerphoneOn(speakerOn); + /// [speakerOn] set speakerphone on or off, by default wired/bluetooth headsets will still + /// be prioritized even if set to true. + /// [forceSpeakerOutput] if true, will force speaker output even if headphones + /// or bluetooth is connected, only supported on iOS for now + Future setSpeakerOn(bool speakerOn, + {bool forceSpeakerOutput = false}) async { + if (lkPlatformIsMobile()) { + await Hardware.instance + .setSpeakerphoneOn(speakerOn, forceSpeakerOutput: forceSpeakerOutput); engine.roomOptions = engine.roomOptions.copyWith( defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith( @@ -1082,8 +1088,7 @@ extension RoomHardwareManagementMethods on Room { @internal Future applyAudioSpeakerSettings() async { if (roomOptions.defaultAudioOutputOptions.speakerOn != null) { - if (lkPlatformIs(PlatformType.iOS) || - lkPlatformIs(PlatformType.android)) { + if (lkPlatformIsMobile()) { await Hardware.instance.setSpeakerphoneOn( roomOptions.defaultAudioOutputOptions.speakerOn!); } diff --git a/lib/src/core/transport.dart b/lib/src/core/transport.dart index 8eea6ac65..c6e684ead 100644 --- a/lib/src/core/transport.dart +++ b/lib/src/core/transport.dart @@ -85,15 +85,15 @@ class Transport extends Disposable { List senders = []; try { senders = await pc.getSenders(); - } catch (_) { - logger.warning('getSenders() failed with error: $_'); + } catch (err) { + logger.warning('getSenders() failed with error: $err'); } for (final e in senders) { try { await pc.removeTrack(e); - } catch (_) { - logger.warning('removeTrack() failed with error: $_'); + } catch (err) { + logger.warning('removeTrack() failed with error: $err'); } } @@ -261,8 +261,8 @@ class Transport extends Disposable { try { final result = await pc.getRemoteDescription(); return result; - } catch (_) { - logger.warning('pc.getRemoteDescription failed with error: $_'); + } catch (err) { + logger.warning('pc.getRemoteDescription failed with error: $err'); } return null; } diff --git a/lib/src/hardware/hardware.dart b/lib/src/hardware/hardware.dart index dd44a01fb..7ba971158 100644 --- a/lib/src/hardware/hardware.dart +++ b/lib/src/hardware/hardware.dart @@ -76,14 +76,18 @@ class Hardware { MediaDevice? selectedVideoInput; - bool? _speakerOn; + bool? get speakerOn => _preferSpeakerOutput; - bool? get speakerOn => _speakerOn; - - bool _preferSpeakerOutput = false; + bool _preferSpeakerOutput = true; bool get preferSpeakerOutput => _preferSpeakerOutput; + bool _forceSpeakerOutput = false; + + /// if true, will force speaker output even if headphones or bluetooth is connected + /// only supported on iOS for now + bool get forceSpeakerOutput => _forceSpeakerOutput && _preferSpeakerOutput; + Future> enumerateDevices({String? type}) async { var infos = await rtc.navigator.mediaDevices.enumerateDevices(); var devices = infos @@ -126,13 +130,32 @@ class Hardware { await rtc.Helper.selectAudioInput(device.deviceId); } - Future setPreferSpeakerOutput(bool enable) async { - if (lkPlatformIs(PlatformType.iOS)) { - if (_preferSpeakerOutput != enable) { + @Deprecated('use setSpeakerphoneOn') + Future setPreferSpeakerOutput(bool enable) => setSpeakerphoneOn(enable); + + bool get canSwitchSpeakerphone => lkPlatformIsMobile(); + + /// [enable] set speakerphone on or off, by default wired/bluetooth headsets will still + /// be prioritized even if set to true. + /// [forceSpeakerOutput] if true, will force speaker output even if headphones + /// or bluetooth is connected, only supported on iOS for now + Future setSpeakerphoneOn(bool enable, + {bool forceSpeakerOutput = false}) async { + if (canSwitchSpeakerphone) { + _preferSpeakerOutput = enable; + _forceSpeakerOutput = forceSpeakerOutput; + if (lkPlatformIs(PlatformType.iOS)) { NativeAudioConfiguration? config; if (lkPlatformIs(PlatformType.iOS)) { // Only iOS for now... config = await onConfigureNativeAudio.call(audioTrackState); + if (_preferSpeakerOutput && _forceSpeakerOutput) { + config = config.copyWith( + appleAudioCategoryOptions: { + AppleAudioCategoryOption.defaultToSpeaker, + }, + ); + } logger.fine('configuring for ${audioTrackState} using ${config}...'); try { await Native.configureAudio(config); @@ -140,19 +163,9 @@ class Hardware { logger.warning('failed to configure ${error}'); } } + } else { + await rtc.Helper.setSpeakerphoneOn(enable); } - _preferSpeakerOutput = enable; - } else { - logger.warning('setPreferSpeakerOutput only support on iOS'); - } - } - - bool get canSwitchSpeakerphone => lkPlatformIsMobile(); - - Future setSpeakerphoneOn(bool enable) async { - if (canSwitchSpeakerphone) { - _speakerOn = enable; - await rtc.Helper.setSpeakerphoneOn(enable); } else { logger.warning('setSpeakerphoneOn only support on iOS/Android'); } diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index 459d02e24..c5fb83ee0 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -31,11 +31,6 @@ class Native { static Future configureAudio( NativeAudioConfiguration configuration) async { try { - if (bypassVoiceProcessing) { - /// skip configuring audio if bypassVoiceProcessing - /// is enabled - return false; - } final result = await channel.invokeMethod( 'configureNativeAudio', configuration.toMap(), diff --git a/lib/src/support/native_audio.dart b/lib/src/support/native_audio.dart index 2e3e92f2d..2409d2f58 100644 --- a/lib/src/support/native_audio.dart +++ b/lib/src/support/native_audio.dart @@ -86,15 +86,50 @@ class NativeAudioConfiguration { final AppleAudioCategory? appleAudioCategory; final Set? appleAudioCategoryOptions; final AppleAudioMode? appleAudioMode; + final bool? preferSpeakerOutput; - NativeAudioConfiguration({ - // for iOS / Mac - this.appleAudioCategory, - this.appleAudioCategoryOptions, - this.appleAudioMode, - // Android options - // ... - }); + static final soloAmbient = NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.soloAmbient, + appleAudioCategoryOptions: {}, + appleAudioMode: AppleAudioMode.default_, + ); + + static final playback = NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playback, + appleAudioCategoryOptions: {AppleAudioCategoryOption.mixWithOthers}, + appleAudioMode: AppleAudioMode.spokenAudio, + ); + + static final playAndRecordSpeaker = NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + appleAudioMode: AppleAudioMode.videoChat, + ); + + static final playAndRecordReceiver = NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + appleAudioMode: AppleAudioMode.voiceChat, + ); + + NativeAudioConfiguration( + { + // for iOS / Mac + this.appleAudioCategory, + this.appleAudioCategoryOptions, + this.appleAudioMode, + this.preferSpeakerOutput + // Android options + // ... + }); Map toMap() => { if (appleAudioCategory != null) @@ -104,17 +139,21 @@ class NativeAudioConfiguration { appleAudioCategoryOptions!.map((e) => e.toStringValue()).toList(), if (appleAudioMode != null) 'appleAudioMode': appleAudioMode!.toStringValue(), + if (preferSpeakerOutput != null) + 'preferSpeakerOutput': preferSpeakerOutput, }; NativeAudioConfiguration copyWith({ AppleAudioCategory? appleAudioCategory, Set? appleAudioCategoryOptions, AppleAudioMode? appleAudioMode, + bool? preferSpeakerOutput, }) => NativeAudioConfiguration( appleAudioCategory: appleAudioCategory ?? this.appleAudioCategory, appleAudioCategoryOptions: appleAudioCategoryOptions ?? this.appleAudioCategoryOptions, appleAudioMode: appleAudioMode ?? this.appleAudioMode, + preferSpeakerOutput: preferSpeakerOutput ?? this.preferSpeakerOutput, ); } diff --git a/lib/src/track/audio_management.dart b/lib/src/track/audio_management.dart index a97abd2a3..b44e7e099 100644 --- a/lib/src/track/audio_management.dart +++ b/lib/src/track/audio_management.dart @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:synchronized/synchronized.dart' as sync; import '../hardware/hardware.dart'; @@ -114,6 +113,14 @@ Future _onAudioTrackCountDidChange() async { if (lkPlatformIs(PlatformType.iOS)) { // Only iOS for now... config = await onConfigureNativeAudio.call(_audioTrackState); + + if (Hardware.instance.forceSpeakerOutput) { + config = config.copyWith( + appleAudioCategoryOptions: { + AppleAudioCategoryOption.defaultToSpeaker, + }, + ); + } } if (config != null) { @@ -124,13 +131,6 @@ Future _onAudioTrackCountDidChange() async { logger.warning('failed to configure ${error}'); } } - - if (lkPlatformIs(PlatformType.iOS)) { - if (Hardware.instance.speakerOn != null && - Hardware.instance.canSwitchSpeakerphone) { - await rtc.Helper.setSpeakerphoneOn(Hardware.instance.speakerOn!); - } - } } } @@ -148,38 +148,14 @@ AudioTrackState _computeAudioTrackState() { Future defaultNativeAudioConfigurationFunc( AudioTrackState state) async { - // - if (state == AudioTrackState.remoteOnly && + if (state == AudioTrackState.none) { + return NativeAudioConfiguration.soloAmbient; + } else if (state == AudioTrackState.remoteOnly && Hardware.instance.preferSpeakerOutput) { - return NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playback, - appleAudioCategoryOptions: { - AppleAudioCategoryOption.mixWithOthers, - }, - appleAudioMode: AppleAudioMode.spokenAudio, - ); - } else if ([ - AudioTrackState.localOnly, - AudioTrackState.localAndRemote, - ].contains(state) || - (state == AudioTrackState.remoteOnly && - !Hardware.instance.preferSpeakerOutput)) { - return NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playAndRecord, - appleAudioCategoryOptions: { - AppleAudioCategoryOption.allowBluetooth, - AppleAudioCategoryOption.allowBluetoothA2DP, - AppleAudioCategoryOption.allowAirPlay, - }, - appleAudioMode: Hardware.instance.preferSpeakerOutput - ? AppleAudioMode.videoChat - : AppleAudioMode.voiceChat, - ); + return NativeAudioConfiguration.playback; } - return NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.soloAmbient, - appleAudioCategoryOptions: {}, - appleAudioMode: AppleAudioMode.default_, - ); + return Hardware.instance.preferSpeakerOutput + ? NativeAudioConfiguration.playAndRecordSpeaker + : NativeAudioConfiguration.playAndRecordReceiver; } diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index cbddaabc0..6c4fad18e 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -133,6 +133,8 @@ abstract class Track extends DisposableChangeNotifier logger.fine('$objectId.stop()'); + await mediaStreamTrack.stop(); + _active = false; return true; } diff --git a/lib/src/widgets/video_track_renderer.dart b/lib/src/widgets/video_track_renderer.dart index 88f4b4796..a18ae2d19 100644 --- a/lib/src/widgets/video_track_renderer.dart +++ b/lib/src/widgets/video_track_renderer.dart @@ -70,11 +70,14 @@ class _VideoTrackRendererState extends State { late GlobalKey _internalKey; Future _initializeRenderer() async { - if (widget.renderMode == VideoRenderMode.platformView) { + if (lkPlatformIs(PlatformType.iOS) && + widget.renderMode == VideoRenderMode.platformView) { return Null as Future; } - _renderer ??= rtc.RTCVideoRenderer(); - await _renderer!.initialize(); + if (_renderer == null) { + _renderer = rtc.RTCVideoRenderer(); + await _renderer!.initialize(); + } await _attach(); return _renderer!; } @@ -181,8 +184,7 @@ class _VideoTrackRendererState extends State { Widget _videoRendererView() { if (lkPlatformIs(PlatformType.iOS) && - [VideoRenderMode.auto, VideoRenderMode.platformView] - .contains(widget.renderMode)) { + widget.renderMode == VideoRenderMode.platformView) { return rtc.RTCVideoPlatFormView( mirror: _shouldMirror(), objectFit: widget.fit, @@ -205,8 +207,8 @@ class _VideoTrackRendererState extends State { future: _initializeRenderer(), builder: (context, snapshot) { if ((snapshot.hasData && _renderer != null) || - [VideoRenderMode.auto, VideoRenderMode.platformView] - .contains(widget.renderMode)) { + (lkPlatformIs(PlatformType.iOS) && + widget.renderMode == VideoRenderMode.platformView)) { return Builder( key: _internalKey, builder: (ctx) { diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index c942afbd0..b3f0fc982 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -198,6 +198,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { // options: configuration.categoryOptions) // print("[LiveKit] AVAudioSession Configure success") + // preferSpeakerOutput + if let preferSpeakerOutput = args["preferSpeakerOutput"] as? Bool { + try rtcSession.overrideOutputAudioPort(preferSpeakerOutput ? .speaker : .none) + } result(true) } catch let error { print("[LiveKit] Configure audio error: ", error)