From ced271325d4e134304fb7258ec770f226ae7593b Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Tue, 19 Nov 2024 14:52:08 +0800 Subject: [PATCH] wip. --- ios/Classes/AudioTrack.swift | 1 + ios/Classes/LocalAudioTrack.swift | 1 + ios/Classes/RemoteAudioTrack.swift | 1 + ios/Classes/Track.swift | 1 + lib/src/events.dart | 13 +++++++++ lib/src/track/local/local.dart | 1 + lib/src/track/remote/audio.dart | 1 - lib/src/track/track.dart | 26 +++++++++++++++++ macos/Classes/AudioTrack.swift | 1 + macos/Classes/LocalAudioTrack.swift | 1 + macos/Classes/RemoteAudioTrack.swift | 1 + macos/Classes/Track.swift | 1 + shared_swift/AudioTrack.swift | 28 +++++++++++++++++++ shared_swift/LiveKitPlugin.swift | 34 +++++++++++++++------- shared_swift/LocalAudioTrack.swift | 39 ++++++++++++++++++++++++++ shared_swift/RemoteAudioTrack.swift | 42 ++++++++++++++++++++++++++++ shared_swift/Track.swift | 30 ++++++++++++++++++++ shared_swift/Visualizer.swift | 41 ++++++++++++++++++++++----- 18 files changed, 245 insertions(+), 18 deletions(-) create mode 120000 ios/Classes/AudioTrack.swift create mode 120000 ios/Classes/LocalAudioTrack.swift create mode 120000 ios/Classes/RemoteAudioTrack.swift create mode 120000 ios/Classes/Track.swift create mode 120000 macos/Classes/AudioTrack.swift create mode 120000 macos/Classes/LocalAudioTrack.swift create mode 120000 macos/Classes/RemoteAudioTrack.swift create mode 120000 macos/Classes/Track.swift create mode 100644 shared_swift/AudioTrack.swift create mode 100644 shared_swift/LocalAudioTrack.swift create mode 100644 shared_swift/RemoteAudioTrack.swift create mode 100644 shared_swift/Track.swift diff --git a/ios/Classes/AudioTrack.swift b/ios/Classes/AudioTrack.swift new file mode 120000 index 000000000..cc8fcabbd --- /dev/null +++ b/ios/Classes/AudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/AudioTrack.swift \ No newline at end of file diff --git a/ios/Classes/LocalAudioTrack.swift b/ios/Classes/LocalAudioTrack.swift new file mode 120000 index 000000000..1c6546546 --- /dev/null +++ b/ios/Classes/LocalAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/LocalAudioTrack.swift \ No newline at end of file diff --git a/ios/Classes/RemoteAudioTrack.swift b/ios/Classes/RemoteAudioTrack.swift new file mode 120000 index 000000000..c320b7d1a --- /dev/null +++ b/ios/Classes/RemoteAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/RemoteAudioTrack.swift \ No newline at end of file diff --git a/ios/Classes/Track.swift b/ios/Classes/Track.swift new file mode 120000 index 000000000..8da7267a9 --- /dev/null +++ b/ios/Classes/Track.swift @@ -0,0 +1 @@ +../../shared_swift/Track.swift \ No newline at end of file diff --git a/lib/src/events.dart b/lib/src/events.dart index 04ef406eb..52bef6999 100644 --- a/lib/src/events.dart +++ b/lib/src/events.dart @@ -565,3 +565,16 @@ class VideoReceiverStatsEvent with TrackEvent { String toString() => '${runtimeType}' 'stats: ${stats})'; } + +class AudioVisualizerEvent with TrackEvent { + final Track track; + final List event; + const AudioVisualizerEvent({ + required this.track, + required this.event, + }); + + @override + String toString() => '${runtimeType}' + 'track: ${track})'; +} \ No newline at end of file diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index 3e3177157..ae6e78937 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -16,6 +16,7 @@ import 'dart:async'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; diff --git a/lib/src/track/remote/audio.dart b/lib/src/track/remote/audio.dart index c2b3b3388..184bbcb58 100644 --- a/lib/src/track/remote/audio.dart +++ b/lib/src/track/remote/audio.dart @@ -20,7 +20,6 @@ import '../../internal/events.dart'; import '../../logger.dart'; import '../../stats/audio_source_stats.dart'; import '../../stats/stats.dart'; -import '../../support/native.dart'; import '../../types/other.dart'; import '../audio_management.dart'; import '../local/local.dart'; diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index 43e96305d..c74eb6bc5 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -14,6 +14,7 @@ import 'dart:async'; +import 'package:flutter/services.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; import 'package:uuid/uuid.dart'; @@ -111,6 +112,7 @@ abstract class Track extends DisposableChangeNotifier if(source == TrackSource.microphone) { await Native.startVisualizer(mediaStreamTrack.id!); + listenVisualizerEvent(); } startMonitor(); @@ -130,6 +132,7 @@ abstract class Track extends DisposableChangeNotifier if(source == TrackSource.microphone) { await Native.stopVisualizer(mediaStreamTrack.id!); + stopVisualizerEventListen(); } stopMonitor(); @@ -140,6 +143,29 @@ abstract class Track extends DisposableChangeNotifier return true; } + + EventChannel? _eventChannel ; + StreamSubscription? _streamSubscription; + + @internal + void listenVisualizerEvent() { + _eventChannel = EventChannel('io.livekit.audio.visualizer/eventChannel-${mediaStreamTrack.id}'); + + _eventChannel?.receiveBroadcastStream().listen((event) { + logger.fine('[$objectId] visualizer event(${event})'); + events.emit(AudioVisualizerEvent( + track: this, + event: event, + )); + }); + } + +@internal + void stopVisualizerEventListen() { + _streamSubscription?.cancel(); + _eventChannel = null; + } + Future enable() async { logger.fine('$objectId.enable() enabling ${mediaStreamTrack.objectId}...'); try { diff --git a/macos/Classes/AudioTrack.swift b/macos/Classes/AudioTrack.swift new file mode 120000 index 000000000..cc8fcabbd --- /dev/null +++ b/macos/Classes/AudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/AudioTrack.swift \ No newline at end of file diff --git a/macos/Classes/LocalAudioTrack.swift b/macos/Classes/LocalAudioTrack.swift new file mode 120000 index 000000000..1c6546546 --- /dev/null +++ b/macos/Classes/LocalAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/LocalAudioTrack.swift \ No newline at end of file diff --git a/macos/Classes/RemoteAudioTrack.swift b/macos/Classes/RemoteAudioTrack.swift new file mode 120000 index 000000000..c320b7d1a --- /dev/null +++ b/macos/Classes/RemoteAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/RemoteAudioTrack.swift \ No newline at end of file diff --git a/macos/Classes/Track.swift b/macos/Classes/Track.swift new file mode 120000 index 000000000..8da7267a9 --- /dev/null +++ b/macos/Classes/Track.swift @@ -0,0 +1 @@ +../../shared_swift/Track.swift \ No newline at end of file diff --git a/shared_swift/AudioTrack.swift b/shared_swift/AudioTrack.swift new file mode 100644 index 000000000..d19946523 --- /dev/null +++ b/shared_swift/AudioTrack.swift @@ -0,0 +1,28 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC + +@objc +public protocol AudioTrack where Self: Track { + + @objc(addAudioRenderer:) + func add(audioRenderer: RTCAudioRenderer) + + @objc(removeAudioRenderer:) + func remove(audioRenderer: RTCAudioRenderer) +} diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index 1a60a8f69..9a8d5ea95 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -24,8 +24,10 @@ import UIKit #endif public class LiveKitPlugin: NSObject, FlutterPlugin { + + var processers: Dictionary = [:] - private var processor: AudioProcessor? = nil + var binaryMessenger: FlutterBinaryMessenger? public static func register(with registrar: FlutterPluginRegistrar) { @@ -37,6 +39,7 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let channel = FlutterMethodChannel(name: "livekit_client", binaryMessenger: messenger) let instance = LiveKitPlugin() + instance.binaryMessenger = messenger registrar.addMethodCallDelegate(instance, channel: channel) } @@ -94,24 +97,35 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let trackId = args["trackId"] as? String if let unwrappedTrackId = trackId { - let track = webrtc?.track(forId: unwrappedTrackId, peerConnectionId: nil) + + let localTrack = webrtc?.localTracks![unwrappedTrackId] + if let audioTrack = localTrack as? LocalAudioTrack { + let lkLocalTrack = LKLocalAudioTrack(name: unwrappedTrackId, track: audioTrack); + let processor = AudioProcessor(track: lkLocalTrack, binaryMessenger: self.binaryMessenger!) + processers[lkLocalTrack] = processor + } + + let track = webrtc?.remoteTrack(forId: unwrappedTrackId) if let audioTrack = track as? RTCAudioTrack { - if processor == nil { - processor = AudioProcessor(track: audioTrack) - } + let lkLocalTrack = LKRemoteAudioTrack(name: unwrappedTrackId, track: audioTrack); + let processor = AudioProcessor(track: lkLocalTrack, binaryMessenger: self.binaryMessenger!) + processers[lkLocalTrack] = processor } } - let audioManager = webrtc?.audioManager - - if audioManager != nil && processor != nil { - //audioManager?.addLocalAudioRenderer(processor!) - } result(true) } public func handleStopAudioVisualizer(args: [String: Any?], result: @escaping FlutterResult) { + let trackId = args["trackId"] as? String + if let unwrappedTrackId = trackId { + for key in processers.keys { + if key.mediaTrack.trackId == unwrappedTrackId { + processers.removeValue(forKey: key) + } + } + } result(true) } diff --git a/shared_swift/LocalAudioTrack.swift b/shared_swift/LocalAudioTrack.swift new file mode 100644 index 000000000..2a5568d8a --- /dev/null +++ b/shared_swift/LocalAudioTrack.swift @@ -0,0 +1,39 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC +import flutter_webrtc + +public class LKLocalAudioTrack: Track, AudioTrack { + let audioTrack: LocalAudioTrack + init(name: String, + track: LocalAudioTrack) + { + audioTrack = track + super.init(track: track.track()) + } +} + +public extension LKLocalAudioTrack { + func add(audioRenderer: RTCAudioRenderer) { + audioTrack.add(audioRenderer) + } + + func remove(audioRenderer: RTCAudioRenderer) { + audioTrack.remove(audioRenderer) + } +} diff --git a/shared_swift/RemoteAudioTrack.swift b/shared_swift/RemoteAudioTrack.swift new file mode 100644 index 000000000..c155999b7 --- /dev/null +++ b/shared_swift/RemoteAudioTrack.swift @@ -0,0 +1,42 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC + +import Foundation +import WebRTC +import flutter_webrtc + +public class LKRemoteAudioTrack: Track, AudioTrack { + let audioTrack: RTCAudioTrack + init(name: String, + track: RTCAudioTrack) + { + audioTrack = track + super.init(track: track) + } +} + +public extension LKRemoteAudioTrack { + func add(audioRenderer: RTCAudioRenderer) { + audioTrack.add(audioRenderer) + } + + func remove(audioRenderer: RTCAudioRenderer) { + audioTrack.remove(audioRenderer) + } +} diff --git a/shared_swift/Track.swift b/shared_swift/Track.swift new file mode 100644 index 000000000..a149baf21 --- /dev/null +++ b/shared_swift/Track.swift @@ -0,0 +1,30 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC + +@objc +public class Track: NSObject { + + let mediaTrack: RTCMediaStreamTrack + + init(track: RTCMediaStreamTrack) + { + mediaTrack = track + super.init() + } +} diff --git a/shared_swift/Visualizer.swift b/shared_swift/Visualizer.swift index b57fcf4e3..852071c65 100644 --- a/shared_swift/Visualizer.swift +++ b/shared_swift/Visualizer.swift @@ -17,16 +17,40 @@ import AVFoundation import WebRTC -public class AudioProcessor: NSObject, RTCAudioRenderer { +#if os(macOS) +import Cocoa +import FlutterMacOS +#else +import Flutter +import UIKit +#endif + +public class AudioProcessor: NSObject, RTCAudioRenderer, FlutterStreamHandler { + + private var eventSink: FlutterEventSink? + + private var channel: FlutterEventChannel? + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + eventSink = nil + return nil + } + public let isCentered: Bool public let smoothingFactor: Float public var bands: [Float] private let _processor: AudioVisualizeProcessor - private weak var _track: RTCAudioTrack? + private weak var _track: AudioTrack? - public init(track: RTCAudioTrack?, + public init(track: AudioTrack?, + binaryMessenger: FlutterBinaryMessenger, bandCount: Int = 7, isCentered: Bool = true, smoothingFactor: Float = 0.3) @@ -34,15 +58,18 @@ public class AudioProcessor: NSObject, RTCAudioRenderer { self.isCentered = isCentered self.smoothingFactor = smoothingFactor bands = Array(repeating: 0.0, count: bandCount) - _processor = AudioVisualizeProcessor(bandsCount: bandCount) _track = track super.init() - _track?.add(self) + _track?.add(audioRenderer: self) + let channelName = "io.livekit.audio.visualizer/eventChannel-" + (track?.mediaTrack.trackId ?? "") + channel = FlutterEventChannel(name: channelName, binaryMessenger: binaryMessenger) + channel?.setStreamHandler(self) } deinit { - _track?.remove(self) + _track?.remove(audioRenderer: self) + channel?.setStreamHandler(nil) } public func render(pcmBuffer: AVAudioPCMBuffer) { @@ -57,7 +84,7 @@ public class AudioProcessor: NSObject, RTCAudioRenderer { DispatchQueue.main.async { [weak self] in guard let self else { return } - + eventSink?(newBands) self.bands = zip(self.bands, newBands).map { old, new in self._smoothTransition(from: old, to: new, factor: self.smoothingFactor) }