From 3f39030ff22d5ae590e7bbeee56f9c1e8ed55bd6 Mon Sep 17 00:00:00 2001 From: Hugo Arregui <969314+hugoArregui@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:12:25 -0300 Subject: [PATCH] chore: simplify voice chat (#6033) --- .../shared/comms/adapters/LivekitAdapter.ts | 18 +- .../shared/comms/adapters/OfflineAdapter.ts | 9 +- .../shared/comms/adapters/SimulatorAdapter.ts | 15 +- .../shared/comms/adapters/WebSocketAdapter.ts | 10 +- .../packages/shared/comms/adapters/types.ts | 2 +- .../shared/comms/adapters/voice/Html.ts | 12 - .../comms/adapters/voice/audioDebugger.ts | 333 ----------- .../adapters/voice/liveKitVoiceHandler.ts | 180 +----- .../shared/comms/adapters/voice/loopback.ts | 71 +-- .../comms/adapters/voice/opusVoiceHandler.ts | 80 --- .../packages/shared/comms/handlers.ts | 12 +- .../packages/shared/comms/interface/index.ts | 4 +- .../comms/logic/rfc-4-room-connection.ts | 22 +- .../packages/shared/comms/peers.ts | 4 - .../packages/shared/comms/sagas.ts | 18 +- .../packages/shared/comms/selectors.ts | 10 +- .../downloadManager.ts | 1 - .../shared/session/getPerformanceInfo.ts | 4 +- .../shared/voiceChat/VoiceCommunicator.ts | 545 ------------------ .../packages/shared/voiceChat/handlers.ts | 40 -- .../packages/shared/voiceChat/sagas.ts | 23 +- .../packages/shared/voiceChat/utils.ts | 14 - 22 files changed, 97 insertions(+), 1330 deletions(-) delete mode 100644 browser-interface/packages/shared/comms/adapters/voice/Html.ts delete mode 100644 browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts delete mode 100644 browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts delete mode 100644 browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts delete mode 100644 browser-interface/packages/shared/voiceChat/handlers.ts delete mode 100644 browser-interface/packages/shared/voiceChat/utils.ts diff --git a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts index f20229e8c0..9f897b7b12 100644 --- a/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/LivekitAdapter.ts @@ -17,13 +17,12 @@ import type { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { commsLogger } from '../logger' import type { ActiveVideoStreams, CommsAdapterEvents, MinimumCommunicationsAdapter, SendHints } from './types' import { createLiveKitVoiceHandler } from './voice/liveKitVoiceHandler' -import { GlobalAudioStream } from './voice/loopback' export type LivekitConfig = { url: string token: string logger: ILogger - globalAudioStream: GlobalAudioStream + voiceChatEnabled: boolean } export class LivekitAdapter implements MinimumCommunicationsAdapter { @@ -31,12 +30,13 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { private disposed = false private readonly room: Room - private voiceHandler: VoiceHandler + private voiceHandler: VoiceHandler | undefined = undefined constructor(private config: LivekitConfig) { this.room = new Room() - - this.voiceHandler = createLiveKitVoiceHandler(this.room, this.config.globalAudioStream) + if (config.voiceChatEnabled) { + this.voiceHandler = createLiveKitVoiceHandler(this.room) + } this.room .on(RoomEvent.ParticipantConnected, (_: RemoteParticipant) => { @@ -52,7 +52,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { this.config.logger.log(this.room.name, 'connection state changed', state) }) .on(RoomEvent.Disconnected, (reason: DisconnectReason | undefined) => { - this.config.logger.log('[BOEDO]: on disconnect', reason, this.room.name) if (this.disposed) { return } @@ -79,7 +78,7 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { }) } - async createVoiceHandler(): Promise { + async getVoiceHandler(): Promise { return this.voiceHandler } @@ -111,6 +110,9 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { try { await this.room.localParticipant.publishData(data, reliable ? DataPacket_Kind.RELIABLE : DataPacket_Kind.LOSSY) } catch (err: any) { + if (this.disposed) { + return + } // NOTE: for tracking purposes only, this is not a "code" error, this is a failed connection or a problem with the livekit instance trackEvent('error', { context: 'livekit-adapter', @@ -118,7 +120,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { stack: err.stack, saga_stack: `room session id: ${this.room.sid}, participant id: ${this.room.localParticipant.sid}, state: ${state}` }) - this.config.logger.log('[BOEDO]: error sending data', err, err.mesage) await this.disconnect() } } @@ -132,7 +133,6 @@ export class LivekitAdapter implements MinimumCommunicationsAdapter { return } - this.config.logger.log('[BOEDO]: do_disconnect', this.room.name) this.disposed = true await this.room.disconnect().catch(commsLogger.error) this.events.emit('DISCONNECTION', { kicked }) diff --git a/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts b/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts index 67f6896400..77b380bb63 100644 --- a/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/OfflineAdapter.ts @@ -1,17 +1,18 @@ import mitt from 'mitt' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { CommsAdapterEvents, MinimumCommunicationsAdapter, SendHints } from './types' -import { createOpusVoiceHandler } from './voice/opusVoiceHandler' export class OfflineAdapter implements MinimumCommunicationsAdapter { events = mitt() constructor() {} - async createVoiceHandler(): Promise { - return createOpusVoiceHandler() + async getVoiceHandler(): Promise { + return undefined } async disconnect(_error?: Error | undefined): Promise {} send(_data: Uint8Array, _hints: SendHints): void {} async connect(): Promise {} - async getParticipants() { return [] } + async getParticipants() { + return [] + } } diff --git a/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts b/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts index e708d43f29..637457d99d 100644 --- a/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/SimulatorAdapter.ts @@ -12,15 +12,13 @@ import { Position, ProfileRequest, ProfileResponse, - Scene, - Voice + Scene } from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' import { lastPlayerPosition } from 'shared/world/positionThings' import { CommsEvents, RoomConnection } from '../interface' import { Rfc4RoomConnection } from '../logic/rfc-4-room-connection' import { CommsAdapterEvents, SendHints } from './types' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' -import { createOpusVoiceHandler } from './voice/opusVoiceHandler' export class SimulationRoom implements RoomConnection { events = mitt() @@ -51,8 +49,8 @@ export class SimulationRoom implements RoomConnection { send(_data: Uint8Array, _hints: SendHints): void {}, async connect(): Promise {}, async disconnect(_error?: Error): Promise {}, - async createVoiceHandler() { - throw new Error('not implemented') + async getVoiceHandler(): Promise { + return undefined }, async getParticipants() { return Array.from(peers.keys()) @@ -60,8 +58,8 @@ export class SimulationRoom implements RoomConnection { }) } - async createVoiceHandler(): Promise { - return createOpusVoiceHandler() + async getVoiceHandler(): Promise { + return this.roomConnection.getVoiceHandler() } async spawnPeer(): Promise { @@ -135,9 +133,6 @@ export class SimulationRoom implements RoomConnection { async sendChatMessage(message: Chat): Promise { await this.roomConnection.sendChatMessage(message) } - async sendVoiceMessage(message: Voice): Promise { - await this.roomConnection.sendVoiceMessage(message) - } update() { let i = 0 diff --git a/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts b/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts index 5622d650e6..836684f9ff 100644 --- a/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts +++ b/browser-interface/packages/shared/comms/adapters/WebSocketAdapter.ts @@ -7,7 +7,6 @@ import { ExplorerIdentity } from 'shared/session/types' import { Authenticator } from '@dcl/crypto' import mitt from 'mitt' import { CommsAdapterEvents, MinimumCommunicationsAdapter, SendHints } from './types' -import { createOpusVoiceHandler } from './voice/opusVoiceHandler' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' import { notifyStatusThroughChat } from 'shared/chat' import { wsAsAsyncChannel } from '../logic/ws-async-channel' @@ -36,7 +35,10 @@ export class WebSocketAdapter implements MinimumCommunicationsAdapter { private ws: WebSocket | null = null - constructor(public url: string, private identity: ExplorerIdentity) {} + constructor( + public url: string, + private identity: ExplorerIdentity + ) {} async connect(): Promise { if (this.ws) throw new Error('Cannot call connect twice per IBrokerTransport') @@ -131,8 +133,8 @@ export class WebSocketAdapter implements MinimumCommunicationsAdapter { } } - async createVoiceHandler(): Promise { - return createOpusVoiceHandler() + async getVoiceHandler(): Promise { + return undefined } handleWelcomeMessage(welcomeMessage: rfc5.WsWelcome, socket: WebSocket) { diff --git a/browser-interface/packages/shared/comms/adapters/types.ts b/browser-interface/packages/shared/comms/adapters/types.ts index cc3d7db853..9690478e94 100644 --- a/browser-interface/packages/shared/comms/adapters/types.ts +++ b/browser-interface/packages/shared/comms/adapters/types.ts @@ -28,7 +28,7 @@ export interface MinimumCommunicationsAdapter { */ events: Emitter - createVoiceHandler(): Promise + getVoiceHandler(): Promise getParticipants(): Promise } diff --git a/browser-interface/packages/shared/comms/adapters/voice/Html.ts b/browser-interface/packages/shared/comms/adapters/voice/Html.ts deleted file mode 100644 index b4cfffe201..0000000000 --- a/browser-interface/packages/shared/comms/adapters/voice/Html.ts +++ /dev/null @@ -1,12 +0,0 @@ -export default class Html { - static loopbackAudioElement() { - const audio = document.getElementById('voice-chat-audio') - if (!audio) { - console.error('There is no #voice-chat-audio element to use with VoiceChat. Returning a `new Audio()`') - const audio = new Audio() - document.body.append(audio) - return audio - } - return audio as HTMLAudioElement | undefined - } -} diff --git a/browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts b/browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts deleted file mode 100644 index 3f2f30526d..0000000000 --- a/browser-interface/packages/shared/comms/adapters/voice/audioDebugger.ts +++ /dev/null @@ -1,333 +0,0 @@ -import mitt from 'mitt' -// eslint-ignore @typescript-eslint/no-unused-vars - -type BaseNode = { - cyId: number - nodeName: string -} - -type Graph = { - edges: Record - nodes: Record - nodeNum: number -} - -const events = mitt<{ - addEdge: { from: BaseNode; to: BaseNode } - addNode: { node: BaseNode } - removeNode: { node: BaseNode } - graphChanged: Graph -}>() - -// state and redux ------------------------------------------------------------- - -let currentGraphState: Graph = { edges: {}, nodes: {}, nodeNum: 0 } - -function addNodeToGraphReducer(graph: Graph, node: BaseNode): Graph { - if (!node.cyId) { - graph.nodeNum++ - node.cyId = graph.nodeNum - } - - let nodeName = getConstructorName(node) - - if (node instanceof MediaStreamAudioSourceNode) { - nodeName = nodeName + '\n' + node.mediaStream.id - } - - return { - ...graph, - nodes: { ...graph.nodes, [node.cyId]: Object.assign(node, { nodeName }) } - } -} - -events.on('addEdge', (evt) => { - currentGraphState = addNodeToGraphReducer(addNodeToGraphReducer(currentGraphState, evt.to), evt.from) - - const edgeKey = evt.from.cyId + '_' + evt.to.cyId - - currentGraphState = { - ...currentGraphState, - edges: { - ...currentGraphState.edges, - [edgeKey]: evt - } - } - - events.emit('graphChanged', currentGraphState) -}) - -events.on('removeNode', (evt) => { - const newEdges: Graph['edges'] = {} - - for (const i in currentGraphState.edges) { - const edge = currentGraphState.edges[i] - if (edge.from.cyId !== evt.node.cyId && edge.to.cyId !== evt.node.cyId) { - newEdges[i] = edge - } - } - - currentGraphState = { - ...currentGraphState, - edges: newEdges - } - - delete currentGraphState.nodes[evt.node.cyId] - - events.emit('graphChanged', currentGraphState) -}) - -events.on('addNode', (evt) => { - currentGraphState = addNodeToGraphReducer(currentGraphState, evt.node) - events.emit('graphChanged', currentGraphState) -}) - -globalThis.getAudioGraph = getAudioGraph -globalThis.getAudioGraphViz = getAudioGraphViz - -export function getAudioGraph() { - return currentGraphState -} - -export function getAudioGraphViz() { - const nodes: string[] = [] - const edges: string[] = [] - - function nodeName(node: BaseNode): string { - return 'node' + node.cyId - } - - for (const n in currentGraphState.nodes) { - const node = currentGraphState.nodes[n] - - if (node instanceof AudioContext) { - nodes.push(` - subgraph cluster_${n} { - style=filled; - color=lightgrey; - node [style=filled,color=white]; - ${nodeName(node.destination as any)}; - ${nodeName(node.listener as any)} [label="listener ${node.listener.positionX.value.toFixed( - 2 - )},${node.listener.positionY.value.toFixed(2)},${node.listener.positionZ.value.toFixed(2)}"]; - label = ${JSON.stringify('AudioContext\nglobalThis.__node_' + n)}; - } - `) - } else if (node instanceof PannerNode) { - nodes.push( - `${nodeName(node)} [label=${JSON.stringify( - `Panner ${node.positionX.value.toFixed(2)},${ - (node.positionY.value.toFixed(2), node.positionZ.value.toFixed(2)) - }\nglobalThis.__node_${n}` - )},shape="diamond"];` - ) - } else if (node instanceof GainNode) { - nodes.push( - `${nodeName(node)} [label=${JSON.stringify(`Gain ${node.gain.value}\nglobalThis.__node_${n}`)},shape="oval"];` - ) - } else if (node instanceof MediaStreamAudioDestinationNode) { - nodes.push(`${nodeName(node)} [label=${JSON.stringify(node.nodeName + '\nglobalThis.__node_' + n)}];`) - if (node.context['cyId']) { - edges.push(`${nodeName(node)} -> ${nodeName(node.context.destination as any)}`) - } - } else { - nodes.push(`${nodeName(node)} [label=${JSON.stringify(node.nodeName + '\nglobalThis.__node_' + n)}];`) - } - - if (node instanceof MediaStreamAudioSourceNode && node.mediaStream instanceof MediaStream) { - if ('_videoPreMute' in node.mediaStream) { - nodes.push(`${nodeName(node)}_media [label=${JSON.stringify('RemoteStream')}];`) - } else if ('pc' in node.mediaStream) { - nodes.push(`${nodeName(node)}_media [label=${JSON.stringify('LocalStream')}];`) - } - nodes.push(`${nodeName(node)}_media -> ${nodeName(node)};`) - } - - ;(globalThis as any)['__node_' + n] = node - } - - for (const e in currentGraphState.edges) { - const edge = currentGraphState.edges[e] - nodes.push(`${nodeName(edge.from)} -> ${nodeName(edge.to)};`) - } - - return `digraph G { - graph [fontname = "arial", fontsize="10", color="grey", fontcolor="grey"]; - node [fontname = "arial",fontsize="10", shape="box", style="rounded"]; - edge [fontname = "arial",color="blue", fontcolor="blue",fontsize="10"]; -# nodes -${nodes.join('\n')} -# edges -${edges.join('\n')} -}` -} - -function decoratePrototype(originalFunction: any, decorator: any) { - return function (this: any, ...args: any[]) { - const res = originalFunction.apply(this, args) - decorator.call(this, res, args) - return res - } -} - -function getConstructorName(obj: any) { - if (obj.constructor.name) { - return obj.constructor.name - } - const matches = obj.constructor.toString().match(/function (\w*)/) - if (matches && matches.length) { - return matches[1] - } -} - -if (document.location.search.includes('AUDIO_DEBUG')) { - // hack ------------------------------------------------------------------------ - - AudioNode.prototype.connect = decoratePrototype( - AudioNode.prototype.connect, - function (this: AudioNode & BaseNode, result: any, args: any[]) { - events.emit('addEdge', { - from: this, - to: args[0] - }) - } - ) - - AudioNode.prototype.disconnect = decoratePrototype( - AudioNode.prototype.disconnect, - function (this: any, _result: any, _args: any[]) { - events.emit('removeNode', { - node: this - }) - } - ) - - AudioBufferSourceNode.prototype.start = decoratePrototype( - AudioBufferSourceNode.prototype.start, - function (this: any, _result: any, _args: any[]) { - console.log('WebAudioDebugger: AudioBufferSourceNode start') - } - ) - - function addNode(node: any) { - if (node instanceof AudioContext) { - addNode(node.destination) - addNode(node.listener) - } - events.emit('addNode', { - node: node as any - }) - } - - PannerNode.prototype.setPosition = decoratePrototype( - PannerNode.prototype.setPosition, - function (this: PannerNode, _result: any, _args: any[]) { - events.emit('graphChanged', { ...currentGraphState }) - } - ) - - AudioListener.prototype.setPosition = decoratePrototype( - AudioListener.prototype.setPosition, - function (this: PannerNode, _result: any, _args: any[]) { - events.emit('graphChanged', { ...currentGraphState }) - } - ) - - AudioContext.prototype.createBufferSource = decoratePrototype( - AudioContext.prototype.createBufferSource, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create BufferSourceNode', { this: this, result, args }) - this.addEventListener('ended', function () { - console.log('WebAudioDebugger: AudioBufferSourceNode ended') - }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createGain = decoratePrototype( - AudioContext.prototype.createGain, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create GainNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createPanner = decoratePrototype( - AudioContext.prototype.createPanner, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create PannerNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createDynamicsCompressor = decoratePrototype( - AudioContext.prototype.createDynamicsCompressor, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create DynamicsCompressorNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createDelay = decoratePrototype( - AudioContext.prototype.createDelay, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create DelayNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createConvolver = decoratePrototype( - AudioContext.prototype.createConvolver, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create ConvolverNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createAnalyser = decoratePrototype( - AudioContext.prototype.createAnalyser, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create AnalyserNode', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createBiquadFilter = decoratePrototype( - AudioContext.prototype.createBiquadFilter, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create BiquadFilter', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.createOscillator = decoratePrototype( - AudioContext.prototype.createOscillator, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Create Oscillator', { this: this, result, args }) - addNode(this) - addNode(result) - } - ) - - AudioContext.prototype.decodeAudioData = decoratePrototype( - AudioContext.prototype.decodeAudioData, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger:decodeAudioData', args[0].byteLength, { this: this, result, args }) - } - ) - - AudioContext.prototype.constructor = decoratePrototype( - AudioContext.prototype.constructor, - function (this: AudioContext, result: any, args: any[]) { - console.log('WebAudioDebugger: Constructor', { this: this, result, args }) - } - ) -} diff --git a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts index 85df9c3633..aa281103d5 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/liveKitVoiceHandler.ts @@ -2,7 +2,6 @@ import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' import { createLogger } from 'lib/logger' import { ParticipantEvent, - RemoteAudioTrack, RemoteParticipant, RemoteTrack, RemoteTrackPublication, @@ -10,108 +9,50 @@ import { RoomEvent, Track } from 'livekit-client' -import { getPeer } from 'shared/comms/peers' -import { getCurrentUserProfile } from 'shared/profiles/selectors' -import { store } from 'shared/store/isolatedStore' -import { shouldPlayVoice } from 'shared/voiceChat/selectors' -import { getSpatialParamsFor } from 'shared/voiceChat/utils' import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' -import { GlobalAudioStream } from './loopback' +import { loopbackAudioElement } from './loopback' -type ParticipantInfo = { - tracks: Map -} - -type ParticipantTrack = { - streamNode: MediaStreamAudioSourceNode - panNode: PannerNode -} - -export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalAudioStream): VoiceHandler { +export function createLiveKitVoiceHandler(room: Room): VoiceHandler { const logger = createLogger('🎙 LiveKitVoiceCommunicator: ') + // const globalAudioStream = await getGlobalAudioStream() + let recordingListener: ((state: boolean) => void) | undefined let errorListener: ((message: string) => void) | undefined + let onUserTalkingCallback: ((userId: string, talking: boolean) => void) | undefined = undefined - let globalVolume: number = 1.0 + // let globalVolume: number = 1.0 let validInput = false - let onUserTalkingCallback: (userId: string, talking: boolean) => void = () => {} - - const participantsInfo = new Map() function handleTrackSubscribed( track: RemoteTrack, _publication: RemoteTrackPublication, participant: RemoteParticipant ) { - if (track.kind !== Track.Kind.Audio || !track.sid) { + if (track.kind !== Track.Kind.Audio) { return } participant.on(ParticipantEvent.IsSpeakingChanged, (talking: boolean) => { - onUserTalkingCallback(participant.identity, talking) + if (onUserTalkingCallback) { + onUserTalkingCallback(participant.identity, talking) + } }) - let info = participantsInfo.get(participant.identity) - if (!info) { - info = { tracks: new Map() } - participantsInfo.set(participant.identity, info) - } - - const audioContext = globalAudioStream.getAudioContext() - const streamNode = audioContext.createMediaStreamSource(track.mediaStream!) - const panNode = audioContext.createPanner() - - streamNode.connect(panNode) - panNode.connect(globalAudioStream.getGainNode()) - - panNode.panningModel = 'equalpower' - panNode.distanceModel = 'inverse' - panNode.refDistance = 5 - panNode.maxDistance = 10000 - panNode.coneOuterAngle = 360 - panNode.coneInnerAngle = 180 - panNode.coneOuterGain = 0.9 - panNode.rolloffFactor = 1.0 - - info.tracks.set(track.sid, { panNode, streamNode }) + const element = track.attach() + loopbackAudioElement().appendChild(element) } function handleTrackUnsubscribed( - remoteTrack: RemoteTrack, + track: RemoteTrack, _publication: RemoteTrackPublication, - participant: RemoteParticipant + _participant: RemoteParticipant ) { - if (remoteTrack.kind !== Track.Kind.Audio || !remoteTrack.sid) { - return - } - - const info = participantsInfo.get(participant.identity) - if (!info) { + if (track.kind !== Track.Kind.Audio) { return } - const track = info.tracks.get(remoteTrack.sid) - if (track) { - track.panNode.disconnect() - track.streamNode.disconnect() - } - - info.tracks.delete(remoteTrack.sid) - } - - function handleParticipantDisconnected(p: RemoteParticipant) { - const info = participantsInfo.get(p.identity) - if (!info) { - return - } - - for (const track of info.tracks.values()) { - track.panNode.disconnect() - track.streamNode.disconnect() - } - - participantsInfo.delete(p.identity) + track.detach() } room @@ -122,9 +63,8 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA errorListener('Media Device Error') } }) - .on(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected) - logger.log('initialized') + logger.log(`initialized ${room.name}`) return { async setRecording(recording) { @@ -147,7 +87,7 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA room.startAudio().catch(logger.error) } - globalAudioStream.play() + // globalAudioStream.play() } catch (err: any) { logger.error(err) } @@ -158,81 +98,15 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA onError(cb) { errorListener = cb }, - reportPosition(position: rfc4.Position) { - const spatialParams = getSpatialParamsFor(position) - const audioContext = globalAudioStream.getAudioContext() - const listener = audioContext.listener - - if (listener.positionX) { - listener.positionX.setValueAtTime(spatialParams.position[0], audioContext.currentTime) - listener.positionY.setValueAtTime(spatialParams.position[1], audioContext.currentTime) - listener.positionZ.setValueAtTime(spatialParams.position[2], audioContext.currentTime) - } else { - listener.setPosition(spatialParams.position[0], spatialParams.position[1], spatialParams.position[2]) - } - - if (listener.forwardX) { - listener.forwardX.setValueAtTime(spatialParams.orientation[0], audioContext.currentTime) - listener.forwardY.setValueAtTime(spatialParams.orientation[1], audioContext.currentTime) - listener.forwardZ.setValueAtTime(spatialParams.orientation[2], audioContext.currentTime) - listener.upX.setValueAtTime(0, audioContext.currentTime) - listener.upY.setValueAtTime(1, audioContext.currentTime) - listener.upZ.setValueAtTime(0, audioContext.currentTime) - } else { - listener.setOrientation( - spatialParams.orientation[0], - spatialParams.orientation[1], - spatialParams.orientation[2], - 0, - 1, - 0 - ) - } - - for (const participant of room.participants.values()) { - const address = participant.identity - const peer = getPeer(address) - const participantInfo = participantsInfo.get(address) - - const state = store.getState() - const profile = getCurrentUserProfile(state) - if (profile) { - const muted = !shouldPlayVoice(state, profile, address) - const audioPublication = participant.getTrack(Track.Source.Microphone) - if (audioPublication && audioPublication.track) { - const audioTrack = audioPublication.track as RemoteAudioTrack - audioTrack.setMuted(muted) - } - } - - if (participantInfo) { - const spatialParams = peer?.position || position - for (const { panNode } of participantInfo.tracks.values()) { - if (panNode.positionX) { - panNode.positionX.setValueAtTime(spatialParams.positionX, audioContext.currentTime) - panNode.positionY.setValueAtTime(spatialParams.positionY, audioContext.currentTime) - panNode.positionZ.setValueAtTime(spatialParams.positionZ, audioContext.currentTime) - } else { - panNode.setPosition(spatialParams.positionX, spatialParams.positionY, spatialParams.positionZ) - } - - if (panNode.orientationX) { - panNode.orientationX.setValueAtTime(0, audioContext.currentTime) - panNode.orientationY.setValueAtTime(0, audioContext.currentTime) - panNode.orientationZ.setValueAtTime(1, audioContext.currentTime) - } else { - panNode.setOrientation(0, 0, 1) - } - } - } - } - }, + reportPosition(_position: rfc4.Position) {}, setVolume: function (volume) { - globalVolume = volume - globalAudioStream.setGainVolume(volume) + // TODO + // globalVolume = volume + // globalAudioStream.setGainVolume(volume) }, setMute: (mute) => { - globalAudioStream.setGainVolume(mute ? 0 : globalVolume) + // TODO + // globalAudioStream.setGainVolume(mute ? 0 : globalVolume) }, setInputStream: async (localStream) => { try { @@ -246,6 +120,10 @@ export function createLiveKitVoiceHandler(room: Room, globalAudioStream: GlobalA hasInput: () => { return validInput }, - async destroy() {} + async destroy() { + onUserTalkingCallback = undefined + recordingListener = undefined + errorListener = undefined + } } } diff --git a/browser-interface/packages/shared/comms/adapters/voice/loopback.ts b/browser-interface/packages/shared/comms/adapters/voice/loopback.ts index 6c4253ef1a..1371991770 100644 --- a/browser-interface/packages/shared/comms/adapters/voice/loopback.ts +++ b/browser-interface/packages/shared/comms/adapters/voice/loopback.ts @@ -1,63 +1,10 @@ -import Html from './Html' - -export type GlobalAudioStream = { - setGainVolume(volume: number): void - getDestinationStream(): MediaStream - getAudioContext(): AudioContext - getGainNode(): GainNode - play(): void -} - -let globalAudioStream: undefined | GlobalAudioStream = undefined - -export async function getGlobalAudioStream() { - if (!globalAudioStream) { - globalAudioStream = await createAudioStream() - } - - return globalAudioStream -} - -export async function createAudioStream(): Promise { - const parentElement = Html.loopbackAudioElement() - if (!parentElement) { - throw new Error('Cannot create global audio stream: no parent element') - } - - const audioContext = new AudioContext() - const destination = audioContext.createMediaStreamDestination() - - const gainNode = new GainNode(audioContext) - gainNode.connect(destination) - gainNode.gain.value = 1 - - parentElement.srcObject = destination.stream - - function getGainNode() { - return gainNode - } - - function setGainVolume(volume: number) { - gainNode.gain.value = volume - } - - function getDestinationStream() { - return destination.stream - } - - function getAudioContext() { - return audioContext - } - - function play() { - parentElement!.play().catch(console.error) - } - - return { - getGainNode, - setGainVolume, - getDestinationStream, - getAudioContext, - play - } +export function loopbackAudioElement() { + const audio = document.getElementById('voice-chat-audio') + if (!audio) { + console.error('There is no #voice-chat-audio element to use with VoiceChat. Returning a `new Audio()`') + const audio = new Audio() + document.body.append(audio) + return audio + } + return audio as HTMLAudioElement } diff --git a/browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts b/browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts deleted file mode 100644 index 40e38615f1..0000000000 --- a/browser-interface/packages/shared/comms/adapters/voice/opusVoiceHandler.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { createLogger } from 'lib/logger' -import { VoiceHandler } from 'shared/voiceChat/VoiceHandler' -import { VoiceCommunicator } from 'shared/voiceChat/VoiceCommunicator' -import { commConfigurations } from 'config' -import Html from './Html' -import { getCommsRoom } from 'shared/comms/selectors' -import { getSpatialParamsFor } from 'shared/voiceChat/utils' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' -import { store } from 'shared/store/isolatedStore' -import withCache from 'lib/javascript/withCache' - -import './audioDebugger' - -const getVoiceCommunicator = withCache(() => { - const logger = createLogger('OpusVoiceCommunicator: ') - return new VoiceCommunicator( - { - send(frame: rfc4.Voice) { - const transport = getCommsRoom(store.getState()) - transport?.sendVoiceMessage(frame).catch(logger.error) - } - }, - { - initialListenerParams: undefined, - panningModel: commConfigurations.voiceChatUseHRTF ? 'HRTF' : 'equalpower', - loopbackAudioElement: Html.loopbackAudioElement() - } - ) -}) - -export const createOpusVoiceHandler = (): VoiceHandler => { - const voiceCommunicator = getVoiceCommunicator() - - return { - setRecording(recording) { - if (recording) { - voiceCommunicator.start() - } else { - voiceCommunicator.pause() - } - return Promise.resolve() - }, - onUserTalking(cb) { - voiceCommunicator.addStreamPlayingListener((streamId: string, playing: boolean) => { - cb(streamId, playing) - }) - }, - onRecording(cb) { - voiceCommunicator.addStreamRecordingListener((recording: boolean) => { - cb(recording) - }) - }, - onError(cb) { - voiceCommunicator.addStreamRecordingErrorListener((message) => { - cb(message) - }) - }, - reportPosition(position: rfc4.Position) { - voiceCommunicator.setListenerSpatialParams(getSpatialParamsFor(position)) - }, - setVolume: function (volume) { - voiceCommunicator.setVolume(volume) - }, - setMute: (mute) => { - voiceCommunicator.setMute(mute) - }, - setInputStream: (stream) => { - return voiceCommunicator.setInputStream(stream) - }, - hasInput: () => { - return voiceCommunicator.hasInput() - }, - playEncodedAudio: (src, position, encoded) => { - return voiceCommunicator.playEncodedAudio(src, getSpatialParamsFor(position), encoded) - }, - async destroy() { - // noop - } - } -} diff --git a/browser-interface/packages/shared/comms/handlers.ts b/browser-interface/packages/shared/comms/handlers.ts index cd825d429d..cd0adb207c 100644 --- a/browser-interface/packages/shared/comms/handlers.ts +++ b/browser-interface/packages/shared/comms/handlers.ts @@ -11,7 +11,6 @@ import { incrementCommsMessageReceived, incrementCommsMessageReceivedByName } fr import { getCurrentUserId } from 'shared/session/selectors' import { store } from 'shared/store/isolatedStore' import { ChatMessage as InternalChatMessage, ChatMessageType } from 'shared/types' -import { processVoiceFragment } from 'shared/voiceChat/handlers' import { isBlockedOrBanned } from 'shared/voiceChat/selectors' import { messageReceived } from '../chat/actions' import { handleRoomDisconnection } from './actions' @@ -47,7 +46,6 @@ export async function bindHandlersToCommsContext(room: RoomConnection) { room.events.on('sceneMessageBus', processParcelSceneCommsMessage) room.events.on('profileRequest', processProfileRequest) room.events.on('profileResponse', processProfileResponse) - room.events.on('voiceMessage', processVoiceFragment) room.events.on('*', (type, _) => { incrementCommsMessageReceived() @@ -79,10 +77,9 @@ export async function requestProfileFromPeers( } async function handleDisconnectionEvent(data: AdapterDisconnectedEvent, room: RoomConnection) { - try { - await onRoomLeft(room) - } catch(err) { + await onRoomLeft(room) + } catch (err) { console.error(err) // TODO: handle this } @@ -140,9 +137,8 @@ function processChatMessage(message: Package) { senderPeer.lastUpdate = Date.now() if (senderPeer.ethereumAddress) { - if (message.data.message.startsWith('␆') /* pong */ || - message.data.message.startsWith('␑') /* ping */) { - // TODO: remove this + if (message.data.message.startsWith('␆') /* pong */ || message.data.message.startsWith('␑') /* ping */) { + // TODO: remove this } else if (message.data.message.startsWith('␐')) { const [id, timestamp] = message.data.message.split(' ') diff --git a/browser-interface/packages/shared/comms/interface/index.ts b/browser-interface/packages/shared/comms/interface/index.ts index ffba64785d..5b5b79e474 100644 --- a/browser-interface/packages/shared/comms/interface/index.ts +++ b/browser-interface/packages/shared/comms/interface/index.ts @@ -10,7 +10,6 @@ export type CommsEvents = CommsAdapterEvents & { chatMessage: Package profileMessage: Package position: Package - voiceMessage: Package profileResponse: Package profileRequest: Package } @@ -30,9 +29,8 @@ export interface RoomConnection { sendPositionMessage(position: Omit): Promise sendParcelSceneMessage(message: proto.Scene): Promise sendChatMessage(message: proto.Chat): Promise - sendVoiceMessage(message: proto.Voice): Promise - createVoiceHandler(): Promise + getVoiceHandler(): Promise getParticipants(): Promise } diff --git a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts index 64360bee66..5f34197940 100644 --- a/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts +++ b/browser-interface/packages/shared/comms/logic/rfc-4-room-connection.ts @@ -24,17 +24,14 @@ export class Rfc4RoomConnection implements RoomConnection { } async connect(): Promise { - console.log('[RoomConnection Comms]: connect', this.id) await this.transport.connect() } - createVoiceHandler(): Promise { - // console.log('[RoomConnection Comms]: createVoiceHandler', this.id) - return this.transport.createVoiceHandler() + getVoiceHandler(): Promise { + return this.transport.getVoiceHandler() } sendPositionMessage(p: Omit): Promise { - // console.log('[RoomConnection Comms]: sendPositionMessage', this.id) return this.sendMessage(false, { message: { $case: 'position', @@ -46,32 +43,22 @@ export class Rfc4RoomConnection implements RoomConnection { }) } sendParcelSceneMessage(scene: proto.Scene): Promise { - // console.log('[RoomConnection Comms]: sendParcelSceneMessage', this.id) return this.sendMessage(false, { message: { $case: 'scene', scene } }) } sendProfileMessage(profileVersion: proto.AnnounceProfileVersion): Promise { - // console.log('[RoomConnection Comms]: sendProfileMessage', this.id) return this.sendMessage(false, { message: { $case: 'profileVersion', profileVersion } }) } sendProfileRequest(profileRequest: proto.ProfileRequest): Promise { - // console.log('[RoomConnection Comms]: sendProfileRequest', this.id) return this.sendMessage(false, { message: { $case: 'profileRequest', profileRequest } }) } sendProfileResponse(profileResponse: proto.ProfileResponse): Promise { - // console.log('[RoomConnection Comms]: sendProfileResponse', this.id) return this.sendMessage(false, { message: { $case: 'profileResponse', profileResponse } }) } sendChatMessage(chat: proto.Chat): Promise { - // console.log('[RoomConnection Comms]: sendChatMessage', this.id) return this.sendMessage(true, { message: { $case: 'chat', chat } }) } - sendVoiceMessage(voice: proto.Voice): Promise { - // console.log('[RoomConnection Comms]: sendVoiceMessage', this.id) - return this.sendMessage(false, { message: { $case: 'voice', voice } }) - } async disconnect() { - console.log('[RoomConnection Comms]: disconnect', this.id) await this.transport.disconnect() } @@ -86,7 +73,6 @@ export class Rfc4RoomConnection implements RoomConnection { return } - // console.log('[RoomConnection Comms]: handleMessage', message.$case, this.id) switch (message.$case) { case 'position': { this.events.emit('position', { address, data: message.position }) @@ -100,10 +86,6 @@ export class Rfc4RoomConnection implements RoomConnection { this.events.emit('chatMessage', { address, data: message.chat }) break } - case 'voice': { - this.events.emit('voiceMessage', { address, data: message.voice }) - break - } case 'profileRequest': { this.events.emit('profileRequest', { address, diff --git a/browser-interface/packages/shared/comms/peers.ts b/browser-interface/packages/shared/comms/peers.ts index 2d973d5aed..2c8993418c 100644 --- a/browser-interface/packages/shared/comms/peers.ts +++ b/browser-interface/packages/shared/comms/peers.ts @@ -80,7 +80,6 @@ export function setupPeer(address: string): PeerInformation { const ethereumAddress = address.toLowerCase() if (!peerInformationMap.has(ethereumAddress)) { - console.log('[BOEDO] Adding Peer information', ethereumAddress) const peer: PeerInformation = { ethereumAddress, lastPositionIndex: 0, @@ -213,7 +212,6 @@ function getActiveRooms(): RoomConnection[] { export async function onRoomLeft(oldRoom: RoomConnection) { const rooms = getActiveRooms() const newPeerInformationMap = new Map() - console.log('[onRoomLeft] rooms', rooms) for (const room of rooms) { if (room.id === oldRoom.id) { continue @@ -222,7 +220,6 @@ export async function onRoomLeft(oldRoom: RoomConnection) { for (const participant of await room.getParticipants()) { const info = peerInformationMap.get(participant) if (info) { - console.log('[onRoomLeft] addparticipant', participant) newPeerInformationMap.set(participant, info) } } @@ -230,7 +227,6 @@ export async function onRoomLeft(oldRoom: RoomConnection) { for (const participant of peerInformationMap.keys()) { if (!newPeerInformationMap.has(participant)) { - console.log('[onRoomLeft] removeUser', participant) avatarMessageObservable.notifyObservers({ type: AvatarMessageType.USER_REMOVED, userId: participant diff --git a/browser-interface/packages/shared/comms/sagas.ts b/browser-interface/packages/shared/comms/sagas.ts index 12af9a5f64..937e84c877 100644 --- a/browser-interface/packages/shared/comms/sagas.ts +++ b/browser-interface/packages/shared/comms/sagas.ts @@ -52,7 +52,6 @@ import { processAvatarVisibility } from './peers' import { getCommsRoom, getSceneRoom, getSceneRooms, reconnectionState } from './selectors' import { RootState } from 'shared/store/rootTypes' import { now } from 'lib/javascript/now' -import { getGlobalAudioStream } from './adapters/voice/loopback' import { store } from 'shared/store/isolatedStore' import { buildSnapshotContent } from 'shared/profiles/sagas/handleDeployProfile' import { isBase64 } from 'lib/encoding/base64ToBlob' @@ -171,6 +170,7 @@ function* handleConnectToComms(action: ConnectToCommsAction) { const adapter: RoomConnection = yield call( connectAdapter, action.payload.event.connStr, + false, identity, action.payload.event.islandId ) @@ -196,11 +196,11 @@ function* handleConnectToComms(action: ConnectToCommsAction) { async function connectAdapter( connStr: string, + voiceChatEnabled: boolean, identity: ExplorerIdentity, - id: string = 'island', + id: string, dispatchAction = true ): Promise { - console.log('[connectAdapter] ', { connStr, identity, id }) const ix = connStr.indexOf(':') const protocol = connStr.substring(0, ix) const url = connStr.substring(ix + 1) @@ -237,7 +237,7 @@ async function connectAdapter( } if (typeof response.fixedAdapter === 'string' && !response.fixedAdapter.startsWith('signed-login:')) { - return connectAdapter(response.fixedAdapter, identity, id) + return connectAdapter(response.fixedAdapter, voiceChatEnabled, identity, id) } if (typeof response.message === 'string') { @@ -274,7 +274,7 @@ async function connectAdapter( logger: commsLogger, url: theUrl.origin + theUrl.pathname, token, - globalAudioStream: await getGlobalAudioStream() + voiceChatEnabled }) if (dispatchAction) { @@ -524,8 +524,6 @@ function* sceneRoomComms() { return } - console.log('[BOEDO] isWorldLoaderActive(adapter!)', isWorldLoaderActive(adapter!)) - while (true) { const reason: { timeout?: unknown; newParcel?: { payload: { position: Vector2 } } } = yield race({ newParcel: take(SET_PARCEL_POSITION), @@ -556,13 +554,11 @@ function* checkDisconnectScene(currentSceneId: string, commsSceneToRemove: Map = yield select(getSceneRooms) for (const [roomId, room] of sceneRooms) { if (roomId === currentSceneId) continue if (commsSceneToRemove.has(roomId)) continue const timeout = setTimeout(() => { - console.log('[BOEDO SceneComms]: disconnectSceneComms', roomId) void room.disconnect() commsSceneToRemove.delete(roomId) sceneRooms.delete(roomId) @@ -572,8 +568,6 @@ function* checkDisconnectScene(currentSceneId: string, commsSceneToRemove: Map { - console.log('[BOEDO] selectors disconnect') await islandRoom.disconnect() // TBD: should we disconnect from scenes here too ? }, @@ -83,18 +82,15 @@ export const getCommsRoom = (state: RootCommsState): RoomConnection | undefined const scene = sceneRoom?.sendChatMessage(message) await Promise.all([island, scene]) }, - // TBD: how voice chat works? - createVoiceHandler: async () => { + getVoiceHandler: async () => { if (await isWorld()) { - return islandRoom.createVoiceHandler() + return islandRoom.getVoiceHandler() } - - // TBD: Feature flag for backwards compatibility if (!sceneRoom) { debugger throw new Error('Scene room not avaialble') } - return sceneRoom.createVoiceHandler() + return sceneRoom.getVoiceHandler() } } as any as RoomConnection } diff --git a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts index 290f650977..0037320eb1 100644 --- a/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts +++ b/browser-interface/packages/shared/scene-loader/genesis-city-loader-impl/downloadManager.ts @@ -27,7 +27,6 @@ export class SceneDataDownloadManager { } async resolveEntitiesByPointer(pointers: string[]): Promise> { - console.log('[DownloadManager]resolveEntitiesByPointer ', pointers) const futures: Promise[] = [] const missingPointers: string[] = [] diff --git a/browser-interface/packages/shared/session/getPerformanceInfo.ts b/browser-interface/packages/shared/session/getPerformanceInfo.ts index 8640f6bb5a..954e187c67 100644 --- a/browser-interface/packages/shared/session/getPerformanceInfo.ts +++ b/browser-interface/packages/shared/session/getPerformanceInfo.ts @@ -54,9 +54,9 @@ export function incrementAvatarSceneMessages(value: number) { export function incrementCommsMessageSent(type: string, size: number, sceneId?: string) { if (!sceneId) { - sceneId = 'no-scene' + sceneId = 'noscene' } - const key = `${sceneId}:${type}` + const key = `${sceneId}-${type}` sentCommsMessages[key] = (sentCommsMessages[key] ?? 0) + size diff --git a/browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts b/browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts deleted file mode 100644 index 0bf3af908d..0000000000 --- a/browser-interface/packages/shared/voiceChat/VoiceCommunicator.ts +++ /dev/null @@ -1,545 +0,0 @@ -import { VoiceChatCodecWorkerMain, EncodeStream } from 'voice-chat-codec/VoiceChatCodecWorkerMain' -import { SortedLimitedQueue } from 'lib/data-structures/SortedLimitedQueue' -import defaultLogger from 'lib/logger' -import { VOICE_CHAT_SAMPLE_RATE, OPUS_FRAME_SIZE_MS } from 'voice-chat-codec/constants' -import { parse, write } from 'sdp-transform' -import { InputWorkletRequestTopic, OutputWorkletRequestTopic } from 'voice-chat-codec/types' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const workletWorkerRaw = require('../../../static/voice-chat-codec/audioWorkletProcessors.js.txt') -const workletWorkerUrl = URL.createObjectURL(new Blob([workletWorkerRaw], { type: 'application/javascript' })) - -export type AudioCommunicatorChannel = { - send(data: rfc4.Voice): any -} - -export type StreamPlayingListener = (streamId: string, playing: boolean) => any -export type StreamRecordingListener = (recording: boolean) => any -export type StreamRecordingErrorListener = (message: string) => any - -type VoiceOutput = { - encodedFramesQueue: SortedLimitedQueue - workletNode?: AudioWorkletNode - panNode?: PannerNode - spatialParams: VoiceSpatialParams - lastUpdateTime: number - playing: boolean - lastDecodedFrameOrder?: number -} - -type OutputStats = { - lostFrames: number - skippedFramesNotQueued: number - skippedFramesQueued: number -} - -type VoiceInput = { - workletNode: AudioWorkletNode - inputStream: MediaStreamAudioSourceNode - recordingContext: AudioContextWithInitPromise - encodeStream: EncodeStream -} - -export type VoiceCommunicatorOptions = { - sampleRate?: number - outputBufferLength?: number - maxDistance?: number - refDistance?: number - initialListenerParams?: VoiceSpatialParams - panningModel?: PanningModelType - distanceModel?: DistanceModelType - loopbackAudioElement?: HTMLAudioElement - volume?: number - mute?: boolean -} - -export type VoiceSpatialParams = { - position: [number, number, number] - orientation: [number, number, number] -} - -type AudioContextWithInitPromise = [AudioContext, Promise] - -const SELF_STREAM_ID = 'localhost' - -export class VoiceCommunicator { - private contextWithInitPromise: AudioContextWithInitPromise - private outputGainNode: GainNode - private outputStreamNode?: MediaStreamAudioDestinationNode - private loopbackConnections?: { src: RTCPeerConnection; dst: RTCPeerConnection } - private input?: VoiceInput - private voiceChatWorkerMain: VoiceChatCodecWorkerMain - private outputs: Record = {} - - private outputStats: Record = {} - - private streamPlayingListeners: StreamPlayingListener[] = [] - private streamRecordingListeners: StreamRecordingListener[] = [] - private streamRecordingErrorListeners: StreamRecordingErrorListener[] = [] - - private readonly sampleRate: number - private readonly outputBufferLength: number - private readonly outputExpireTime = 60 * 1000 - - private inputFramesIndex = 0 - - private checkStateTimeout: any | undefined = undefined - - private get context(): AudioContext { - return this.contextWithInitPromise[0] - } - - constructor(private channel: AudioCommunicatorChannel, private options: VoiceCommunicatorOptions) { - this.sampleRate = this.options.sampleRate ?? VOICE_CHAT_SAMPLE_RATE - this.outputBufferLength = this.options.outputBufferLength ?? 2.0 - - this.contextWithInitPromise = this.createContext({ sampleRate: this.sampleRate }) - - this.outputGainNode = this.context.createGain() - - if (options.loopbackAudioElement) { - // Workaround for echo cancellation. See: https://bugs.chromium.org/p/chromium/issues/detail?id=687574#c71 - this.outputStreamNode = this.context.createMediaStreamDestination() - this.outputGainNode.connect(this.outputStreamNode) - this.loopbackConnections = this.createRTCLoopbackConnection() - } else { - this.outputGainNode.connect(this.context.destination) - } - - if (this.options.initialListenerParams) { - this.setListenerSpatialParams(this.options.initialListenerParams) - } - - this.voiceChatWorkerMain = new VoiceChatCodecWorkerMain() - - this.startOutputsExpiration() - } - - public addStreamPlayingListener(listener: StreamPlayingListener) { - this.streamPlayingListeners.push(listener) - } - - public addStreamRecordingListener(listener: StreamRecordingListener) { - this.streamRecordingListeners.push(listener) - } - - public addStreamRecordingErrorListener(listener: StreamRecordingErrorListener) { - this.streamRecordingErrorListeners.push(listener) - } - - public hasInput() { - return !!this.input - } - - public statsFor(outputId: string) { - if (!this.outputStats[outputId]) { - this.outputStats[outputId] = { - lostFrames: 0, - skippedFramesNotQueued: 0, - skippedFramesQueued: 0 - } - } - - return this.outputStats[outputId] - } - - async playEncodedAudio(src: string, relativePosition: VoiceSpatialParams, encoded: rfc4.Voice) { - if (!this.outputs[src]) { - await this.createOutput(src, relativePosition) - } else { - this.outputs[src].lastUpdateTime = Date.now() - this.setVoiceRelativePosition(src, relativePosition) - } - - const output = this.outputs[src] - - if (output.lastDecodedFrameOrder && output.lastDecodedFrameOrder > encoded.index) { - this.statsFor(src).skippedFramesNotQueued += 1 - return - } - - const discarded = output.encodedFramesQueue.queue(encoded) - if (discarded) { - this.statsFor(src).skippedFramesQueued += 1 - } - } - - setListenerSpatialParams(spatialParams: VoiceSpatialParams) { - const listener = this.context.listener - listener.setPosition(spatialParams.position[0], spatialParams.position[1], spatialParams.position[2]) - listener.setOrientation( - spatialParams.orientation[0], - spatialParams.orientation[1], - spatialParams.orientation[2], - 0, - 1, - 0 - ) - } - - updatePannerNodeParameters(src: string) { - const panNode = this.outputs[src].panNode - const spatialParams = this.outputs[src].spatialParams - - if (panNode) { - panNode.positionX.value = spatialParams.position[0] - panNode.positionY.value = spatialParams.position[1] - panNode.positionZ.value = spatialParams.position[2] - panNode.orientationX.value = spatialParams.orientation[0] - panNode.orientationY.value = spatialParams.orientation[1] - panNode.orientationZ.value = spatialParams.orientation[2] - } - } - - setVolume(value: number) { - this.options.volume = value - const muted = this.options.mute ?? false - if (!muted) { - this.outputGainNode.gain.value = value - } - } - - setMute(mute: boolean) { - this.options.mute = mute - this.outputGainNode.gain.value = mute ? 0 : this.options.volume ?? 1 - } - - createWorkletFor(src: string) { - const workletNode = new AudioWorkletNode(this.context, 'outputProcessor', { - numberOfInputs: 0, - numberOfOutputs: 1, - processorOptions: { sampleRate: this.sampleRate, bufferLength: this.outputBufferLength } - }) - - workletNode.port.onmessage = (e) => { - if (e.data.topic === OutputWorkletRequestTopic.STREAM_PLAYING) { - if (this.outputs[src]) { - this.outputs[src].playing = e.data.playing - } - this.streamPlayingListeners.forEach((listener) => listener(src, e.data.playing)) - } - } - - return workletNode - } - - async setInputStream(stream: MediaStream) { - if (this.input) { - this.voiceChatWorkerMain.destroyEncodeStream(SELF_STREAM_ID) - if (this.input.recordingContext[0] !== this.context) { - this.input.recordingContext[0].close().catch((e) => defaultLogger.error('Error closing recording context', e)) - } - } - - try { - this.input = await this.createInputFor(stream, this.contextWithInitPromise) - } catch (e: any) { - // If this fails, then it most likely it is because the sample rate of the stream is incompatible with the context's, so we create a special context for recording - if (e.message.includes('sample-rate is currently not supported')) { - const recordingContext = this.createContext() - this.input = await this.createInputFor(stream, recordingContext) - } else { - throw e - } - } - } - - checkStatusTimeout() { - if (this.checkStateTimeout === undefined) { - this.checkStateTimeout = setTimeout(() => { - this.sendToInputWorklet(InputWorkletRequestTopic.CHECK_STATUS) - this.checkStateTimeout = undefined - }, 1200) - } - } - - start() { - if (this.input) { - this.input.workletNode.connect(this.input.recordingContext[0].destination) - this.sendToInputWorklet(InputWorkletRequestTopic.RESUME) - this.checkStatusTimeout() - } else { - this.notifyRecording(false) - } - } - - pause() { - if (this.input) { - this.sendToInputWorklet(InputWorkletRequestTopic.PAUSE) - this.checkStatusTimeout() - } else { - this.notifyRecording(false) - } - } - - private async createOutputNodes(src: string): Promise<{ workletNode: AudioWorkletNode; panNode: PannerNode }> { - await this.contextWithInitPromise[1] - const workletNode = this.createWorkletFor(src) - const panNode = this.context.createPanner() - panNode.coneInnerAngle = 180 - panNode.coneOuterAngle = 360 - panNode.coneOuterGain = 0.9 - panNode.maxDistance = this.options.maxDistance ?? 10000 - panNode.refDistance = this.options.refDistance ?? 5 - panNode.panningModel = this.options.panningModel ?? 'equalpower' - panNode.distanceModel = this.options.distanceModel ?? 'inverse' - panNode.rolloffFactor = 1.0 - workletNode.connect(panNode) - panNode.connect(this.outputGainNode) - - return { workletNode, panNode } - } - - private sendToInputWorklet(topic: InputWorkletRequestTopic) { - this.input?.workletNode.port.postMessage({ topic: topic }) - } - - private createRTCLoopbackConnection(currentRetryNumber: number = 0): { - src: RTCPeerConnection - dst: RTCPeerConnection - } { - const src = new RTCPeerConnection() - const dst = new RTCPeerConnection() - - let retryNumber = currentRetryNumber - - ;(async () => { - // When having an error, we retry in a couple of seconds. Up to 10 retries. - src.onconnectionstatechange = (_e) => { - if ( - src.connectionState === 'closed' || - src.connectionState === 'disconnected' || - (src.connectionState === 'failed' && currentRetryNumber < 10) - ) { - // Just in case, we close connections to free resources - this.closeLoopbackConnections() - this.loopbackConnections = this.createRTCLoopbackConnection(retryNumber) - } else if (src.connectionState === 'connected') { - // We reset retry number when the connection succeeds - retryNumber = 0 - } - } - - src.onicecandidate = (e) => e.candidate && dst.addIceCandidate(new RTCIceCandidate(e.candidate)) - dst.onicecandidate = (e) => e.candidate && src.addIceCandidate(new RTCIceCandidate(e.candidate)) - - dst.ontrack = (e) => (this.options.loopbackAudioElement!.srcObject = e.streams[0]) - - this.outputStreamNode!.stream.getTracks().forEach((track) => src.addTrack(track, this.outputStreamNode!.stream)) - - const offer = await src.createOffer() - - await src.setLocalDescription(offer) - - await dst.setRemoteDescription(offer) - const answer = await dst.createAnswer() - - const answerSdp = parse(answer.sdp!) - - answerSdp.media[0].fmtp[0].config = 'ptime=5;stereo=1;sprop-stereo=1;maxaveragebitrate=256000' - - answer.sdp = write(answerSdp) - - await dst.setLocalDescription(answer) - - await src.setRemoteDescription(answer) - })().catch((e) => { - defaultLogger.error('Error creating loopback connection', e) - src.close() - dst.close() - }) - - return { src, dst } - } - - private closeLoopbackConnections() { - if (this.loopbackConnections) { - const { src, dst } = this.loopbackConnections - - src.close() - dst.close() - } - } - - private async createOutput(src: string, relativePosition: VoiceSpatialParams) { - this.outputs[src] = { - encodedFramesQueue: new SortedLimitedQueue( - Math.ceil((this.outputBufferLength * 1000) / OPUS_FRAME_SIZE_MS), - (frameA, frameB) => frameA.index - frameB.index - ), - spatialParams: relativePosition, - lastUpdateTime: Date.now(), - playing: false - } - - const { workletNode, panNode } = await this.createOutputNodes(src) - - this.outputs[src].workletNode = workletNode - this.outputs[src].panNode = panNode - - const readEncodedBufferLoop = async () => { - if (this.outputs[src]) { - // We use three frames (120ms) as a jitter buffer. This is not mutch, but we don't want to add much latency. In the future we should maybe make this dynamic based on packet loss - const framesToRead = this.outputs[src].playing ? 3 : 1 - - const frames = await this.outputs[src].encodedFramesQueue.dequeueItemsWhenAvailable(framesToRead, 2000) - - if (frames.length > 0) { - this.countLostFrames(src, frames) - let stream = this.voiceChatWorkerMain.decodeStreams[src] - - if (!stream) { - stream = this.voiceChatWorkerMain.getOrCreateDecodeStream(src, this.sampleRate) - - stream.addAudioDecodedListener((samples) => { - this.outputs[src].lastUpdateTime = Date.now() - this.outputs[src].workletNode?.port.postMessage( - { topic: OutputWorkletRequestTopic.WRITE_SAMPLES, samples }, - [samples.buffer] - ) - }) - } - - frames.forEach((it) => stream.decode(it.encodedSamples)) - this.outputs[src].lastDecodedFrameOrder = frames[frames.length - 1].index - } - - await readEncodedBufferLoop() - } - } - - readEncodedBufferLoop().catch((e) => defaultLogger.log('Error while reading encoded buffer of ' + src, e)) - } - - private countLostFrames(src: string, frames: rfc4.Voice[]) { - // We can know a frame is lost if we have a missing frame index - let lostFrames = 0 - if (this.outputs[src].lastDecodedFrameOrder && this.outputs[src].lastDecodedFrameOrder! < frames[0].index) { - // We count the missing frame indexes from the last decoded frame. If there are no missin frames, 0 is added - lostFrames += frames[0].index - this.outputs[src].lastDecodedFrameOrder! - 1 - } - - for (let i = 0; i < frames.length - 1; i++) { - // We count the missing frame indexes in the current frames to decode. If there are no missin frames, 0 is added - lostFrames += frames[i + 1].index - frames[i].index - 1 - } - - this.statsFor(src).lostFrames += lostFrames - } - - private async createInputFor(stream: MediaStream, context: AudioContextWithInitPromise) { - await context[1] - const streamSource = context[0].createMediaStreamSource(stream) - const workletNode = new AudioWorkletNode(context[0], 'inputProcessor', { - numberOfInputs: 1, - numberOfOutputs: 1 - }) - - streamSource.connect(workletNode) - return { - recordingContext: context, - encodeStream: this.createInputEncodeStream(context[0], workletNode), - workletNode, - inputStream: streamSource - } - } - - private createInputEncodeStream(recordingContext: AudioContext, workletNode: AudioWorkletNode) { - const encodeStream = this.voiceChatWorkerMain.getOrCreateEncodeStream( - SELF_STREAM_ID, - this.sampleRate, - recordingContext.sampleRate - ) - - encodeStream.addAudioEncodedListener((data) => { - this.inputFramesIndex += 1 - this.channel.send({ - encodedSamples: data, - index: this.inputFramesIndex, - codec: rfc4.Voice_VoiceCodec.VC_OPUS - }) - }) - - workletNode.port.onmessage = (e) => { - if (e.data.topic === InputWorkletRequestTopic.ENCODE) { - encodeStream.encode(e.data.samples) - } - - if (e.data.topic === InputWorkletRequestTopic.ON_PAUSED) { - this.notifyRecording(false) - this.input?.workletNode.disconnect() - } - - if (e.data.topic === InputWorkletRequestTopic.ON_RECORDING) { - this.notifyRecording(true) - } - - if (e.data.topic === InputWorkletRequestTopic.TIMEOUT) { - this.notifyRecordingError('Something went wrong with the microphone. No message was recorded.') - } - } - - workletNode.onprocessorerror = (_e) => { - this.notifyRecording(false) - } - - return encodeStream - } - - private setVoiceRelativePosition(src: string, spatialParams: VoiceSpatialParams) { - this.outputs[src].spatialParams = spatialParams - this.updatePannerNodeParameters(src) - } - - private notifyRecording(recording: boolean) { - this.streamRecordingListeners.forEach((listener) => listener(recording)) - } - - private notifyRecordingError(message: string) { - this.streamRecordingErrorListeners.forEach((listener) => listener(message)) - } - - private startOutputsExpiration() { - const expireOutputs = () => { - Object.keys(this.outputs).forEach((outputId) => { - const output = this.outputs[outputId] - if (Date.now() - output.lastUpdateTime > this.outputExpireTime) { - this.destroyOutput(outputId) - } - }) - - setTimeout(expireOutputs, 2000) - } - - setTimeout(expireOutputs, 0) - } - - private createContext(contextOptions?: AudioContextOptions): AudioContextWithInitPromise { - const aContext = new AudioContext(contextOptions) - if (aContext.audioWorklet) { - const workletInitializedPromise = aContext.audioWorklet - .addModule(workletWorkerUrl) - .catch((e) => defaultLogger.error('Error loading worklet modules: ', e)) - return [aContext, workletInitializedPromise] - } else { - // TODO: trackEvent('error_initializing_worklet') to gain visibility about how many times is this issue happening - defaultLogger.error('Error loading worklet modules: audioWorklet undefined') - return [aContext, Promise.resolve()] - } - } - - private destroyOutput(outputId: string) { - this.disconnectOutputNodes(outputId) - - this.voiceChatWorkerMain.destroyDecodeStream(outputId) - - delete this.outputs[outputId] - } - - private disconnectOutputNodes(outputId: string) { - const output = this.outputs[outputId] - output.panNode?.disconnect() - output.workletNode?.disconnect() - } -} diff --git a/browser-interface/packages/shared/voiceChat/handlers.ts b/browser-interface/packages/shared/voiceChat/handlers.ts deleted file mode 100644 index fc7b7dece6..0000000000 --- a/browser-interface/packages/shared/voiceChat/handlers.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { getCurrentUserProfile } from 'shared/profiles/selectors' -import { Package } from 'shared/comms/interface/types' -import { getPeer } from 'shared/comms/peers' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' -import { store } from 'shared/store/isolatedStore' -import { getVoiceHandler, shouldPlayVoice } from './selectors' -import { voiceChatLogger } from './logger' -import { trackEvent } from 'shared/analytics/trackEvent' - -// TODO: create a component to emit opus audio in a specific position that can be used -// by the voicechat and by the SDK -export function processVoiceFragment(message: Package) { - const state = store.getState() - const voiceHandler = getVoiceHandler(state) - const profile = getCurrentUserProfile(state) - - // use getPeer instead of setupPeer to only reproduce voice messages from - // known avatars - const peerTrackingInfo = getPeer(message.address) - - if ( - voiceHandler && - profile && - peerTrackingInfo && - peerTrackingInfo.position && - shouldPlayVoice(state, profile, peerTrackingInfo.ethereumAddress) && - voiceHandler.playEncodedAudio - ) { - voiceHandler - .playEncodedAudio(peerTrackingInfo.ethereumAddress, peerTrackingInfo.position, message.data) - .catch((e: any) => { - trackEvent('error', { - context: 'voice-chat', - message: e.message, - stack: '' - }) - voiceChatLogger.error('Error playing encoded audio!', e) - }) - } -} diff --git a/browser-interface/packages/shared/voiceChat/sagas.ts b/browser-interface/packages/shared/voiceChat/sagas.ts index 3339f64d3a..a8c3b9f32c 100644 --- a/browser-interface/packages/shared/voiceChat/sagas.ts +++ b/browser-interface/packages/shared/voiceChat/sagas.ts @@ -42,13 +42,16 @@ import { import { RootVoiceChatState, VoiceChatState } from './types' import { VoiceHandler } from './VoiceHandler' -import { SET_ROOM_CONNECTION } from 'shared/comms/actions' +import { SET_SCENE_ROOM_CONNECTION } from 'shared/comms/actions' import { RoomConnection } from 'shared/comms/interface' -import { getCommsRoom } from 'shared/comms/selectors' +import { getCommsRoom, getSceneRoom } from 'shared/comms/selectors' import { waitForMetaConfigurationInitialization } from 'shared/meta/sagas' import { incrementCounter } from 'shared/analytics/occurences' import { RootWorldState } from 'shared/world/types' -import { waitForSelector } from 'lib/redux' +import { waitFor, waitForSelector } from 'lib/redux' +import { IRealmAdapter } from '../realm/types' +import { ensureRealmAdapter } from '../realm/ensureRealmAdapter' +import { isWorldLoaderActive } from '../realm/selectors' let audioRequestInitialized = false @@ -75,16 +78,24 @@ function* handleConnectVoiceChatToRoom() { yield call(waitForMetaConfigurationInitialization) while (true) { + // wait for next event to happen + yield take([SET_SCENE_ROOM_CONNECTION, JOIN_VOICE_CHAT, LEAVE_VOICE_CHAT]) + const joined: boolean = yield select(hasJoinedVoiceChat) const prevHandler = yield select(getVoiceHandler) let handler: VoiceHandler | null = null // if we are supposed to be joined, then ask the RoomConnection about the handler if (joined) { + const realmAdapter: IRealmAdapter = yield call(ensureRealmAdapter) + const isWorld = isWorldLoaderActive(realmAdapter) + if (!isWorld) { + yield call(waitFor(getSceneRoom, SET_SCENE_ROOM_CONNECTION)) + } const room: RoomConnection = yield select(getCommsRoom) if (room) { try { - handler = yield apply(room, room.createVoiceHandler, []) + handler = yield apply(room, room.getVoiceHandler, []) } catch (err: any) { yield put(setVoiceChatError(err.toString())) } @@ -101,9 +112,6 @@ function* handleConnectVoiceChatToRoom() { yield prevHandler.destroy() } } - - // wait for next event to happen - yield take([SET_ROOM_CONNECTION, JOIN_VOICE_CHAT, LEAVE_VOICE_CHAT]) } } @@ -116,7 +124,6 @@ function* handleRecordingRequest() { if (voiceHandler) { if (!isAllowedByScene || !requestedRecording) { - // Ensure that we're recording, to stop recording yield call(waitForSelector, isVoiceChatRecording) yield call(voiceHandler.setRecording, false) diff --git a/browser-interface/packages/shared/voiceChat/utils.ts b/browser-interface/packages/shared/voiceChat/utils.ts deleted file mode 100644 index 125f76ed0c..0000000000 --- a/browser-interface/packages/shared/voiceChat/utils.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { VoiceSpatialParams } from './VoiceCommunicator' -import * as rfc4 from 'shared/protocol/decentraland/kernel/comms/rfc4/comms.gen' -import { Quaternion, Vector3 } from '@dcl/ecs-math' - -export function getSpatialParamsFor(position: rfc4.Position): VoiceSpatialParams { - const orientation = Vector3.Backward().rotate( - Quaternion.FromArray([position.rotationX, position.rotationY, position.rotationZ, position.rotationW]) - ) - - return { - position: [position.positionX, position.positionY, position.positionZ], - orientation: [orientation.x, orientation.y, orientation.z] - } -}