From e83efacb21d883caf6fc781b40c7ab3bcaee6c81 Mon Sep 17 00:00:00 2001 From: Vittorio Palmisano Date: Wed, 22 May 2024 13:49:51 +0200 Subject: [PATCH] added audio e2e metric --- scripts/e2e-audio-stats.js | 201 +++++++++++++++++++++++++++++++++++++ scripts/e2e-video-stats.js | 8 +- scripts/get-user-media.js | 8 +- scripts/peer-connection.js | 7 +- src/rtcstats.ts | 3 + src/session.ts | 32 ++++-- 6 files changed, 245 insertions(+), 14 deletions(-) create mode 100644 scripts/e2e-audio-stats.js diff --git a/scripts/e2e-audio-stats.js b/scripts/e2e-audio-stats.js new file mode 100644 index 0000000..19167ef --- /dev/null +++ b/scripts/e2e-audio-stats.js @@ -0,0 +1,201 @@ +/* global log, enabledForSession, ggwave_factory, MeasuredStats */ + +/** + * Audio end-to-end delay stats. + * @type MeasuredStats + */ +const audioEndToEndDelay = new MeasuredStats({ ttl: 15 }) + +window.collectAudioEndToEndDelayStats = () => { + return audioEndToEndDelay.mean() +} + +function convertTypedArray(src, type) { + const buffer = new ArrayBuffer(src.byteLength) + new src.constructor(buffer).set(src) + return new type(buffer) +} + +let ggwave = null + +if (enabledForSession(window.PARAMS?.timestampWatermarkAudio)) { + document.addEventListener('DOMContentLoaded', async () => { + try { + ggwave = await ggwave_factory() + } catch (e) { + log(`ggwave error: ${e}`) + } + }) +} + +/** @type AudioContext */ +let audioContext = null +/** @type MediaStreamAudioDestinationNode */ +let audioDestination = null + +const SEND_PERIOD = 5000 + +function initAudioTimestampWatermarkSender() { + if (audioContext) return + log(`initAudioTimestampWatermarkSender with interval ${SEND_PERIOD}ms`) + + const AudioContext = window.AudioContext || window.webkitAudioContext + audioContext = new AudioContext({ + latencyHint: 'interactive', + sampleRate: 48000, + }) + audioDestination = audioContext.createMediaStreamDestination() + const parameters = ggwave.getDefaultParameters() + parameters.sampleRateInp = audioContext.sampleRate + parameters.sampleRateOut = audioContext.sampleRate + parameters.operatingMode = + ggwave.GGWAVE_OPERATING_MODE_TX | ggwave.GGWAVE_OPERATING_MODE_USE_DSS + const instance = ggwave.init(parameters) + + setInterval(() => { + const now = Date.now() + const waveform = ggwave.encode( + instance, + now.toString(), + ggwave.ProtocolId.GGWAVE_PROTOCOL_AUDIBLE_FAST, + 10, + ) + const buf = convertTypedArray(waveform, Float32Array) + const buffer = audioContext.createBuffer( + 1, + buf.length, + audioContext.sampleRate, + ) + buffer.copyToChannel(buf, 0) + const source = audioContext.createBufferSource() + source.buffer = buffer + source.connect(audioDestination) + source.start() + }, SEND_PERIOD) +} + +window.applyAudioTimestampWatermark = mediaStream => { + if (mediaStream.getAudioTracks().length === 0) { + return mediaStream + } + if (!audioDestination) { + initAudioTimestampWatermarkSender() + } + log( + `AudioTimestampWatermark tx overrideGetUserMediaStream`, + mediaStream.getAudioTracks()[0].id, + '->', + audioDestination.stream.getAudioTracks()[0].id, + ) + + // Mix original track with watermark. + const track = mediaStream.getAudioTracks()[0] + const trackSource = audioContext.createMediaStreamSource( + new MediaStream([track]), + ) + const gain = audioContext.createGain() + gain.gain.value = 0.005 + trackSource.connect(gain) + gain.connect(audioDestination) + + track.addEventListener('ended', () => { + trackSource.disconnect(gain) + gain.disconnect(audioDestination) + }) + + const newMediaStream = new MediaStream([ + audioDestination.stream.getAudioTracks()[0].clone(), + ...mediaStream.getVideoTracks(), + ]) + + return newMediaStream +} + +let processingAudioTracks = 0 + +window.recognizeAudioTimestampWatermark = track => { + if (processingAudioTracks > 4) { + return + } + processingAudioTracks += 1 + + const samplesPerFrame = 1024 + const buf = new Float32Array(samplesPerFrame) + let bufIndex = 0 + let instance = null + + const writableStream = new window.WritableStream( + { + async write(frame) { + const { numberOfFrames, sampleRate } = frame + if (instance === null) { + const parameters = ggwave.getDefaultParameters() + parameters.sampleRateInp = sampleRate + parameters.sampleRateOut = sampleRate + parameters.samplesPerFrame = samplesPerFrame + parameters.operatingMode = + ggwave.GGWAVE_OPERATING_MODE_RX | + ggwave.GGWAVE_OPERATING_MODE_USE_DSS + instance = ggwave.init(parameters) + if (instance < 0) { + log(`AudioTimestampWatermark rx init failed: ${instance}`) + return + } + processingAudioTracks += 1 + } + + try { + const tmp = new Float32Array(numberOfFrames) + frame.copyTo(tmp, { planeIndex: 0 }) + + const addedFrames = Math.min( + numberOfFrames, + samplesPerFrame - bufIndex, + ) + buf.set(tmp.slice(0, addedFrames), bufIndex) + bufIndex += numberOfFrames + + if (bufIndex < samplesPerFrame) return + + const now = Date.now() + const res = ggwave.decode(instance, convertTypedArray(buf, Int8Array)) + buf.set(tmp.slice(addedFrames), 0) + bufIndex = numberOfFrames - addedFrames + + if (res && res.length > 0) { + const data = new TextDecoder('utf-8').decode(res) + try { + const ts = parseInt(data) + const rxFrames = ggwave.rxDurationFrames(instance) + 4 + const rxFramesDuration = + (rxFrames * 1000 * samplesPerFrame) / sampleRate + const delay = now - ts - rxFramesDuration + log( + `AudioTimestampWatermark rx delay: ${delay}ms rxFrames: ${rxFrames} rxFramesDuration: ${rxFramesDuration}ms`, + ) + if (isFinite(delay) && delay > 0 && delay < 5000) { + audioEndToEndDelay.push(now, delay / 1000) + } + } catch (e) { + log( + `AudioTimestampWatermark rx failed to parse ${data}: ${e.message}`, + ) + } + } + } catch (err) { + log(`AudioTimestampWatermark error: ${err.message}`) + } + }, + close() { + processingAudioTracks -= 1 + if (instance) ggwave.free(instance) + }, + abort(err) { + log('AudioTimestampWatermark error:', err) + }, + }, + new CountQueuingStrategy({ highWaterMark: 100 }), + ) + const trackProcessor = new window.MediaStreamTrackProcessor({ track }) + trackProcessor.readable.pipeTo(writableStream) +} diff --git a/scripts/e2e-video-stats.js b/scripts/e2e-video-stats.js index 42c7e2b..6d797a5 100644 --- a/scripts/e2e-video-stats.js +++ b/scripts/e2e-video-stats.js @@ -17,7 +17,7 @@ window.collectVideoEndToEndDelayStats = () => { * @param {MediaStream} mediaStream * @returns {MediaStream} */ -window.applyTimestampWatermark = mediaStream => { +window.applyVideoTimestampWatermark = mediaStream => { if ( !('MediaStreamTrackProcessor' in window) || !('MediaStreamTrackGenerator' in window) @@ -145,11 +145,11 @@ async function loadTesseract() { } /** - * recognizeTimestampWatermark + * recognizeVideoTimestampWatermark * @param {MediaStreamTrack} videoTrack * @param {number} measureInterval */ -window.recognizeTimestampWatermark = async ( +window.recognizeVideoTimestampWatermark = async ( videoTrack, measureInterval = 5, ) => { @@ -205,7 +205,7 @@ window.recognizeTimestampWatermark = async ( } } } catch (err) { - log(`recognizeTimestampWatermark error: ${err.message}`) + log(`recognizeVideoTimestampWatermark error: ${err.message}`) } } videoFrame.close() diff --git a/scripts/get-user-media.js b/scripts/get-user-media.js index ced0d5c..4f26bb2 100644 --- a/scripts/get-user-media.js +++ b/scripts/get-user-media.js @@ -1,4 +1,4 @@ -/* global log, sleep, applyTimestampWatermark, enabledForSession */ +/* global log, sleep, applyAudioTimestampWatermark, applyVideoTimestampWatermark, enabledForSession */ const applyOverride = (constraints, override) => { if (override) { @@ -165,8 +165,12 @@ if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { log(`collectMediaTracks error:`, err) } + if (enabledForSession(window.PARAMS?.timestampWatermarkAudio)) { + mediaStream = applyAudioTimestampWatermark(mediaStream) + } + if (enabledForSession(window.PARAMS?.timestampWatermarkVideo)) { - mediaStream = applyTimestampWatermark(mediaStream) + mediaStream = applyVideoTimestampWatermark(mediaStream) } return mediaStream diff --git a/scripts/peer-connection.js b/scripts/peer-connection.js index 4557bb0..34c91de 100644 --- a/scripts/peer-connection.js +++ b/scripts/peer-connection.js @@ -1,4 +1,4 @@ -/* global log, PeerConnections, handleTransceiverForInsertableStreams, handleTransceiverForPlayoutDelayHint, recognizeTimestampWatermark, saveMediaTrack, enabledForSession, watchObjectProperty */ +/* global log, PeerConnections, handleTransceiverForInsertableStreams, handleTransceiverForPlayoutDelayHint, recognizeAudioTimestampWatermark, recognizeVideoTimestampWatermark, saveMediaTrack, enabledForSession, watchObjectProperty */ const timestampInsertableStreams = !!window.PARAMS?.timestampInsertableStreams @@ -171,13 +171,16 @@ window.RTCPeerConnection = function (conf, options) { } if (receiver.track.kind === 'video') { if (enabledForSession(window.PARAMS?.timestampWatermarkVideo)) { - recognizeTimestampWatermark(receiver.track) + recognizeVideoTimestampWatermark(receiver.track) } if (enabledForSession(window.PARAMS?.saveRecvVideoTrack)) { await saveMediaTrack(receiver.track, 'recv') } } else if (receiver.track.kind === 'audio') { + if (window.PARAMS?.timestampWatermarkAudio) { + recognizeAudioTimestampWatermark(receiver.track) + } if (enabledForSession(window.PARAMS?.saveRecvAudioTrack)) { await saveMediaTrack(receiver.track, 'recv') } diff --git a/src/rtcstats.ts b/src/rtcstats.ts index e3401f6..ab9f492 100644 --- a/src/rtcstats.ts +++ b/src/rtcstats.ts @@ -42,6 +42,9 @@ export enum PageStatsNames { /** The page HTTP receive latency. */ httpRecvLatency = 'httpRecvLatency', + /** The audio end to end total delay. */ + audioEndToEndDelay = 'audioEndToEndDelay', + /** The video end to end total delay. */ videoEndToEndDelay = 'videoEndToEndDelay', /** diff --git a/src/session.ts b/src/session.ts index d152499..677a5c1 100644 --- a/src/session.ts +++ b/src/session.ts @@ -56,6 +56,7 @@ declare global { signalingHost?: string participantName?: string }> + let collectAudioEndToEndDelayStats: () => number let collectVideoEndToEndDelayStats: () => number let collectVideoEndToEndNetworkDelayStats: () => number let collectHttpResourcesStats: () => { @@ -926,14 +927,28 @@ window.SERVER_USE_HTTPS = ${this.serverUseHttps}; process.env.EXTERNAL_PEER_CONNECTION === 'true' ? '-external' : '' }.js`, 'scripts/e2e-network-stats.js', + 'https://raw.githubusercontent.com/ggerganov/ggwave/master/bindings/javascript/ggwave.js', + 'scripts/e2e-audio-stats.js', 'scripts/e2e-video-stats.js', 'scripts/playout-delay-hint.js', 'scripts/page-stats.js', 'scripts/save-tracks.js', ]) { - const filePath = resolvePackagePath(name) - log.debug(`loading ${name} script from: ${filePath}`) - await page.evaluateOnNewDocument(fs.readFileSync(filePath, 'utf8')) + if (name.startsWith('http')) { + log.debug(`loading ${name} script`) + const res = await downloadUrl(name) + if (!res?.data) { + throw new Error(`Failed to download script from: ${name}`) + } + await page.evaluateOnNewDocument(res.data) + } else { + const filePath = resolvePackagePath(name) + if (!fs.existsSync(filePath)) { + throw new Error(`${name} script not found: ${filePath}`) + } + log.debug(`loading ${name} script from: ${filePath}`) + await page.evaluateOnNewDocument(fs.readFileSync(filePath, 'utf8')) + } } // Execute external script(s). @@ -964,9 +979,7 @@ window.SERVER_USE_HTTPS = ${this.serverUseHttps}; continue } log.debug(`loading custom script from file: ${filePath}`) - await page.evaluateOnNewDocument( - await fs.readFileSync(filePath, 'utf8'), - ) + await page.evaluateOnNewDocument(fs.readFileSync(filePath, 'utf8')) } } } @@ -1491,6 +1504,7 @@ window.SERVER_USE_HTTPS = ${this.serverUseHttps}; const pages: Record = {} const peerConnections: Record = {} + const audioEndToEndDelayStats: Record = {} const videoEndToEndDelayStats: Record = {} const videoEndToEndNetworkDelayStats: Record = {} const httpRecvBytesStats: Record = {} @@ -1515,11 +1529,13 @@ window.SERVER_USE_HTTPS = ${this.serverUseHttps}; // Collect stats from page. const { peerConnectionStats, + audioEndToEndDelay, videoEndToEndDelay, videoEndToEndNetworkDelay, httpResourcesStats, } = await page.evaluate(async () => ({ peerConnectionStats: await collectPeerConnectionStats(), + audioEndToEndDelay: collectAudioEndToEndDelayStats(), videoEndToEndDelay: collectVideoEndToEndDelayStats(), videoEndToEndNetworkDelay: collectVideoEndToEndNetworkDelayStats(), httpResourcesStats: collectHttpResourcesStats(), @@ -1562,6 +1578,9 @@ window.SERVER_USE_HTTPS = ${this.serverUseHttps}; peerConnections[hostKey] += activePeerConnections // E2E stats. + if (audioEndToEndDelay) { + audioEndToEndDelayStats[pageKey] = audioEndToEndDelay + } if (videoEndToEndDelay) { videoEndToEndDelayStats[pageKey] = videoEndToEndDelay } @@ -1682,6 +1701,7 @@ window.SERVER_USE_HTTPS = ${this.serverUseHttps}; collectedStats.errors = this.pageErrors collectedStats.warnings = this.pageWarnings collectedStats.peerConnections = peerConnections + collectedStats.audioEndToEndDelay = audioEndToEndDelayStats collectedStats.videoEndToEndDelay = videoEndToEndDelayStats collectedStats.videoEndToEndNetworkDelay = videoEndToEndNetworkDelayStats collectedStats.httpRecvBytes = httpRecvBytesStats