save

src/ClientMonitorConfig.ts

@@ -102,6 +102,25 @@ export type ClientMonitorConfig = {
         thresholdInMs: number;
     };
 
+    playoutDiscrepancyDetector: {
+        /**
+         * If true, the detection of playout discrepancies is disabled.
+         *
+         * DEFAULT: false
+         */
+        disabled?: boolean;
+
+        /**
+         * The low watermark for the skew of frames between the received and rendered
+         */
+        lowSkewThreshold: number;
+
+        /**
+         * The high watermark for the skew of frames between the received and rendered
+         */
+        highSkewThreshold: number;
+    };
+
     syntheticSamplesDetector: {
         /**
          * If true, the detection of synthesized samples is disabled.

src/ClientMonitorEvents.ts

@@ -88,6 +88,10 @@ export type IceTupleChangedEventPayload = ClientMonitorBaseEvent & {
 	peerConnectionMonitor: PeerConnectionMonitor,
 }
 
+export type InboundVideoPlayoutDiscrepancyEventPayload = ClientMonitorBaseEvent & {	
+	trackMonitor: InboundTrackMonitor,
+}
+
 export type ScoreEventPayload = ClientMonitorBaseEvent & {
 	clientScore: number,
 	remarks?: string[],
@@ -174,7 +178,8 @@ export type ClientMonitorEvents = {
 	'dry-inbound-track': [DryInboundTrackEventPayload],
 	'dry-outbound-track': [DryOutboundTrackEventPayload],
 	'ice-tuple-changed': [IceTupleChangedEventPayload],
-	'too-long-pc-connection-establishment': [TooLongPcConnectionEstablishmentEventPayload]
+	'too-long-pc-connection-establishment': [TooLongPcConnectionEstablishmentEventPayload],
+	'inbound-video-playout-discrepancy': [InboundVideoPlayoutDiscrepancyEventPayload],
 	// 'ice-restart': [peerConnectionMonitor: PeerConnectionMonitor],
 	'score': [ScoreEventPayload],
 

src/detectors/PlayoutDiscrepancyDetector.ts

@@ -1,5 +1,52 @@
-// ice tuple change detector
+import { InboundTrackMonitor } from "../monitors/InboundTrackMonitor";
+import { Detector } from "./Detector";
 
-// bufferedoverflowdetector
 
-export const empty = "empty";
+export class PlayoutDiscrepancyDetector implements Detector {
+	public readonly name = 'playout-discrepancy-detector';
+
+	public constructor(
+		public readonly trackMonitor: InboundTrackMonitor,
+	) {
+	}
+
+	private get peerConnection() {
+		return this.trackMonitor.getPeerConnection();
+	}
+
+	private get config() {
+		return this.peerConnection.parent.config.playoutDiscrepancyDetector;
+	}
+
+	public active = false;
+
+	public update() {
+		if (this.config.disabled) return;
+
+		const inboundRtp = this.trackMonitor.getInboundRtp();
+
+		if (!inboundRtp || !inboundRtp.deltaFramesReceived || !inboundRtp.deltaFramesRendered || !inboundRtp.ewmaFps) return;
+
+		const frameSkew = inboundRtp.deltaFramesReceived - inboundRtp.deltaFramesRendered;
+
+		if (this.active) {
+			if (frameSkew < this.config.lowSkewThreshold) {
+				this.active = false;
+				return;
+			}
+
+			return;
+		}
+
+		if (frameSkew < this.config.highSkewThreshold) return;
+
+		this.active = true;
+
+		const clientMonitor = this.peerConnection.parent;
+
+		clientMonitor.emit('inbound-video-playout-discrepancy', {
+			trackMonitor: this.trackMonitor,
+			clientMonitor: clientMonitor,
+		});
+	}
+}

src/detectors/SynthesizedSamplesDetector.ts

@@ -10,8 +10,6 @@ export class SynthesizedSamplesDetector implements Detector {
     ) {
     }
 
-    private _evented = false;
-
     private get peerConnection() {
         return this.mediaPlayout.getPeerConnection();
     }
@@ -23,7 +21,7 @@ export class SynthesizedSamplesDetector implements Detector {
     public update() {
         if (this.config.disabled) return;
         if (this.mediaPlayout.deltaSynthesizedSamplesDuration <= this.config.minSynthesizedSamplesDuration) {
-            return this._evented = false;
+            return;
         }
 
 
@@ -33,7 +31,5 @@ export class SynthesizedSamplesDetector implements Detector {
             mediaPlayoutMonitor: this.mediaPlayout,
             clientMonitor: clientMonitor,
         });
-
-        this._evented = true;
     }
 }

src/monitors/InboundRtpMonitor.ts

@@ -87,12 +87,17 @@ export class InboundRtpMonitor implements InboundRtpStats {
 	fractionLost?: number;
 	bitPerPixel?: number;
 	packetRate?: number | undefined;
+	ewmaFps?: number;
 
 	deltaPacketsLost?: number;
 	deltaPacketsReceived?: number;
 	deltaBytesReceived?: number;
 	deltaJitterBufferDelay?: number;
 	deltaCorruptionProbability?: number;
+	deltaFramesDecoded?: number;
+	deltaFramesReceived?: number;
+	deltaFramesRendered?: number;
+	deltaTime?: number;
 
 	/**
 	 * Additional data attached to this stats, will be shipped to the server
@@ -167,9 +172,20 @@ export class InboundRtpMonitor implements InboundRtpStats {
 				deltaCoruption / deltaMeasurements
 			);
 		}
+
 		if (this.jitterBufferDelay !== undefined && stats.jitterBufferDelay !== undefined) {
 			this.deltaJitterBufferDelay = stats.jitterBufferDelay - this.jitterBufferDelay;
 		}
+		if (this.framesDecoded !== undefined && stats.framesDecoded !== undefined) {
+			this.deltaFramesDecoded = stats.framesDecoded - this.framesDecoded;
+		}
+		if (this.framesReceived !== undefined && stats.framesReceived !== undefined) {
+			this.deltaFramesReceived = stats.framesReceived - this.framesReceived;
+		}
+		if (this.framesRendered !== undefined && stats.framesRendered !== undefined) {
+			this.deltaFramesRendered = stats.framesRendered - this.framesRendered;
+		}
+		this.deltaTime = elapsedInMs;
 
 		Object.assign(this, stats);
 
@@ -195,6 +211,9 @@ export class InboundRtpMonitor implements InboundRtpStats {
 			this.fractionLost = 0 < this.packetsReceived && 0 < this.packetsLost
 				? (this.packetsLost) / (this.packetsLost + this.packetsReceived) : 0.0;
 		}
+		if (this.framesDecoded !== undefined) {
+			this.ewmaFps = this.ewmaFps ? 0.9 * this.ewmaFps + 0.1 * this.framesDecoded : this.framesDecoded;
+		}
 	}
 
 	public getRemoteOutboundRtp(): RemoteOutboundRtpMonitor | undefined {

src/monitors/InboundTrackMonitor.ts

@@ -5,6 +5,7 @@ import { DryInboundTrackDetector } from "../detectors/DryInboundTrack";
 import { CalculatedScore } from "../scores/CalculatedScore";
 import { InboundRtpMonitor } from "./InboundRtpMonitor";
 import { OutboundTrackSample } from "../schema/ClientSample";
+import { PlayoutDiscrepancyDetector } from "../detectors/PlayoutDiscrepancyDetector";
 
 export class InboundTrackMonitor {
 	public readonly direction = 'inbound';
@@ -46,6 +47,7 @@ export class InboundTrackMonitor {
 			this.detectors.add(new AudioDesyncDetector(this));
 		} else if (this.kind === 'video') {
 			this.detectors.add(new FreezedVideoTrackDetector(this));
+			this.detectors.add(new PlayoutDiscrepancyDetector(this));
 		}
 
 		// for mediasoup probator we don't need to run detectors

src/scores/DefaultScoreCalculator.ts

@@ -172,7 +172,10 @@ export class DefaultScoreCalculator {
 		);
 
 		appData.lastNScores.push(scoreValue);
-		score.value = this._calculateFinalScore(appData.lastNScores);
+
+		const finalScore = this._calculateFinalScore(appData.lastNScores);
+
+		score.value = finalScore ? this._getRoundedScore(finalScore) : undefined;
 	}
 
 	public _calculateTrackScore(trackMonitor: TrackMonitor) {
@@ -286,8 +289,10 @@ export class DefaultScoreCalculator {
 		);
 
 		appData.lastNScores.push(scoreValue);
-
-		trackMonitor.calculatedScore.value = this._calculateFinalScore(appData.lastNScores);
+
+		const finalScore = this._calculateFinalScore(appData.lastNScores)
+
+		trackMonitor.calculatedScore.value = finalScore ? this._getRoundedScore(finalScore) : undefined;
 	}
 
 	private _calculateInboundAudioTrackScore(trackMonitor: InboundTrackMonitor) {
@@ -313,13 +318,15 @@ export class DefaultScoreCalculator {
 			return;
 		}
 
-		trackMonitor.calculatedScore.value = calculateAudioMOS(
-			bitrate,
-			packetLoss,
-			bufferDelayInMs,
-			roundTripTimeInMs,
-			dtxMode,
-			fec,
+		trackMonitor.calculatedScore.value = this._getRoundedScore(
+			calculateAudioMOS(
+				bitrate,
+				packetLoss,
+				bufferDelayInMs,
+				roundTripTimeInMs,
+				dtxMode,
+				fec,
+			)
 		);
 	}
 
@@ -463,16 +470,14 @@ export class DefaultScoreCalculator {
 		) / DefaultScoreCalculator.NORMALIZATION_FACTOR
 
 		const lossPenalty = Math.exp(-(outboundRtp.getRemoteInboundRtp()?.deltaPacketsLost ?? 0) / 2); // Exponential decay for packet loss impact
-
-		console.warn('normalizedBitrate', normalizedBitrate, 'lossPenalty', lossPenalty);
-
-		trackMonitor.calculatedScore.value = Math.max(
+		const score = Math.max(
 			DefaultScoreCalculator.MIN_SCORE,
 			Math.min(
 				DefaultScoreCalculator.MAX_SCORE,
 				5 * normalizedBitrate * lossPenalty
 			)
-		)
+		);
+		trackMonitor.calculatedScore.value = this._getRoundedScore(score);
 	}
 
 	private _calculateFinalScore(scores: number[]) {
@@ -494,4 +499,8 @@ export class DefaultScoreCalculator {
 
 		return totalScore / counter;
 	}
+
+	private _getRoundedScore(score: number) {
+		return Math.round(score * 100) / 100;
+	}
 }