Add support for playout-delay extension

http://www.webrtc.org/experiments/rtp-hdrext/playout-delay Like video-orientation extension, an SFU must proxy this extension to the consumers or discard it if the consumer doesn't support this extension.
versatica · Jun 21, 2024 · b445c03 · b445c03
1 parent dedfb24
commit b445c03
Show file tree

Hide file tree

Showing 14 changed files with 144 additions and 1 deletion.
diff --git a/node/src/RtpParameters.ts b/node/src/RtpParameters.ts
@@ -321,7 +321,8 @@ export type RtpHeaderExtensionUri =
 	| 'urn:ietf:params:rtp-hdrext:toffset'
 	| 'http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01'
 	| 'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time'
-	| 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time';
+	| 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time'
+	| 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay';
 
 /**
  * Defines a RTP header extension within the RTP parameters. The list of RTP
@@ -758,6 +759,10 @@ export function rtpHeaderExtensionUriFromFbs(
 		case FbsRtpHeaderExtensionUri.AbsCaptureTime: {
 			return 'http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time';
 		}
+
+		case FbsRtpHeaderExtensionUri.PlayoutDelay: {
+			return 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay';
+		}
 	}
 }
 
@@ -809,6 +814,10 @@ export function rtpHeaderExtensionUriToFbs(
 			return FbsRtpHeaderExtensionUri.AbsCaptureTime;
 		}
 
+		case 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay': {
+			return FbsRtpHeaderExtensionUri.PlayoutDelay;
+		}
+
 		default: {
 			throw new TypeError(`invalid RtpHeaderExtensionUri: ${uri}`);
 		}

diff --git a/node/src/supportedRtpCapabilities.ts b/node/src/supportedRtpCapabilities.ts
@@ -328,6 +328,13 @@ const supportedRtpCapabilities: RtpCapabilities = {
 			preferredEncrypt: false,
 			direction: 'sendrecv',
 		},
+		{
+			kind: 'video',
+			uri: 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay',
+			preferredId: 14,
+			preferredEncrypt: false,
+			direction: 'sendrecv',
+		},
 	],
 };
 

diff --git a/node/src/test/test-PipeTransport.ts b/node/src/test/test-PipeTransport.ts
@@ -387,6 +387,12 @@ test('router.pipeToRouter() succeeds with video', async () => {
 			encrypt: false,
 			parameters: {},
 		},
+		{
+			uri: 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay',
+			id: 14,
+			encrypt: false,
+			parameters: {},
+		},
 	]);
 
 	expect(pipeConsumer.type).toBe('pipe');
@@ -448,6 +454,12 @@ test('router.pipeToRouter() succeeds with video', async () => {
 			encrypt: false,
 			parameters: {},
 		},
+		{
+			uri: 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay',
+			id: 14,
+			encrypt: false,
+			parameters: {},
+		},
 	]);
 	expect(pipeProducer.paused).toBe(true);
 }, 2000);
@@ -561,6 +573,12 @@ test('router.createPipeTransport() with enableRtx succeeds', async () => {
 			encrypt: false,
 			parameters: {},
 		},
+		{
+			uri: 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay',
+			id: 14,
+			encrypt: false,
+			parameters: {},
+		},
 	]);
 
 	expect(pipeConsumer.type).toBe('pipe');

diff --git a/rust/src/rtp_parameters.rs b/rust/src/rtp_parameters.rs
@@ -585,6 +585,10 @@ pub enum RtpHeaderExtensionUri {
     /// <http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time>
     #[serde(rename = "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time")]
     AbsCaptureTime,
+    /// <http://www.webrtc.org/experiments/rtp-hdrext/playout-delay>
+    #[serde(rename = "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay")]
+    PlayoutDelay,
+
     #[doc(hidden)]
     #[serde(other, rename = "unsupported")]
     Unsupported,
@@ -620,6 +624,9 @@ impl RtpHeaderExtensionUri {
             RtpHeaderExtensionUri::AbsCaptureTime => {
                 rtp_parameters::RtpHeaderExtensionUri::AbsCaptureTime
             }
+            RtpHeaderExtensionUri::PlayoutDelay => {
+                rtp_parameters::RtpHeaderExtensionUri::PlayoutDelay
+            }
             RtpHeaderExtensionUri::Unsupported => panic!("Invalid RTP extension header URI"),
         }
     }
@@ -653,6 +660,9 @@ impl RtpHeaderExtensionUri {
             rtp_parameters::RtpHeaderExtensionUri::AbsCaptureTime => {
                 RtpHeaderExtensionUri::AbsCaptureTime
             }
+            rtp_parameters::RtpHeaderExtensionUri::PlayoutDelay => {
+                RtpHeaderExtensionUri::PlayoutDelay
+            }
         }
     }
 }
@@ -679,6 +689,7 @@ impl FromStr for RtpHeaderExtensionUri {
             "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time" => {
                 Ok(Self::AbsCaptureTime)
             }
+            "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay" => Ok(Self::PlayoutDelay),
             _ => Err(RtpHeaderExtensionUriParseError::Unsupported),
         }
     }
@@ -710,6 +721,9 @@ impl RtpHeaderExtensionUri {
             RtpHeaderExtensionUri::AbsCaptureTime => {
                 "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time"
             }
+            RtpHeaderExtensionUri::PlayoutDelay => {
+                "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"
+            }
             RtpHeaderExtensionUri::Unsupported => "unsupported",
         }
     }

diff --git a/rust/src/supported_rtp_capabilities.rs b/rust/src/supported_rtp_capabilities.rs
@@ -386,6 +386,13 @@ pub fn get_supported_rtp_capabilities() -> RtpCapabilities {
                 preferred_encrypt: false,
                 direction: RtpHeaderExtensionDirection::SendRecv,
             },
+            RtpHeaderExtension {
+                kind: MediaKind::Video,
+                uri: RtpHeaderExtensionUri::PlayoutDelay,
+                preferred_id: 14,
+                preferred_encrypt: false,
+                direction: RtpHeaderExtensionDirection::SendRecv,
+            },
         ],
     }
 }
diff --git a/rust/tests/integration/pipe_transport.rs b/rust/tests/integration/pipe_transport.rs
@@ -499,6 +499,11 @@ fn pipe_to_router_succeeds_with_video() {
                     id: 13,
                     encrypt: false,
                 },
+                RtpHeaderExtensionParameters {
+                    uri: RtpHeaderExtensionUri::PlayoutDelay,
+                    id: 14,
+                    encrypt: false,
+                },
             ],
         );
         assert_eq!(pipe_consumer.r#type(), ConsumerType::Pipe);
@@ -556,6 +561,11 @@ fn pipe_to_router_succeeds_with_video() {
                     id: 13,
                     encrypt: false,
                 },
+                RtpHeaderExtensionParameters {
+                    uri: RtpHeaderExtensionUri::PlayoutDelay,
+                    id: 14,
+                    encrypt: false,
+                },
             ],
         );
         assert!(pipe_producer.paused());
@@ -753,6 +763,11 @@ fn create_with_enable_rtx_succeeds() {
                     id: 13,
                     encrypt: false,
                 },
+                RtpHeaderExtensionParameters {
+                    uri: RtpHeaderExtensionUri::PlayoutDelay,
+                    id: 14,
+                    encrypt: false,
+                },
             ],
         );
         assert_eq!(pipe_consumer.r#type(), ConsumerType::Pipe);

diff --git a/worker/fbs/rtpParameters.fbs b/worker/fbs/rtpParameters.fbs
@@ -73,6 +73,7 @@ enum RtpHeaderExtensionUri: uint8 {
     TransportWideCcDraft01,
     AbsSendTime,
     AbsCaptureTime,
+    PlayoutDelay,
 }
 
 table RtpHeaderExtensionParameters {

diff --git a/worker/include/RTC/RtpDictionaries.hpp b/worker/include/RTC/RtpDictionaries.hpp
@@ -127,6 +127,7 @@ namespace RTC
 			VIDEO_ORIENTATION      = 11,
 			TOFFSET                = 12,
 			ABS_CAPTURE_TIME       = 13,
+			PLAYOUT_DELAY          = 14,
 		};
 
 	public:

diff --git a/worker/include/RTC/RtpHeaderExtensionIds.hpp b/worker/include/RTC/RtpHeaderExtensionIds.hpp
@@ -19,6 +19,7 @@ namespace RTC
 		uint8_t videoOrientation{ 0u };
 		uint8_t toffset{ 0u };
 		uint8_t absCaptureTime{ 0u };
+		uint8_t playoutDelay{ 0u };
 	};
 } // namespace RTC
 

diff --git a/worker/include/RTC/RtpPacket.hpp b/worker/include/RTC/RtpPacket.hpp
@@ -309,6 +309,11 @@ namespace RTC
 			this->videoOrientationExtensionId = id;
 		}
 
+		void SetPlayoutDelayExtensionId(uint8_t id)
+		{
+			this->playoutDelayExtensionId = id;
+		}
+
 		bool ReadMid(std::string& mid) const
 		{
 			uint8_t extenLen;
@@ -489,6 +494,20 @@ namespace RTC
 			return true;
 		}
 
+		bool ReadPlayoutDelay(uint16_t& minDelay, uint16_t& maxDelay) const
+		{
+			uint8_t extenLen;
+			uint8_t* extenValue = GetExtension(this->playoutDelayExtensionId, extenLen);
+			if (extenLen != 3)
+			{
+				return false;
+			}
+			uint32_t v = Utils::Byte::Get3Bytes(extenValue, 0);
+			minDelay   = v >> 12u;
+			maxDelay   = v & 0xFFFu;
+			return true;
+		}
+
 		bool HasExtension(uint8_t id) const
 		{
 			if (id == 0u)
@@ -672,6 +691,7 @@ namespace RTC
 		uint8_t frameMarkingExtensionId{ 0u };
 		uint8_t ssrcAudioLevelExtensionId{ 0u };
 		uint8_t videoOrientationExtensionId{ 0u };
+		uint8_t playoutDelayExtensionId{ 0u };
 		uint8_t* payload{ nullptr };
 		size_t payloadLength{ 0u };
 		uint8_t payloadPadding{ 0u };

diff --git a/worker/src/RTC/Consumer.cpp b/worker/src/RTC/Consumer.cpp
@@ -95,6 +95,11 @@ namespace RTC
 				this->rtpHeaderExtensionIds.transportWideCc01 = exten.id;
 			}
 
+			if (this->rtpHeaderExtensionIds.playoutDelay == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::PLAYOUT_DELAY)
+			{
+				this->rtpHeaderExtensionIds.playoutDelay = exten.id;
+			}
+
 			if (this->rtpHeaderExtensionIds.mid == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::MID)
 			{
 				this->rtpHeaderExtensionIds.mid = exten.id;

diff --git a/worker/src/RTC/Producer.cpp b/worker/src/RTC/Producer.cpp
@@ -192,6 +192,11 @@ namespace RTC
 			{
 				this->rtpHeaderExtensionIds.absCaptureTime = exten.id;
 			}
+
+			if (this->rtpHeaderExtensionIds.playoutDelay == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::PLAYOUT_DELAY)
+			{
+				this->rtpHeaderExtensionIds.playoutDelay = exten.id;
+			}
 		}
 
 		// Set the RTCP report generation interval.
@@ -1264,6 +1269,19 @@ namespace RTC
 				bufferPtr += extenLen;
 			}
 
+			// Proxy http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
+			extenValue = packet->GetExtension(this->rtpHeaderExtensionIds.playoutDelay, extenLen);
+
+			if (extenValue)
+			{
+				std::memcpy(bufferPtr, extenValue, extenLen);
+
+				extensions.emplace_back(
+				  static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::PLAYOUT_DELAY), extenLen, bufferPtr);
+
+				bufferPtr += extenLen;
+			}
+
 			if (this->kind == RTC::Media::Kind::AUDIO)
 			{
 				// Proxy urn:ietf:params:rtp-hdrext:ssrc-audio-level.
@@ -1385,6 +1403,8 @@ namespace RTC
 			packet->SetMidExtensionId(static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::MID));
 			packet->SetAbsSendTimeExtensionId(
 			  static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::ABS_SEND_TIME));
+			packet->SetPlayoutDelayExtensionId(
+			  static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::PLAYOUT_DELAY));
 			packet->SetTransportWideCc01ExtensionId(
 			  static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::TRANSPORT_WIDE_CC_01));
 			// NOTE: Remove this once framemarking draft becomes RFC.

diff --git a/worker/src/RTC/RtpDictionaries/RtpHeaderExtensionUri.cpp b/worker/src/RTC/RtpDictionaries/RtpHeaderExtensionUri.cpp
@@ -67,6 +67,11 @@ namespace RTC
 			{
 				return RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME;
 			}
+
+			case FBS::RtpParameters::RtpHeaderExtensionUri::PlayoutDelay:
+			{
+				return RtpHeaderExtensionUri::Type::PLAYOUT_DELAY;
+			}
 		}
 	}
 
@@ -129,6 +134,11 @@ namespace RTC
 			{
 				return FBS::RtpParameters::RtpHeaderExtensionUri::AbsCaptureTime;
 			}
+
+			case RtpHeaderExtensionUri::Type::PLAYOUT_DELAY:
+			{
+				return FBS::RtpParameters::RtpHeaderExtensionUri::PlayoutDelay;
+			}
 		}
 	}
 

diff --git a/worker/src/RTC/RtpPacket.cpp b/worker/src/RTC/RtpPacket.cpp
@@ -300,6 +300,19 @@ namespace RTC
 				  rotation);
 			}
 		}
+		if (this->playoutDelayExtensionId != 0u)
+		{
+			uint16_t minDelay;
+			uint16_t maxDelay;
+			if (ReadPlayoutDelay(minDelay, maxDelay))
+			{
+				MS_DUMP(
+				  "  playoutDelay: extId:%" PRIu8 ", minDelay:%" PRIu16 ", maxDelay:%" PRIu16,
+				  this->videoOrientationExtensionId,
+				  minDelay,
+				  maxDelay);
+			}
+		}
 		MS_DUMP("  csrc count: %" PRIu8, this->header->csrcCount);
 		MS_DUMP("  marker: %s", HasMarker() ? "true" : "false");
 		MS_DUMP("  payload type: %" PRIu8, GetPayloadType());
@@ -387,6 +400,7 @@ namespace RTC
 		this->frameMarkingExtensionId      = 0u;
 		this->ssrcAudioLevelExtensionId    = 0u;
 		this->videoOrientationExtensionId  = 0u;
+		this->playoutDelayExtensionId      = 0u;
 
 		// Clear the One-Byte and Two-Bytes extension elements maps.
 		std::fill(std::begin(this->oneByteExtensions), std::end(this->oneByteExtensions), nullptr);
@@ -748,6 +762,7 @@ namespace RTC
 		packet->frameMarkingExtensionId      = this->frameMarkingExtensionId;
 		packet->ssrcAudioLevelExtensionId    = this->ssrcAudioLevelExtensionId;
 		packet->videoOrientationExtensionId  = this->videoOrientationExtensionId;
+		packet->playoutDelayExtensionId      = this->playoutDelayExtensionId;
 		// Assign the payload descriptor handler.
 		packet->payloadDescriptorHandler = this->payloadDescriptorHandler;
 		// Store allocated buffer.