diff --git a/d.ts/src/player/mse-player.d.ts b/d.ts/src/player/mse-player.d.ts index ec1161d9..6b434c95 100644 --- a/d.ts/src/player/mse-player.d.ts +++ b/d.ts/src/player/mse-player.d.ts @@ -14,6 +14,7 @@ declare class MSEPlayer { unload(): void; play(): Promise; pause(): void; + selectAudioTrack(index: number): void; get type(): string; get buffered(): TimeRanges; get duration(): number; diff --git a/d.ts/src/player/player-engine-dedicated-thread.d.ts b/d.ts/src/player/player-engine-dedicated-thread.d.ts index b2c74844..f6933ba4 100644 --- a/d.ts/src/player/player-engine-dedicated-thread.d.ts +++ b/d.ts/src/player/player-engine-dedicated-thread.d.ts @@ -29,6 +29,7 @@ declare class PlayerEngineDedicatedThread implements PlayerEngine { play(): Promise; pause(): void; seek(seconds: number): void; + selectAudioTrack(track: number): void; get mediaInfo(): MediaInfo; get statisticsInfo(): any; _onLoggingConfigChanged(config: any): void; diff --git a/d.ts/src/player/player-engine-main-thread.d.ts b/d.ts/src/player/player-engine-main-thread.d.ts index d43e6fa1..2deda5a8 100644 --- a/d.ts/src/player/player-engine-main-thread.d.ts +++ b/d.ts/src/player/player-engine-main-thread.d.ts @@ -31,6 +31,7 @@ declare class PlayerEngineMainThread implements PlayerEngine { play(): Promise; pause(): void; seek(seconds: number): void; + selectAudioTrack(track: number): void; get mediaInfo(): MediaInfo; get statisticsInfo(): any; private _onMSESourceOpen; diff --git a/docs/api.md b/docs/api.md index 41db38f2..78a10b80 100644 --- a/docs/api.md +++ b/docs/api.md @@ -158,6 +158,7 @@ interface Player { unload(): void; play(): Promise; pause(): void; + selectAudioTrack(track: number): void; type: string; buffered: TimeRanges; duration: number; diff --git a/src/core/mse-controller.js b/src/core/mse-controller.js index 7aaaa41f..d6572065 100644 --- a/src/core/mse-controller.js +++ b/src/core/mse-controller.js @@ -250,6 +250,7 @@ class MSEController { return; } } else { + this._sourceBuffers[is.type].changeType(mimeType); Log.v(this.TAG, `Notice: ${is.type} mimeType changed, origin: ${this._mimeTypes[is.type]}, target: ${mimeType}`); } this._mimeTypes[is.type] = mimeType; diff --git a/src/core/transmuxer.js b/src/core/transmuxer.js index f313cfaa..1e90f3db 100644 --- a/src/core/transmuxer.js +++ b/src/core/transmuxer.js @@ -142,6 +142,14 @@ class Transmuxer { } } + selectAudioTrack(track) { + if (this._worker) { + this._worker.postMessage({cmd: 'select_audio_track', param: track}); + } else { + this._controller.selectAudioTrack(track); + } + } + _onInitSegment(type, initSegment) { // do async invoke Promise.resolve().then(() => { diff --git a/src/core/transmuxing-controller.js b/src/core/transmuxing-controller.js index f1b6bcc0..81ac0ec3 100644 --- a/src/core/transmuxing-controller.js +++ b/src/core/transmuxing-controller.js @@ -118,8 +118,8 @@ class TransmuxingController { this._emitter.removeListener(event, listener); } - start() { - this._loadSegment(0); + start(optionalFrom) { + this._loadSegment(0, optionalFrom); this._enableStatisticsReporter(); } @@ -134,10 +134,10 @@ class TransmuxingController { ioctl.onRedirect = this._onIORedirect.bind(this); ioctl.onRecoveredEarlyEof = this._onIORecoveredEarlyEof.bind(this); - if (optionalFrom) { - this._demuxer.bindDataSource(this._ioctl); - } else { + if (typeof optionalFrom === 'undefined') { ioctl.onDataArrival = this._onInitChunkArrival.bind(this); + } else { + ioctl.onDataArrival = this._onExistsChunkArrival.bind(this); } ioctl.open(optionalFrom); @@ -220,6 +220,19 @@ class TransmuxingController { this._enableStatisticsReporter(); } + selectAudioTrack(track) { + if (this._config.isLive) { + this._demuxer.selectAudioTrack(track); + } else { // FIXME: Seek needed? + this.stop(); + this._remuxer.insertDiscontinuity(); + this._demuxer.resetMediaInfo(); + this._demuxer._firstParse = true; + this._demuxer.selectAudioTrack(track); + this.start(0); + } + } + _searchSegmentIndexContains(milliseconds) { let segments = this._mediaDataSource.segments; let idx = segments.length - 1; @@ -233,6 +246,14 @@ class TransmuxingController { return idx; } + _onExistsChunkArrival(data, byteStart) { + // IOController seeked immediately after opened, byteStart > 0 callback may received + this._demuxer.bindDataSource(this._ioctl); + this._demuxer.timestampBase = this._mediaDataSource.segments[this._currentSegmentIndex].timestampBase; + + return this._demuxer.parseChunks(data, byteStart); + } + _onInitChunkArrival(data, byteStart) { let consumed = 0; diff --git a/src/core/transmuxing-worker.js b/src/core/transmuxing-worker.js index 525cc8a9..99997c38 100644 --- a/src/core/transmuxing-worker.js +++ b/src/core/transmuxing-worker.js @@ -46,7 +46,7 @@ let TransmuxingWorker = function (self) { self.addEventListener('message', function (e) { switch (e.data.cmd) { case 'init': - controller = new TransmuxingController(e.data.param[0], e.data.param[1]); + controller = new TransmuxingController(e.data.param[0], e.data.param[1], e.data.param[2]); controller.on(TransmuxingEvents.IO_ERROR, onIOError.bind(this)); controller.on(TransmuxingEvents.DEMUX_ERROR, onDemuxError.bind(this)); controller.on(TransmuxingEvents.INIT_SEGMENT, onInitSegment.bind(this)); @@ -88,6 +88,9 @@ let TransmuxingWorker = function (self) { case 'resume': controller.resume(); break; + case 'select_audio_track': + controller.selectAudioTrack(e.data.param); + break; case 'logging_config': { let config = e.data.param; LoggingControl.applyConfig(config); diff --git a/src/demux/flv-demuxer.js b/src/demux/flv-demuxer.js index 6a95ef83..3a5812cc 100644 --- a/src/demux/flv-demuxer.js +++ b/src/demux/flv-demuxer.js @@ -93,6 +93,12 @@ class FLVDemuxer { fps_den: 1000 }; + this._enhancedFlvAudioMultitrackMode = null; + this._enhanedFlvAudioTrackIds = []; + this._currentAudioTrackIndex = 0; + this._currentAudioTrackId = null; + this._audioTrackInitSegments = new Map(); + this._flvSoundRateTable = [5500, 11025, 22050, 44100, 48000]; this._mpegSamplingRates = [ @@ -255,6 +261,52 @@ class FLVDemuxer { this._mediaInfo.hasVideo = hasVideo; } + selectAudioTrack(track) { + let newTrackId = null; + if (track >= 1) { + newTrackId = this._enhanedFlvAudioTrackIds[track - 1] || null; + } + if (this._currentAudioTrackId === newTrackId) { return; } + + this._currentAudioTrackId = newTrackId; + let meta = this._audioTrackInitSegments.get(this._currentAudioTrackId); + + // flush segment + this._onDataAvailable(this._audioTrack, this._videoTrack); + this._videoTrack = {type: 'video', id: 1, sequenceNumber: 0, samples: [], length: 0}; + this._audioTrack = {type: 'audio', id: 2, sequenceNumber: 0, samples: [], length: 0}; + + // then notify new metadata + this._dispatch = false; + this._onTrackMetadata('audio', meta); + + let mi = this._mediaInfo; + mi.audioCodec = meta.originalCodec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; + } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + } + if (mi.isComplete()) { + this._onMediaInfo(mi); + } + } + + _updateAudioTrackIds(trackIndex, newTrackIds) { + this._enhanedFlvAudioTrackIds = newTrackIds; + let newTrackId = null; + if (trackIndex >= 1) { + newTrackId = this._enhanedFlvAudioTrackIds[trackIndex - 1] || null; + } + if (this._currentAudioTrackId !== newTrackId) { + this.selectAudioTrack(trackIndex); + } + } + resetMediaInfo() { this._mediaInfo = new MediaInfo(); } @@ -494,27 +546,63 @@ class FLVDemuxer { let soundFormat = soundSpec >>> 4; if (soundFormat === 9) { // Enhanced FLV - if (dataSize <= 5) { - Log.w(this.TAG, 'Flv: Invalid audio packet, missing AudioFourCC in Ehnanced FLV payload!'); - return; + let meta = this._audioMetadata; + let track = this._audioTrack; + + if (!meta) { + if (this._hasAudio === false && this._hasAudioFlagOverrided === false) { + this._hasAudio = true; + this._mediaInfo.hasAudio = true; + } + + // initial metadata + meta = this._audioMetadata = {}; + meta.type = 'audio'; + meta.id = track.id; + meta.timescale = this._timescale; + meta.duration = this._duration; } + + let packetType = soundSpec & 0x0F; - let fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(1, 5)); + if (packetType === 5) { // AudioPacketType.Multitrack + if (dataSize <= 1) { + Log.w(this.TAG, 'Flv: Invalid audio packet, missing audioMultitrackType in Ehnanced FLV payload!'); + return; + } + this._enhancedFlvAudioMultitrackMode = (v.getUint8(1) & 0xF0) >> 4; + let packetType = v.getUint8(1) & 0x0F; - switch(fourcc){ - case 'Opus': - this._parseOpusAudioPacket(arrayBuffer, dataOffset + 5, dataSize - 5, tagTimestamp, packetType); - break; - case 'fLaC': - this._parseFlacAudioPacket(arrayBuffer, dataOffset + 5, dataSize - 5, tagTimestamp, packetType); - break; - default: - this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec: ' + fourcc); + let fourcc = null; + if (this._enhancedFlvAudioMultitrackMode !== 2) { // not AvMultitrackType.ManyTracksManyCodecs + if (dataSize <= 6) { + Log.w(this.TAG, 'Flv: Invalid audio packet, missing Audio fourcc of OneTrack/ManyTracks in Ehnanced FLV payload!'); + return; + } + fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(2, 6)); + this._parseEnhanedAudioPacket(arrayBuffer, dataOffset + 6, dataSize - 6, tagTimestamp, fourcc, packetType); + return; + } else { + this._parseEnhanedAudioPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, fourcc, packetType); + return; + } + } + + // other Enhanced Audio Payload + if (dataSize <= 5) { + Log.w(this.TAG, 'Flv: Invalid audio packet, missing AudioFourCC in Ehnanced Single Track FLV payload!'); + return; } + let fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(1, 5)); + this._parseEnhanedAudioPacket(arrayBuffer, dataOffset + 5, dataSize - 5, tagTimestamp, fourcc, packetType); return; } + // Legacy FLV + if (this._currentAudioTrackId != null) { + return; + } if (soundFormat !== 2 && soundFormat !== 10) { // MP3 or AAC this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec idx: ' + soundFormat); @@ -553,130 +641,94 @@ class FLVDemuxer { meta.channelCount = (soundType === 0 ? 1 : 2); } + let packetType = v.getUint8(1); if (soundFormat === 10) { // AAC - let aacData = this._parseAACAudioData(arrayBuffer, dataOffset + 1, dataSize - 1); - if (aacData == undefined) { + this._parseAACAudioPacket(arrayBuffer, dataOffset + 2, dataSize - 2, tagTimestamp, packetType, null); + } else if (soundFormat === 2) { // MP3 + this._parseMP3AudioPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, packetType, null); + } + } + + _parseAACAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { + let aacData = this._parseAACAudioData(arrayBuffer, dataOffset, dataSize, packetType); + if (aacData == undefined) { + return; + } + + let meta = { ... this._audioMetadata }; + let track = this._audioTrack; + + if (packetType === 0) { // AAC sequence header (AudioSpecificConfig) + if (meta.config) { + if (this._currentAudioTrackId === trackId && buffersAreEqual(aacData.data.config, meta.config)) { + // If AudioSpecificConfig is not changed, ignore it to avoid generating initialization segment repeatedly + return; + } else if (this._currentAudioTrackId === trackId && !buffersAreEqual(aacData.data.config, meta.config)) { + Log.w(this.TAG, 'AudioSpecificConfig has been changed, re-generate initialization segment'); + } + } + let misc = aacData.data; + meta.audioSampleRate = misc.samplingRate; + meta.channelCount = misc.channelCount; + meta.codec = misc.codec; + meta.originalCodec = misc.originalCodec; + meta.config = misc.config; + // The decode result of an aac sample is 1024 PCM samples + meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; + Log.v(this.TAG, 'Parsed AudioSpecificConfig'); + + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { return; } - if (aacData.packetType === 0) { // AAC sequence header (AudioSpecificConfig) - if (meta.config) { - if (buffersAreEqual(aacData.data.config, meta.config)) { - // If AudioSpecificConfig is not changed, ignore it to avoid generating initialization segment repeatedly - return; - } else { - Log.w(this.TAG, 'AudioSpecificConfig has been changed, re-generate initialization segment'); - } - } - let misc = aacData.data; - meta.audioSampleRate = misc.samplingRate; - meta.channelCount = misc.channelCount; - meta.codec = misc.codec; - meta.originalCodec = misc.originalCodec; - meta.config = misc.config; - // The decode result of an aac sample is 1024 PCM samples - meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; - Log.v(this.TAG, 'Parsed AudioSpecificConfig'); - - if (this._isInitialMetadataDispatched()) { - // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer - if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { - this._onDataAvailable(this._audioTrack, this._videoTrack); - } - } else { - this._audioInitialMetadataDispatched = true; - } - // then notify new metadata - this._dispatch = false; - this._onTrackMetadata('audio', meta); - - let mi = this._mediaInfo; - mi.audioCodec = meta.originalCodec; - mi.audioSampleRate = meta.audioSampleRate; - mi.audioChannelCount = meta.channelCount; - if (mi.hasVideo) { - if (mi.videoCodec != null) { - mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; - } - } else { - mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + if (this._isInitialMetadataDispatched()) { + // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer + if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { + this._onDataAvailable(this._audioTrack, this._videoTrack); } - if (mi.isComplete()) { - this._onMediaInfo(mi); - } - } else if (aacData.packetType === 1) { // AAC raw frame data - let dts = this._timestampBase + tagTimestamp; - let aacSample = {unit: aacData.data, length: aacData.data.byteLength, dts: dts, pts: dts}; - track.samples.push(aacSample); - track.length += aacData.data.length; } else { - Log.e(this.TAG, `Flv: Unsupported AAC data type ${aacData.packetType}`); + this._audioInitialMetadataDispatched = true; } - } else if (soundFormat === 2) { // MP3 - if (!meta.codec) { - // We need metadata for mp3 audio track, extract info from frame header - let misc = this._parseMP3AudioData(arrayBuffer, dataOffset + 1, dataSize - 1, true); - if (misc == undefined) { - return; - } - meta.audioSampleRate = misc.samplingRate; - meta.channelCount = misc.channelCount; - meta.codec = misc.codec; - meta.originalCodec = misc.originalCodec; - // The decode result of an mp3 sample is 1152 PCM samples - meta.refSampleDuration = 1152 / meta.audioSampleRate * meta.timescale; - Log.v(this.TAG, 'Parsed MPEG Audio Frame Header'); + // then notify new metadata + this._dispatch = false; + this._audioMetadata = meta; + this._onTrackMetadata('audio', meta); - this._audioInitialMetadataDispatched = true; - this._onTrackMetadata('audio', meta); - - let mi = this._mediaInfo; - mi.audioCodec = meta.codec; - mi.audioSampleRate = meta.audioSampleRate; - mi.audioChannelCount = meta.channelCount; - mi.audioDataRate = misc.bitRate; - if (mi.hasVideo) { - if (mi.videoCodec != null) { - mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; - } - } else { - mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; - } - if (mi.isComplete()) { - this._onMediaInfo(mi); + let mi = this._mediaInfo; + mi.audioCodec = meta.originalCodec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; } - - // This packet is always a valid audio packet, extract it - let data = this._parseMP3AudioData(arrayBuffer, dataOffset + 1, dataSize - 1, false); - if (data == undefined) { - return; + if (mi.isComplete()) { + this._onMediaInfo(mi); } + } else if (packetType === 1) { // AAC raw frame data + // ignore not current track + if (this._currentAudioTrackId !== trackId) { return; } + let dts = this._timestampBase + tagTimestamp; - let mp3Sample = {unit: data, length: data.byteLength, dts: dts, pts: dts}; - track.samples.push(mp3Sample); - track.length += data.length; + let aacSample = {unit: aacData.data, length: aacData.data.byteLength, dts: dts, pts: dts}; + track.samples.push(aacSample); + track.length += aacData.data.length; + } else { + Log.e(this.TAG, `Flv: Unsupported AAC data type ${packetType}`); } } - _parseAACAudioData(arrayBuffer, dataOffset, dataSize) { - if (dataSize <= 1) { - Log.w(this.TAG, 'Flv: Invalid AAC packet, missing AACPacketType or/and Data!'); - return; - } - - let result = {}; - let array = new Uint8Array(arrayBuffer, dataOffset, dataSize); - - result.packetType = array[0]; - - if (array[0] === 0) { - result.data = this._parseAACAudioSpecificConfig(arrayBuffer, dataOffset + 1, dataSize - 1); + _parseAACAudioData(arrayBuffer, dataOffset, dataSize, packetType) { + if (packetType === 0) { + return { data: this._parseAACAudioSpecificConfig(arrayBuffer, dataOffset, dataSize) }; } else { - result.data = array.subarray(1); + return { data: new Uint8Array(arrayBuffer, dataOffset, dataSize) } } - - return result; } _parseAACAudioSpecificConfig(arrayBuffer, dataOffset, dataSize) { @@ -780,7 +832,63 @@ class FLVDemuxer { }; } - _parseMP3AudioData(arrayBuffer, dataOffset, dataSize, requestHeader) { + _parseMP3AudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { + let meta = { ... this._audioMetadata }; + let track = this._audioTrack; + + if (!meta.codec) { + // We need metadata for mp3 audio track, extract info from frame header + let misc = this._parseMP3AudioData(arrayBuffer, dataOffset, dataSize, packetType, true); + if (misc == undefined) { + return; + } + meta.audioSampleRate = misc.samplingRate; + meta.channelCount = misc.channelCount; + meta.codec = misc.codec; + meta.originalCodec = misc.originalCodec; + // The decode result of an mp3 sample is 1152 PCM samples + meta.refSampleDuration = 1152 / meta.audioSampleRate * meta.timescale; + Log.v(this.TAG, 'Parsed MPEG Audio Frame Header'); + + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + + this._audioInitialMetadataDispatched = true; + this._audioMetadata = meta; + this._onTrackMetadata('audio', meta); + + let mi = this._mediaInfo; + mi.audioCodec = meta.codec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + mi.audioDataRate = misc.bitRate; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; + } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + } + if (mi.isComplete()) { + this._onMediaInfo(mi); + } + } + + // This packet is always a valid audio packet, extract it + let data = this._parseMP3AudioData(arrayBuffer, dataOffset, dataSize, packetType, false); + if (data == undefined) { + return; + } + let dts = this._timestampBase + tagTimestamp; + let mp3Sample = {unit: data, length: data.byteLength, dts: dts, pts: dts}; + track.samples.push(mp3Sample); + track.length += data.length; + } + + _parseMP3AudioData(arrayBuffer, dataOffset, dataSize, packetType, requestHeader) { if (dataSize < 4) { Log.w(this.TAG, 'Flv: Invalid MP3 packet, header missing!'); return; @@ -856,10 +964,66 @@ class FLVDemuxer { return result; } - _parseOpusAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + _parseEnhanedAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, fourcc, packetType) { + let v = new DataView(arrayBuffer, dataOffset, dataSize); + let enhanced_offset = 0, enhanced_datasize = dataSize; + while (enhanced_offset < dataSize) { + if (this._enhancedFlvAudioMultitrackMode === 2) { // is not MultiTrackMultiCodec + if (enhanced_offset + 4 >= dataSize) { + Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, fourcc in ManyTrackManyCodec Missing!'); + return; + } + fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(enhanced_offset + 0, enhanced_offset + 4)); + enhanced_offset += 4; + } + + let trackId = null; + if (this._enhancedFlvAudioMultitrackMode != null) { + if (enhanced_offset + 1 >= dataSize) { + Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, TrackId for MultiTrack Audio Missing!'); + return; + } + trackId = v.getUint8(enhanced_offset); + enhanced_offset += 1; + + if (!this._enhanedFlvAudioTrackIds.includes(trackId)) { + let newTrackIds = [... this._enhanedFlvAudioTrackIds, trackId]; + this._updateAudioTrackIds(this._currentAudioTrackIndex, newTrackIds); + } + } + + if (this._enhancedFlvAudioMultitrackMode != null && this._enhancedFlvAudioMultitrackMode !== 0) { // has ManyTrack + if (enhanced_offset + 3 >= dataSize) { + Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, DataSize for MultiTrack Audio Missing!'); + return; + } + enhanced_datasize = (v.getUint8(enhanced_offset + 0) << 16) | (v.getUint8(enhanced_offset + 1) << 8) | (v.getUint8(enhanced_offset + 2) << 0); + enhanced_offset += 3; + } else { + enhanced_datasize = dataSize - enhanced_offset; + } + + if (fourcc === 'mp4a') { + this._parseAACAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); + } else if (fourcc === '.mp3') { + this._parseMP3AudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); + } else if (fourcc === 'Opus') { + this._parseOpusAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); + } else if (fourcc === 'fLaC') { + this._parseFlacAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); + } else { + this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec: ' + fourcc); + } + + enhanced_offset += enhanced_datasize; + } + } + + _parseOpusAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { if (packetType === 0) { // OpusSequenceHeader - this._parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize); + this._parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId); } else if (packetType === 1) { // OpusCodedData + if (this._currentAudioTrackId !== trackId) { return; } this._parseOpusAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp); } else if (packetType === 2) { // empty, Opus end of sequence @@ -869,27 +1033,12 @@ class FLVDemuxer { } } - _parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize) { + _parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId) { if (dataSize <= 16) { Log.w(this.TAG, 'Flv: Invalid OpusSequenceHeader, lack of data!'); return; } - let meta = this._audioMetadata; - let track = this._audioTrack; - - if (!meta) { - if (this._hasAudio === false && this._hasAudioFlagOverrided === false) { - this._hasAudio = true; - this._mediaInfo.hasAudio = true; - } - - // initial metadata - meta = this._audioMetadata = {}; - meta.type = 'audio'; - meta.id = track.id; - meta.timescale = this._timescale; - meta.duration = this._duration; - } + let meta = { ... this._audioMetadata }; // Identification Header let v = new DataView(arrayBuffer, dataOffset, dataSize); @@ -908,10 +1057,10 @@ class FLVDemuxer { originalCodec: 'opus', }; if (meta.config) { - if (buffersAreEqual(misc.config, meta.config)) { + if (this._currentAudioTrackId === trackId && buffersAreEqual(misc.config, meta.config)) { // If OpusSequenceHeader is not changed, ignore it to avoid generating initialization segment repeatedly return; - } else { + } else if (this._currentAudioTrackId === trackId && !buffersAreEqual(misc.config, meta.config)) { Log.w(this.TAG, 'OpusSequenceHeader has been changed, re-generate initialization segment'); } } @@ -924,6 +1073,12 @@ class FLVDemuxer { meta.refSampleDuration = 20; Log.v(this.TAG, 'Parsed OpusSequenceHeader'); + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + if (this._isInitialMetadataDispatched()) { // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { @@ -934,6 +1089,7 @@ class FLVDemuxer { } // then notify new metadata this._dispatch = false; + this._audioMetadata = meta; this._onTrackMetadata('audio', meta); let mi = this._mediaInfo; @@ -963,10 +1119,11 @@ class FLVDemuxer { track.length += data.length; } - _parseFlacAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + _parseFlacAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { if (packetType === 0) { // FlacSequenceHeader - this._parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize); + this._parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId); } else if (packetType === 1) { // FlacCodedData + if (this._currentAudioTrackId !== trackId) { return; } this._parseFlacAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp); } else if (packetType === 2) { // empty, Flac end of sequence @@ -976,8 +1133,8 @@ class FLVDemuxer { } } - _parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize) { - let meta = this._audioMetadata; + _parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId) { + let meta = { ... this._audioMetadata }; let track = this._audioTrack; if (!meta) { @@ -995,7 +1152,7 @@ class FLVDemuxer { } // METADATA_BLOCK_HEADER - let header = new Uint8Array(arrayBuffer, dataOffset + 4, dataSize - 4); + let header = new Uint8Array(arrayBuffer, dataOffset, dataSize); let gb = new ExpGolomb(header); let minimum_block_size = gb.readBits(16); // minimum_block_size let maximum_block_size = gb.readBits(16); // maximum_block_size @@ -1023,10 +1180,10 @@ class FLVDemuxer { originalCodec: 'flac', }; if (meta.config) { - if (buffersAreEqual(misc.config, meta.config)) { + if (this._currentAudioTrackId === trackId && buffersAreEqual(misc.config, meta.config)) { // If FlacSequenceHeader is not changed, ignore it to avoid generating initialization segment repeatedly return; - } else { + } else if (this._currentAudioTrackId === trackId && !buffersAreEqual(misc.config, meta.config)) { Log.w(this.TAG, 'FlacSequenceHeader has been changed, re-generate initialization segment'); } } @@ -1040,6 +1197,12 @@ class FLVDemuxer { Log.v(this.TAG, 'Parsed FlacSequenceHeader'); + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + if (this._isInitialMetadataDispatched()) { // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { @@ -1050,6 +1213,7 @@ class FLVDemuxer { } // then notify new metadata this._dispatch = false; + this._audioMetadata = meta; this._onTrackMetadata('audio', meta); let mi = this._mediaInfo; diff --git a/src/demux/pat-pmt-pes.ts b/src/demux/pat-pmt-pes.ts index b1c0974f..5eac1410 100644 --- a/src/demux/pat-pmt-pes.ts +++ b/src/demux/pat-pmt-pes.ts @@ -37,23 +37,19 @@ export class PMT { common_pids: { h264: number | undefined, h265: number | undefined; - adts_aac: number | undefined, - loas_aac: number | undefined, - opus: number | undefined, - ac3: number | undefined, - eac3: number | undefined, - mp3: number | undefined } = { h264: undefined, h265: undefined, - adts_aac: undefined, - loas_aac: undefined, - opus: undefined, - ac3: undefined, - eac3: undefined, - mp3: undefined }; + audio_pids: { + [pid: number]: boolean + } = {}; + + opus_pids: { + [pid: number]: boolean; + } = {}; + pes_private_data_pids: { [pid: number]: boolean } = {}; diff --git a/src/demux/ts-demuxer.ts b/src/demux/ts-demuxer.ts index 74431eea..c4952d87 100644 --- a/src/demux/ts-demuxer.ts +++ b/src/demux/ts-demuxer.ts @@ -75,6 +75,7 @@ type MP3AudioMetadata = { sample_rate: number, channel_count: number; }; +type AudioMetadata = AACAudioMetadata | AC3AudioMetadata | EAC3AudioMetadata | OpusAudioMetadata | MP3AudioMetadata; type AudioData = { codec: 'aac'; data: AACFrame; @@ -112,6 +113,12 @@ class TSDemuxer extends BaseDemuxer { private pmt_: PMT; private program_pmt_map_: ProgramToPMTMap = {}; + private audioTrackPids: number[] = []; + private currentAudioTrackIndex = 0; + private currentAudioTrackPid = null; + private audioTrackInitSegments = new Map(); + private audioTrackMetadata = new Map(); + private pes_slice_queues_: PIDToSliceQueues = {}; private section_slice_queues_: PIDToSliceQueues = {}; @@ -127,7 +134,7 @@ class TSDemuxer extends BaseDemuxer { details: undefined }; - private audio_metadata_: AACAudioMetadata | AC3AudioMetadata | EAC3AudioMetadata | OpusAudioMetadata | MP3AudioMetadata = { + private audio_metadata_: AudioMetadata = { codec: undefined, audio_object_type: undefined, sampling_freq_index: undefined, @@ -244,6 +251,37 @@ class TSDemuxer extends BaseDemuxer { this.media_info_ = new MediaInfo(); } + selectAudioTrack(track: number) { + this.currentAudioTrackIndex = track; + let previous_meta = this.audioTrackMetadata.get(this.currentAudioTrackPid); + let newTrackPid = this.audioTrackPids[track]; + if (this.currentAudioTrackPid === newTrackPid) { return; } + + this.currentAudioTrackPid = newTrackPid; + let sample = this.audioTrackInitSegments.get(this.currentAudioTrackPid); + if (!sample) { return; } + let meta = this.audioTrackMetadata.get(this.currentAudioTrackPid); + if (!meta) { return; } + + // flush segment + this.dispatchAudioMediaSegment(); + this.video_track_ = {type: 'video', id: 1, sequenceNumber: 0, samples: [], length: 0}; + this.audio_track_ = {type: 'audio', id: 2, sequenceNumber: 0, samples: [], length: 0}; + + // then notify new metadata + if (previous_meta == null || this.detectAudioMetadataChange(sample, previous_meta, this.currentAudioTrackPid)) { + this.dispatchAudioInitSegment(sample, meta, this.currentAudioTrackPid); + } + } + + _updateAudioTrackIds(trackIndex: number, newTrackIds: number[]) { + this.audioTrackPids = newTrackIds; + let newTrackPid = this.audioTrackPids[trackIndex]; + if (this.currentAudioTrackPid !== newTrackPid) { + this.selectAudioTrack(trackIndex); + } + } + public parseChunks(chunk: ArrayBuffer, byte_start: number): number { if (!this.onError || !this.onMediaInfo @@ -340,12 +378,8 @@ class TSDemuxer extends BaseDemuxer { // process PES only for known common_pids if (pid === this.pmt_.common_pids.h264 || pid === this.pmt_.common_pids.h265 - || pid === this.pmt_.common_pids.adts_aac - || pid === this.pmt_.common_pids.loas_aac - || pid === this.pmt_.common_pids.ac3 - || pid === this.pmt_.common_pids.eac3 - || pid === this.pmt_.common_pids.opus - || pid === this.pmt_.common_pids.mp3 + || this.pmt_.audio_pids[pid] === true + || this.pmt_.opus_pids[pid] === true || this.pmt_.pes_private_data_pids[pid] === true || this.pmt_.timed_id3_pids[pid] === true || this.pmt_.synchronous_klv_pids[pid] === true @@ -594,16 +628,16 @@ class TSDemuxer extends BaseDemuxer { switch (pes_data.stream_type) { case StreamType.kMPEG1Audio: case StreamType.kMPEG2Audio: - this.parseMP3Payload(payload, pts); + this.parseMP3Payload(payload, pts, pes_data.pid); break; case StreamType.kPESPrivateData: - if (this.pmt_.common_pids.opus === pes_data.pid) { - this.parseOpusPayload(payload, pts); - } else if (this.pmt_.common_pids.ac3 === pes_data.pid) { - this.parseAC3Payload(payload, pts); + if (this.pmt_.opus_pids[pes_data.pid]) { + this.parseOpusPayload(payload, pts, pes_data.pid); + } /*else if (this.pmt_.common_pids.ac3 === pes_data.pid) { + this.parseAC3Payload(payload, pts, pes_data.pid); } else if (this.pmt_.common_pids.eac3 === pes_data.pid) { - this.parseEAC3Payload(payload, pts); - } else if (this.pmt_.asynchronous_klv_pids[pes_data.pid]) { + this.parseEAC3Payload(payload, pts, pes_data.pid); + } */else if (this.pmt_.asynchronous_klv_pids[pes_data.pid]) { this.parseAsynchronousKLVMetadataPayload(payload, pes_data.pid, stream_id); } else if (this.pmt_.smpte2038_pids[pes_data.pid]) { this.parseSMPTE2038MetadataPayload(payload, pts, dts, pes_data.pid, stream_id); @@ -612,16 +646,16 @@ class TSDemuxer extends BaseDemuxer { } break; case StreamType.kADTSAAC: - this.parseADTSAACPayload(payload, pts); + this.parseADTSAACPayload(payload, pts, pes_data.pid); break; case StreamType.kLOASAAC: - this.parseLOASAACPayload(payload, pts); + this.parseLOASAACPayload(payload, pts, pes_data.pid); break; case StreamType.kAC3: - this.parseAC3Payload(payload, pts); + this.parseAC3Payload(payload, pts, pes_data.pid); break; case StreamType.kEAC3: - this.parseEAC3Payload(payload, pts); + this.parseEAC3Payload(payload, pts, pes_data.pid); break; case StreamType.kMetadata: if (this.pmt_.timed_id3_pids[pes_data.pid]) { @@ -760,6 +794,8 @@ class TSDemuxer extends BaseDemuxer { let info_start_index = 12 + program_info_length; let info_bytes = section_length - 9 - program_info_length - 4; + let audioPids = []; + let hasVideo = false, hasAudio = false; for (let i = info_start_index; i < info_start_index + info_bytes; ) { let stream_type = data[i] as StreamType; @@ -768,23 +804,34 @@ class TSDemuxer extends BaseDemuxer { pmt.pid_stream_type[elementary_PID] = stream_type; - let already_has_video = pmt.common_pids.h264 || pmt.common_pids.h265; - let already_has_audio = pmt.common_pids.adts_aac || pmt.common_pids.loas_aac || pmt.common_pids.ac3 || pmt.common_pids.eac3 || pmt.common_pids.opus || pmt.common_pids.mp3; + let already_has_video = pmt.common_pids.h264 || pmt.common_pids.h265; if (stream_type === StreamType.kH264 && !already_has_video) { pmt.common_pids.h264 = elementary_PID; + hasVideo = true; } else if (stream_type === StreamType.kH265 && !already_has_video) { pmt.common_pids.h265 = elementary_PID; - } else if (stream_type === StreamType.kADTSAAC && !already_has_audio) { - pmt.common_pids.adts_aac = elementary_PID; - } else if (stream_type === StreamType.kLOASAAC && !already_has_audio) { - pmt.common_pids.loas_aac = elementary_PID; - } else if (stream_type === StreamType.kAC3 && !already_has_audio) { - pmt.common_pids.ac3 = elementary_PID; // ATSC AC-3 - } else if (stream_type === StreamType.kEAC3 && !already_has_audio) { - pmt.common_pids.eac3 = elementary_PID; // ATSC EAC-3 - } else if ((stream_type === StreamType.kMPEG1Audio || stream_type === StreamType.kMPEG2Audio) && !already_has_audio) { - pmt.common_pids.mp3 = elementary_PID; + hasVideo = true; + } else if (stream_type === StreamType.kADTSAAC) { + pmt.audio_pids[elementary_PID] = true; // ADTS AAC + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kLOASAAC) { + pmt.audio_pids[elementary_PID] = true; // LOAS AAC + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kAC3) { + pmt.audio_pids[elementary_PID] = true; // ATSC AC-3 + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kEAC3) { + pmt.audio_pids[elementary_PID] = true; // ATSC EAC-3 + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kMPEG1Audio || stream_type === StreamType.kMPEG2Audio) { + pmt.audio_pids[elementary_PID] = true; // MP3 + audioPids.push(elementary_PID); + hasAudio = true; } else if (stream_type === StreamType.kPESPrivateData) { pmt.pes_private_data_pids[elementary_PID] = true; if (ES_info_length > 0) { @@ -802,12 +849,16 @@ class TSDemuxer extends BaseDemuxer { } */ /* else if (registration === 'EC-3' && !alrady_has_audio) { pmt.common_pids.eac3 = elementary_PID; // DVB EAC-3 (FIXME: NEED VERIFY) } */ else if (registration === 'Opus') { - pmt.common_pids.opus = elementary_PID; + pmt.opus_pids[elementary_PID] = true; + audioPids.push(elementary_PID); + hasAudio = true; } else if (registration === 'KLVA') { pmt.asynchronous_klv_pids[elementary_PID] = true; } } else if (tag === 0x7F) { // DVB extension descriptor - if (elementary_PID === pmt.common_pids.opus) { + if (pmt.opus_pids[elementary_PID]) { + let audio_metadata_ = this.audioTrackMetadata.get(elementary_PID); + let ext_desc_tag = data[offset + 2]; let channel_config_code: number | null = null; if (ext_desc_tag === 0x80) { // User defined (provisional Opus) @@ -830,14 +881,16 @@ class TSDemuxer extends BaseDemuxer { meta } as const; - if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = meta; - this.dispatchAudioInitSegment(sample); - } else if (this.detectAudioMetadataChange(sample)) { + if (this.currentAudioTrackPid !== elementary_PID) { + this.audioTrackInitSegments.set(elementary_PID, { ... sample }); + this.audioTrackMetadata.set(elementary_PID, { ... meta }); + } else if (this.audio_init_segment_dispatched_ == false) { + this.dispatchAudioInitSegment(sample, meta, elementary_PID); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(sample, audio_metadata_, elementary_PID)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(sample); + this.dispatchAudioInitSegment(sample, meta, elementary_PID); } } } @@ -884,18 +937,15 @@ class TSDemuxer extends BaseDemuxer { i += 5 + ES_info_length; } + this._updateAudioTrackIds(this.currentAudioTrackIndex, audioPids); if (program_number === this.current_program_) { if (this.pmt_ == undefined) { Log.v(this.TAG, `Parsed first PMT: ${JSON.stringify(pmt)}`); } this.pmt_ = pmt; - if (pmt.common_pids.h264 || pmt.common_pids.h265) { - this.has_video_ = true; - } - if (pmt.common_pids.adts_aac || pmt.common_pids.loas_aac || pmt.common_pids.ac3 || pmt.common_pids.opus || pmt.common_pids.mp3) { - this.has_audio_ = true; - } + this.has_video_ = hasVideo; + this.has_audio_ = hasAudio; } } @@ -1198,7 +1248,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseADTSAACPayload(data: Uint8Array, pts: number) { + private parseADTSAACPayload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1212,15 +1262,17 @@ class TSDemuxer extends BaseDemuxer { data = buf; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + let ref_sample_duration: number; let base_pts_ms: number; if (pts != undefined) { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'aac') { + if (audio_metadata_ && audio_metadata_.codec === 'aac') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `AAC: Unknown pts`); @@ -1228,7 +1280,7 @@ class TSDemuxer extends BaseDemuxer { } if (this.aac_last_incomplete_data_ && this.audio_last_sample_pts_) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; let new_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; if (Math.abs(new_pts_ms - base_pts_ms) > 1) { @@ -1245,26 +1297,33 @@ class TSDemuxer extends BaseDemuxer { let last_sample_pts_ms: number; while ((aac_frame = adts_parser.readNextAACFrame()) != null) { + const meta = { + codec: 'aac', + audio_object_type: aac_frame.audio_object_type, + sampling_freq_index: aac_frame.sampling_freq_index, + sampling_frequency: aac_frame.sampling_frequency, + channel_config: aac_frame.channel_config + }; + ref_sample_duration = 1024 / aac_frame.sampling_frequency * 1000; const audio_sample = { codec: 'aac', data: aac_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + continue; + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'aac', - audio_object_type: aac_frame.audio_object_type, - sampling_freq_index: aac_frame.sampling_freq_index, - sampling_frequency: aac_frame.sampling_frequency, - channel_config: aac_frame.channel_config - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1291,7 +1350,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseLOASAACPayload(data: Uint8Array, pts: number) { + private parseLOASAACPayload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1305,15 +1364,17 @@ class TSDemuxer extends BaseDemuxer { data = buf; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + let ref_sample_duration: number; let base_pts_ms: number; if (pts != undefined) { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'aac') { + if (audio_metadata_ && audio_metadata_.codec === 'aac') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `AAC: Unknown pts`); @@ -1321,7 +1382,7 @@ class TSDemuxer extends BaseDemuxer { } if (this.aac_last_incomplete_data_ && this.audio_last_sample_pts_) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; let new_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; if (Math.abs(new_pts_ms - base_pts_ms) > 1) { @@ -1338,6 +1399,14 @@ class TSDemuxer extends BaseDemuxer { let last_sample_pts_ms: number; while ((aac_frame = loas_parser.readNextAACFrame(this.loas_previous_frame ?? undefined)) != null) { + const meta = { + codec: 'aac', + audio_object_type: aac_frame.audio_object_type, + sampling_freq_index: aac_frame.sampling_freq_index, + sampling_frequency: aac_frame.sampling_frequency, + channel_config: aac_frame.channel_config + }; + this.loas_previous_frame = aac_frame; ref_sample_duration = 1024 / aac_frame.sampling_frequency * 1000; const audio_sample = { @@ -1345,20 +1414,19 @@ class TSDemuxer extends BaseDemuxer { data: aac_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + continue; + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'aac', - audio_object_type: aac_frame.audio_object_type, - sampling_freq_index: aac_frame.sampling_freq_index, - sampling_frequency: aac_frame.sampling_frequency, - channel_config: aac_frame.channel_config - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1385,13 +1453,15 @@ class TSDemuxer extends BaseDemuxer { } } - private parseAC3Payload(data: Uint8Array, pts: number) { + private parseAC3Payload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched return; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + let ref_sample_duration: number; let base_pts_ms: number; @@ -1399,9 +1469,9 @@ class TSDemuxer extends BaseDemuxer { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'ac-3') { + if (audio_metadata_ && audio_metadata_.codec === 'ac-3') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 1536 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1536 / audio_metadata_.sampling_frequency * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `AC3: Unknown pts`); @@ -1415,27 +1485,34 @@ class TSDemuxer extends BaseDemuxer { let last_sample_pts_ms: number; while ((ac3_frame = adts_parser.readNextAC3Frame()) != null) { + const meta = { + codec: 'ac-3', + sampling_frequency: ac3_frame.sampling_frequency, + bit_stream_identification: ac3_frame.bit_stream_identification, + bit_stream_mode: ac3_frame.bit_stream_mode, + low_frequency_effects_channel_on: ac3_frame.low_frequency_effects_channel_on, + channel_mode: ac3_frame.channel_mode, + }; + ref_sample_duration = 1536 / ac3_frame.sampling_frequency * 1000; const audio_sample = { codec: 'ac-3', data: ac3_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + continue; + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'ac-3', - sampling_frequency: ac3_frame.sampling_frequency, - bit_stream_identification: ac3_frame.bit_stream_identification, - bit_stream_mode: ac3_frame.bit_stream_mode, - low_frequency_effects_channel_on: ac3_frame.low_frequency_effects_channel_on, - channel_mode: ac3_frame.channel_mode, - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1459,7 +1536,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseEAC3Payload(data: Uint8Array, pts: number) { + private parseEAC3Payload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1469,47 +1546,54 @@ class TSDemuxer extends BaseDemuxer { let ref_sample_duration: number; let base_pts_ms: number; - if (pts != undefined) { - base_pts_ms = pts / this.timescale_; - } + let audio_metadata_ = this.audioTrackMetadata.get(pid); - if (this.audio_metadata_.codec === 'ec-3') { + let adts_parser = new EAC3Parser(data); + let eac3_frame: EAC3Frame = null; + let sample_pts_ms = base_pts_ms; + let last_sample_pts_ms: number; + + if (audio_metadata_ && audio_metadata_.codec === 'ec-3') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = (256 * this.audio_metadata_.num_blks) / this.audio_metadata_.sampling_frequency * 1000; // TODO: AEC3 BLK + ref_sample_duration = (256 * audio_metadata_.num_blks) / audio_metadata_.sampling_frequency * 1000; // TODO: AEC3 BLK base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `EAC3: Unknown pts`); return; } } - - let adts_parser = new EAC3Parser(data); - let eac3_frame: EAC3Frame = null; - let sample_pts_ms = base_pts_ms; - let last_sample_pts_ms: number; + if (pts != undefined) { + base_pts_ms = pts / this.timescale_; + } while ((eac3_frame = adts_parser.readNextEAC3Frame()) != null) { + const meta = { + codec: 'ec-3', + sampling_frequency: eac3_frame.sampling_frequency, + bit_stream_identification: eac3_frame.bit_stream_identification, + low_frequency_effects_channel_on: eac3_frame.low_frequency_effects_channel_on, + num_blks: eac3_frame.num_blks, + channel_mode: eac3_frame.channel_mode, + }; + ref_sample_duration = 1536 / eac3_frame.sampling_frequency * 1000; // TODO: EAC3 BLK const audio_sample = { codec: 'ec-3', data: eac3_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'ec-3', - sampling_frequency: eac3_frame.sampling_frequency, - bit_stream_identification: eac3_frame.bit_stream_identification, - low_frequency_effects_channel_on: eac3_frame.low_frequency_effects_channel_on, - num_blks: eac3_frame.num_blks, - channel_mode: eac3_frame.channel_mode, - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1533,12 +1617,16 @@ class TSDemuxer extends BaseDemuxer { } } - private parseOpusPayload(data: Uint8Array, pts: number) { + private parseOpusPayload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched return; } + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, discard it + return; + } let ref_sample_duration: number; let base_pts_ms: number; @@ -1546,14 +1634,13 @@ class TSDemuxer extends BaseDemuxer { if (pts != undefined) { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'opus') { - if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 20; - base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; - } else if (pts == undefined){ - Log.w(this.TAG, `Opus: Unknown pts`); - return; - } + + if (pts == undefined && this.audio_last_sample_pts_ != undefined) { + ref_sample_duration = 20; + base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; + } else if (pts == undefined){ + Log.w(this.TAG, `Opus: Unknown pts`); + return; } let sample_pts_ms = base_pts_ms; @@ -1568,8 +1655,8 @@ class TSDemuxer extends BaseDemuxer { let size = 0; while (data[index] === 0xFF) { - size += 255; - index += 1; + size += 255; + index += 1; } size += data[index]; index += 1; @@ -1598,7 +1685,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseMP3Payload(data: Uint8Array, pts: number) { + private parseMP3Payload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1657,6 +1744,8 @@ class TSDemuxer extends BaseDemuxer { break; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + const sample = new MP3Data(); sample.object_type = object_type; sample.sample_rate = sample_rate; @@ -1667,20 +1756,26 @@ class TSDemuxer extends BaseDemuxer { data: sample } as const; + const meta = { + codec: 'mp3', + object_type, + sample_rate, + channel_count + }; + + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + return; + } if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'mp3', - object_type, - sample_rate, - channel_count - } - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); - // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + // notify new MP3 AudioSpecificConfig + this.dispatchAudioInitSegment(audio_sample, meta, pid); } let mp3_sample = { @@ -1693,94 +1788,98 @@ class TSDemuxer extends BaseDemuxer { this.audio_track_.length += data.byteLength; } - private detectAudioMetadataChange(sample: AudioData): boolean { - if (sample.codec !== this.audio_metadata_.codec) { + private detectAudioMetadataChange(sample: AudioData, previous_audio_metadata_: any, pid: number): boolean { + if (this.currentAudioTrackPid !== pid) { + return false; + } + + if (sample.codec !== previous_audio_metadata_.codec) { Log.v(this.TAG, `Audio: Audio Codecs changed from ` + - `${this.audio_metadata_.codec} to ${sample.codec}`); + `${previous_audio_metadata_.codec} to ${sample.codec}`); return true; } - if (sample.codec === 'aac' && this.audio_metadata_.codec === 'aac') { + if (sample.codec === 'aac' && previous_audio_metadata_.codec === 'aac') { const frame = sample.data; - if (frame.audio_object_type !== this.audio_metadata_.audio_object_type) { + if (frame.audio_object_type !== previous_audio_metadata_.audio_object_type) { Log.v(this.TAG, `AAC: AudioObjectType changed from ` + - `${this.audio_metadata_.audio_object_type} to ${frame.audio_object_type}`); + `${previous_audio_metadata_.audio_object_type} to ${frame.audio_object_type}`); return true; } - if (frame.sampling_freq_index !== this.audio_metadata_.sampling_freq_index) { + if (frame.sampling_freq_index !== previous_audio_metadata_.sampling_freq_index) { Log.v(this.TAG, `AAC: SamplingFrequencyIndex changed from ` + - `${this.audio_metadata_.sampling_freq_index} to ${frame.sampling_freq_index}`); + `${previous_audio_metadata_.sampling_freq_index} to ${frame.sampling_freq_index}`); return true; } - if (frame.channel_config !== this.audio_metadata_.channel_config) { + if (frame.channel_config !== previous_audio_metadata_.channel_config) { Log.v(this.TAG, `AAC: Channel configuration changed from ` + - `${this.audio_metadata_.channel_config} to ${frame.channel_config}`); + `${previous_audio_metadata_.channel_config} to ${frame.channel_config}`); return true; } - } else if (sample.codec === 'ac-3' && this.audio_metadata_.codec === 'ac-3') { + } else if (sample.codec === 'ac-3' && previous_audio_metadata_.codec === 'ac-3') { const frame = sample.data; - if (frame.sampling_frequency !== this.audio_metadata_.sampling_frequency) { + if (frame.sampling_frequency !== previous_audio_metadata_.sampling_frequency) { Log.v(this.TAG, `AC3: Sampling Frequency changed from ` + - `${this.audio_metadata_.sampling_frequency} to ${frame.sampling_frequency}`); + `${previous_audio_metadata_.sampling_frequency} to ${frame.sampling_frequency}`); return true; } - if (frame.bit_stream_identification !== this.audio_metadata_.bit_stream_identification) { + if (frame.bit_stream_identification !== previous_audio_metadata_.bit_stream_identification) { Log.v(this.TAG, `AC3: Bit Stream Identification changed from ` + - `${this.audio_metadata_.bit_stream_identification} to ${frame.bit_stream_identification}`); + `${previous_audio_metadata_.bit_stream_identification} to ${frame.bit_stream_identification}`); return true; } - if (frame.bit_stream_mode !== this.audio_metadata_.bit_stream_mode) { + if (frame.bit_stream_mode !== previous_audio_metadata_.bit_stream_mode) { Log.v(this.TAG, `AC3: BitStream Mode changed from ` + - `${this.audio_metadata_.bit_stream_mode} to ${frame.bit_stream_mode}`); + `${previous_audio_metadata_.bit_stream_mode} to ${frame.bit_stream_mode}`); return true; } - if (frame.channel_mode !== this.audio_metadata_.channel_mode) { + if (frame.channel_mode !== previous_audio_metadata_.channel_mode) { Log.v(this.TAG, `AC3: Channel Mode changed from ` + - `${this.audio_metadata_.channel_mode} to ${frame.channel_mode}`); + `${previous_audio_metadata_.channel_mode} to ${frame.channel_mode}`); return true; } - if (frame.low_frequency_effects_channel_on !== this.audio_metadata_.low_frequency_effects_channel_on) { + if (frame.low_frequency_effects_channel_on !== previous_audio_metadata_.low_frequency_effects_channel_on) { Log.v(this.TAG, `AC3: Low Frequency Effects Channel On changed from ` + - `${this.audio_metadata_.low_frequency_effects_channel_on} to ${frame.low_frequency_effects_channel_on}`); + `${previous_audio_metadata_.low_frequency_effects_channel_on} to ${frame.low_frequency_effects_channel_on}`); return true; } - } else if (sample.codec === 'opus' && this.audio_metadata_.codec === 'opus') { + } else if (sample.codec === 'opus' && previous_audio_metadata_.codec === 'opus') { const data = sample.meta; - if (data.sample_rate !== this.audio_metadata_.sample_rate) { + if (data.sample_rate !== previous_audio_metadata_.sample_rate) { Log.v(this.TAG, `Opus: SamplingFrequencyIndex changed from ` + - `${this.audio_metadata_.sample_rate} to ${data.sample_rate}`); + `${previous_audio_metadata_.sample_rate} to ${data.sample_rate}`); return true; } - if (data.channel_count !== this.audio_metadata_.channel_count) { + if (data.channel_count !== previous_audio_metadata_.channel_count) { Log.v(this.TAG, `Opus: Channel count changed from ` + - `${this.audio_metadata_.channel_count} to ${data.channel_count}`); + `${previous_audio_metadata_.channel_count} to ${data.channel_count}`); return true; } - } else if (sample.codec === 'mp3' && this.audio_metadata_.codec === 'mp3') { + } else if (sample.codec === 'mp3' && previous_audio_metadata_.codec === 'mp3') { const data = sample.data; - if (data.object_type !== this.audio_metadata_.object_type) { + if (data.object_type !== previous_audio_metadata_.object_type) { Log.v(this.TAG, `MP3: AudioObjectType changed from ` + - `${this.audio_metadata_.object_type} to ${data.object_type}`); + `${previous_audio_metadata_.object_type} to ${data.object_type}`); return true; } - if (data.sample_rate !== this.audio_metadata_.sample_rate) { + if (data.sample_rate !== previous_audio_metadata_.sample_rate) { Log.v(this.TAG, `MP3: SamplingFrequencyIndex changed from ` + - `${this.audio_metadata_.sample_rate} to ${data.sample_rate}`); + `${previous_audio_metadata_.sample_rate} to ${data.sample_rate}`); return true; } - if (data.channel_count !== this.audio_metadata_.channel_count) { + if (data.channel_count !== previous_audio_metadata_.channel_count) { Log.v(this.TAG, `MP3: Channel count changed from ` + - `${this.audio_metadata_.channel_count} to ${data.channel_count}`); + `${previous_audio_metadata_.channel_count} to ${data.channel_count}`); return true; } } @@ -1788,14 +1887,14 @@ class TSDemuxer extends BaseDemuxer { return false; } - private dispatchAudioInitSegment(sample: AudioData) { + private dispatchAudioInitSegment(sample: AudioData, audio_metadata_: any, pid: number) { let meta: any = {}; meta.type = 'audio'; meta.id = this.audio_track_.id; meta.timescale = 1000; meta.duration = this.duration_; - if (this.audio_metadata_.codec === 'aac') { + if (audio_metadata_.codec === 'aac') { let aac_frame = sample.codec === 'aac' ? sample.data : null; let audio_specific_config = new AudioSpecificConfig(aac_frame); @@ -1805,7 +1904,7 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = audio_specific_config.original_codec_mimetype; meta.config = audio_specific_config.config; meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; - } else if (this.audio_metadata_.codec === 'ac-3') { + } else if (audio_metadata_.codec === 'ac-3') { let ac3_frame = sample.codec === 'ac-3' ? sample.data : null; let ac3_config = new AC3Config(ac3_frame); meta.audioSampleRate = ac3_config.sampling_rate @@ -1814,7 +1913,7 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = ac3_config.original_codec_mimetype; meta.config = ac3_config.config; meta.refSampleDuration = 1536 / meta.audioSampleRate * meta.timescale; - } else if (this.audio_metadata_.codec === 'ec-3') { + } else if (audio_metadata_.codec === 'ec-3') { let ec3_frame = sample.codec === 'ec-3' ? sample.data : null; let ec3_config = new EAC3Config(ec3_frame); meta.audioSampleRate = ec3_config.sampling_rate @@ -1823,29 +1922,35 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = ec3_config.original_codec_mimetype; meta.config = ec3_config.config; meta.refSampleDuration = (256 * ec3_config.num_blks) / meta.audioSampleRate * meta.timescale; // TODO: blk size - } else if (this.audio_metadata_.codec === 'opus') { - meta.audioSampleRate = this.audio_metadata_.sample_rate; - meta.channelCount = this.audio_metadata_.channel_count; - meta.channelConfigCode = this.audio_metadata_.channel_config_code; + } else if (audio_metadata_.codec === 'opus') { + meta.audioSampleRate = audio_metadata_.sample_rate; + meta.channelCount = audio_metadata_.channel_count; + meta.channelConfigCode = audio_metadata_.channel_config_code; meta.codec = 'opus'; meta.originalCodec = 'opus'; meta.config = undefined; meta.refSampleDuration = 20; - } else if (this.audio_metadata_.codec === 'mp3') { - meta.audioSampleRate = this.audio_metadata_.sample_rate; - meta.channelCount = this.audio_metadata_.channel_count; + } else if (audio_metadata_.codec === 'mp3') { + meta.audioSampleRate = audio_metadata_.sample_rate; + meta.channelCount = audio_metadata_.channel_count; meta.codec = 'mp3'; meta.originalCodec = 'mp3'; meta.config = undefined; } + this.audioTrackInitSegments.set(pid, { ... sample }); + this.audioTrackMetadata.set(pid, { ... audio_metadata_ }); + if (this.currentAudioTrackPid !== pid) { + return; + } + if (this.audio_init_segment_dispatched_ == false) { Log.v(this.TAG, `Generated first AudioSpecificConfig for mimeType: ${meta.codec}`); } this.onTrackMetadata('audio', meta); this.audio_init_segment_dispatched_ = true; - this.video_metadata_changed_ = false; + this.audio_metadata_changed_ = false; // notify new MediaInfo let mi = this.media_info_; diff --git a/src/player/mse-player.ts b/src/player/mse-player.ts index a6a2d0fe..71e49e9e 100644 --- a/src/player/mse-player.ts +++ b/src/player/mse-player.ts @@ -94,6 +94,10 @@ class MSEPlayer { this._player_engine.pause(); } + public selectAudioTrack(index: number) { + this._player_engine.selectAudioTrack(index); + } + public get type(): string { return this._type; } diff --git a/src/player/player-engine-dedicated-thread.ts b/src/player/player-engine-dedicated-thread.ts index 6ab32c5a..af610538 100644 --- a/src/player/player-engine-dedicated-thread.ts +++ b/src/player/player-engine-dedicated-thread.ts @@ -37,7 +37,8 @@ import { WorkerCommandPacketLoggingConfig, WorkerCommandPacketTimeUpdate, WorkerCommandPacketReadyStateChange, - WorkerCommandPacketUnbufferedSeek + WorkerCommandPacketUnbufferedSeek, + WorkerCommandPacketSelectAudioTrack, } from './player-engine-worker-cmd-def.js'; import { WorkerMessagePacket, @@ -278,6 +279,18 @@ class PlayerEngineDedicatedThread implements PlayerEngine { } } + public selectAudioTrack(track: number): void { + if (!this._config.isLive) { + this._worker.postMessage({ + cmd: 'flush', + }); + } + this._worker.postMessage({ + cmd: 'select_audio_track', + track, + } as WorkerCommandPacketSelectAudioTrack) + } + public get mediaInfo(): MediaInfo { return Object.assign({}, this._media_info); } diff --git a/src/player/player-engine-main-thread.ts b/src/player/player-engine-main-thread.ts index 91e39bd8..95c3c9ce 100644 --- a/src/player/player-engine-main-thread.ts +++ b/src/player/player-engine-main-thread.ts @@ -333,6 +333,13 @@ class PlayerEngineMainThread implements PlayerEngine { } } + public selectAudioTrack(track: number): void { + if (!this._config.isLive) { + this._mse_controller?.flush(); + } + this._transmuxer.selectAudioTrack(track); + } + public get mediaInfo(): MediaInfo { return Object.assign({}, this._media_info); } diff --git a/src/player/player-engine-worker-cmd-def.ts b/src/player/player-engine-worker-cmd-def.ts index 56d4ae23..940e085e 100644 --- a/src/player/player-engine-worker-cmd-def.ts +++ b/src/player/player-engine-worker-cmd-def.ts @@ -24,9 +24,11 @@ export type WorkerCommandOp = | 'shutdown_mse' | 'load' | 'unload' + | 'flush' | 'unbuffered_seek' | 'timeupdate' | 'readystatechange' + | 'select_audio_track' | 'pause_transmuxer' | 'resume_transmuxer'; @@ -59,3 +61,8 @@ export type WorkerCommandPacketReadyStateChange = WorkerCommandPacket & { cmd: 'readystatechange', ready_state: number, }; + +export type WorkerCommandPacketSelectAudioTrack = WorkerCommandPacket & { + cmd: 'select_audio_track', + track: number, +}; diff --git a/src/player/player-engine-worker.ts b/src/player/player-engine-worker.ts index 9e4da036..885f3e39 100644 --- a/src/player/player-engine-worker.ts +++ b/src/player/player-engine-worker.ts @@ -33,6 +33,7 @@ import { WorkerCommandPacketUnbufferedSeek, WorkerCommandPacketTimeUpdate, WorkerCommandPacketReadyStateChange, + WorkerCommandPacketSelectAudioTrack, } from './player-engine-worker-cmd-def.js'; import { WorkerMessagePacket, @@ -95,6 +96,10 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { case 'destroy': destroy(); break; + case 'select_audio_track': + const packet = command_packet as WorkerCommandPacketSelectAudioTrack; + transmuxer.selectAudioTrack(packet.track); + break; case 'initialize_mse': initializeMSE(); break; @@ -107,6 +112,9 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { case 'unload': unload(); break; + case 'flush': + mse_controller.flush(); + break; case 'unbuffered_seek': { const packet = command_packet as WorkerCommandPacketUnbufferedSeek; mse_controller.flush(); diff --git a/src/player/player-engine.ts b/src/player/player-engine.ts index 8bebdb04..71c8ef2a 100644 --- a/src/player/player-engine.ts +++ b/src/player/player-engine.ts @@ -29,6 +29,7 @@ export default interface PlayerEngine { play(): Promise; pause(): void; seek(seconds: number): void; + selectAudioTrack(index: number): void; readonly mediaInfo: MediaInfo | undefined; readonly statisticsInfo: any | undefined; }