WIP: Adding Transcription/Subtitle Viewing Support to the Web Player (VTT)

client/components/app/MediaPlayerContainer.vue

@@ -1,5 +1,6 @@
 <template>
-  <div v-if="streamLibraryItem" id="mediaPlayerContainer" class="w-full fixed bottom-0 left-0 right-0 h-48 lg:h-40 z-50 bg-primary px-2 lg:px-4 pb-1 lg:pb-4 pt-2">
+    <div v-if="streamLibraryItem" id="mediaPlayerContainer"  class="w-full fixed bottom-0 left-0 right-0 z-50 bg-primary px-2 lg:px-4 pb-1 lg:pb-4 pt-2"
+         :class="[showTranscriptionUi ? 'h-64' : 'h-48', showTranscriptionUi ? 'lg:h-64' : 'lg:h-40']">
     <div id="videoDock" />
     <div class="absolute left-2 top-2 lg:left-4 cursor-pointer">
       <covers-book-cover expand-on-click :library-item="streamLibraryItem" :width="bookCoverWidth" :book-cover-aspect-ratio="coverAspectRatio" />
@@ -41,6 +42,7 @@
       :sleep-timer-set="sleepTimerSet"
       :sleep-timer-remaining="sleepTimerRemaining"
       :is-podcast="isPodcast"
+      :transcription-enabled="showTranscriptionUi"
       @playPause="playPause"
       @jumpForward="jumpForward"
       @jumpBackward="jumpBackward"
@@ -51,6 +53,12 @@
       @showBookmarks="showBookmarks"
       @showSleepTimer="showSleepTimerModal = true"
       @showPlayerQueueItems="showPlayerQueueItemsModal = true"
+      @showTranscription="showTranscriptionUi = !showTranscriptionUi"
+    />
+
+    <transcription-ui
+      v-if="showTranscriptionUi"
+      @seek="seek"
     />
 
     <modals-bookmarks-modal v-model="showBookmarksModal" :bookmarks="bookmarks" :current-time="bookmarkCurrentTime" :library-item-id="libraryItemId" @select="selectBookmark" />
@@ -63,8 +71,10 @@
 
 <script>
 import PlayerHandler from '@/players/PlayerHandler'
+import TranscriptionUi from "@/components/player/TranscriptionUi.vue"
 
 export default {
+  components: {TranscriptionUi},
   data() {
     return {
       playerHandler: new PlayerHandler(this),
@@ -76,6 +86,7 @@ export default {
       currentTime: 0,
       showSleepTimerModal: false,
       showPlayerQueueItemsModal: false,
+      showTranscriptionUi: false,
       sleepTimerSet: false,
       sleepTimerTime: 0,
       sleepTimerRemaining: 0,
@@ -86,6 +97,17 @@ export default {
       coverAspectRatio: 1
     }
   },
+  watch: {
+    'playerHandler.playerState': function (newVal) {
+      // Refresh the transcription UI when the audio track is changed
+      if (newVal === 'LOADED') {
+        this.showTranscriptionUi = false
+        this.$nextTick(() => {
+          this.showTranscriptionUi = true
+        });
+      }
+    },
+  },
   computed: {
     isSquareCover() {
       return this.coverAspectRatio === 1
@@ -485,6 +507,7 @@ export default {
     this.$eventBus.$on('playback-time-update', this.playbackTimeUpdate)
     this.$eventBus.$on('play-item', this.playLibraryItem)
     this.$eventBus.$on('pause-item', this.pauseItem)
+    this.showTranscriptionUi = false
   },
   beforeDestroy() {
     this.$eventBus.$off('cast-session-active', this.castSessionActive)

client/components/player/PlayerUi.vue

@@ -36,6 +36,13 @@
           </button>
         </ui-tooltip>
 
+        <ui-tooltip direction="top" :text="$strings.LabelViewTranscription">
+          <button :aria-label="$strings.LabelViewTranscription" class="outline-none text-gray-300 mx-1 lg:mx-2 hover:text-white" @mousedown.prevent @mouseup.prevent @click.stop="$emit('showTranscription')">
+            <span v-if="!transcriptionEnabled" class="material-icons text-2xl">subtitles</span>
+            <span v-else  class="material-icons text-2xl text-warning">subtitles</span>
+          </button>
+        </ui-tooltip>
+
         <ui-tooltip v-if="chapters.length" direction="top" :text="useChapterTrack ? $strings.LabelUseFullTrack : $strings.LabelUseChapterTrack">
           <button :aria-label="useChapterTrack ? $strings.LabelUseFullTrack : $strings.LabelUseChapterTrack" class="text-gray-300 mx-1 lg:mx-2 hover:text-white" @mousedown.prevent @mouseup.prevent @click.stop="setUseChapterTrack">
             <span class="material-icons text-2xl sm:text-3xl transform transition-transform" :class="useChapterTrack ? 'rotate-180' : ''">timelapse</span>
@@ -78,7 +85,8 @@ export default {
     },
     sleepTimerSet: Boolean,
     sleepTimerRemaining: Number,
-    isPodcast: Boolean
+    isPodcast: Boolean,
+    transcriptionEnabled: Boolean
   },
   data() {
     return {
@@ -368,4 +376,4 @@ export default {
     left: 100%;
   }
 }
-</style>
+</style>

client/components/player/TranscriptionLine.vue

@@ -0,0 +1,43 @@
+<template>
+  <div :class="{ 'text-warning': isActive }"  class="cursor-pointer" @click.stop="clickSeek">
+    <div v-html="cue.text"></div>
+  </div>
+</template>
+
+<script>
+export default {
+  props: {
+    cue: VTTCue
+  },
+  data() {
+    return {
+      isActive: false
+    };
+  },
+  methods: {
+    clickSeek() {
+      const time = this.cue.startTime
+      this.$emit('seek', time)
+    },
+    scrollIntoView() {
+      this.$el.scrollIntoView({behavior: 'smooth'})
+    }
+  },
+  mounted() {
+    if (this.isActive) {
+      this.scrollIntoView()
+    }
+  },
+  created() {
+    this.cue.onenter = () => (this.isActive = true)
+    this.cue.onexit = () => (this.isActive = false)
+  },
+  watch: {
+    isActive(newVal) {
+      if (newVal) {
+        this.scrollIntoView()
+      }
+    }
+  }
+};
+</script>

client/components/player/TranscriptionUi.vue

@@ -0,0 +1,55 @@
+<template>
+  <div id="transcription-panel">
+    <transcription-line
+      v-for="(cue, index) in cues"
+      :key="index"
+      :cue="cue"
+      ref="transcriptionLine + index"
+      @seek="seek"
+    ></transcription-line>
+  </div>
+</template>
+
+<script>
+import TranscriptionLine from "./TranscriptionLine.vue"
+
+export default {
+  components: {
+    TranscriptionLine
+  },
+  watch: {
+    trackElement() {
+      this.setCues()
+    }
+  },
+  data() {
+    return {
+      cues: [],
+      trackElement: null
+    }
+  },
+  mounted() {
+    this.init()
+  },
+  methods: {
+    init() {
+      this.trackElement = document.getElementById("transcription-track")
+      if (this.trackElement && this.trackElement.track) {
+        this.setCues()
+      }
+    },
+    seek(time) {
+      this.$emit('seek', time)
+    },
+    setCues() {
+      this.cues = this.trackElement.track.cues
+    }
+  },
+}
+</script>
+<style>
+#transcription-panel {
+  max-height: 75px;
+  overflow-y: auto;
+}
+</style>

client/players/AudioTrack.js

@@ -5,6 +5,7 @@ export default class AudioTrack {
     this.duration = track.duration || 0
     this.title = track.title || ''
     this.contentUrl = track.contentUrl || null
+    this.transcriptUrl = track.transcriptUrl || null
     this.mimeType = track.mimeType
     this.metadata = track.metadata || {}
 
@@ -29,4 +30,14 @@ export default class AudioTrack {
 
     return this.contentUrl + `?token=${this.userToken}`
   }
-}
+
+  get relativeTranscriptionUrl() {
+    if (!this.transcriptUrl || this.transcriptUrl.startsWith('http')) return this.transcriptUrl
+
+    if (process.env.NODE_ENV === 'development') {
+      return `${process.env.serverUrl}${this.transcriptUrl}?token=${this.userToken}`
+    }
+
+    return this.transcriptUrl + `?token=${this.userToken}`
+  }
+}

client/players/LocalAudioPlayer.js

@@ -35,6 +35,8 @@ export default class LocalAudioPlayer extends EventEmitter {
     var audioEl = document.createElement('audio')
     audioEl.id = 'audio-player'
     audioEl.style.display = 'none'
+    audioEl.crossOrigin = 'anonymous'
+
     document.body.appendChild(audioEl)
     this.player = audioEl
 
@@ -135,6 +137,8 @@ export default class LocalAudioPlayer extends EventEmitter {
       this.usingNativeplayer = true
       this.player.src = this.currentTrack.relativeContentUrl
       this.player.currentTime = this.startTime
+
+      this.createTranscriptionTrack()
       return
     }
 
@@ -168,7 +172,7 @@ export default class LocalAudioPlayer extends EventEmitter {
 
     this.hlsInstance.attachMedia(this.player)
     this.hlsInstance.on(Hls.Events.MEDIA_ATTACHED, () => {
-      this.hlsInstance.loadSource(this.currentTrack.relativeContentUrl)
+    this.hlsInstance.loadSource(this.currentTrack.relativeContentUrl)
 
       this.hlsInstance.on(Hls.Events.MANIFEST_PARSED, () => {
         console.log('[HLS] Manifest Parsed')
@@ -206,9 +210,28 @@ export default class LocalAudioPlayer extends EventEmitter {
     this.trackStartTime = Math.max(0, this.startTime - (this.currentTrack.startOffset || 0))
     this.player.src = this.currentTrack.relativeContentUrl
     console.log(`[LocalPlayer] Loading track src ${this.currentTrack.relativeContentUrl}`)
+
+    this.createTranscriptionTrack()
+
     this.player.load()
   }
 
+  createTranscriptionTrack() {
+    if (document.getElementById('transcription-track')) {
+      document.getElementById('transcription-track').remove()
+    }
+
+    const trackElement = document.createElement("track")
+
+    trackElement.id = "transcription-track"
+    trackElement.kind = "subtitles"
+    trackElement.label = "Transcription"
+    trackElement.default = true
+    trackElement.src = this.currentTrack.relativeTranscriptionUrl
+
+    this.player.appendChild(trackElement)
+  }
+
   destroyHlsInstance() {
     if (!this.hlsInstance) return
     if (this.hlsInstance.destroy) {
@@ -338,4 +361,4 @@ export default class LocalAudioPlayer extends EventEmitter {
     var last = bufferedRanges[bufferedRanges.length - 1]
     return last.end
   }
-}
+}

client/plugins/constants.js

@@ -4,7 +4,8 @@ const SupportedFileTypes = {
   ebook: ['epub', 'pdf', 'mobi', 'azw3', 'cbr', 'cbz'],
   info: ['nfo'],
   text: ['txt'],
-  metadata: ['opf', 'abs', 'xml', 'json']
+  metadata: ['opf', 'abs', 'xml', 'json'],
+  subtitle: ['srt', 'vtt']
 }
 
 const DownloadStatus = {
@@ -82,4 +83,4 @@ export default ({ app }, inject) => {
   inject('constants', Constants)
   inject('keynames', KeyNames)
   inject('hotkeys', Hotkeys)
-}
+}

client/strings/en-us.json

@@ -562,6 +562,7 @@
   "LabelViewBookmarks": "View bookmarks",
   "LabelViewChapters": "View chapters",
   "LabelViewQueue": "View player queue",
+  "LabelViewTranscription": "View transcription",
   "LabelVolume": "Volume",
   "LabelWeekdaysToRun": "Weekdays to run",
   "LabelYearReviewHide": "Hide Year in Review",
@@ -779,4 +780,4 @@
   "ToastSocketFailedToConnect": "Socket failed to connect",
   "ToastUserDeleteFailed": "Failed to delete user",
   "ToastUserDeleteSuccess": "User deleted"
-}
+}