Skip to content

Commit

Permalink
Run TTS engine service without starting the app. (#553)
Browse files Browse the repository at this point in the history
  • Loading branch information
csukuangfj authored Jan 26, 2024
1 parent 4fbad6a commit 7ae73e7
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 34 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
project(sherpa-onnx)

set(SHERPA_ONNX_VERSION "1.9.8")
set(SHERPA_ONNX_VERSION "1.9.9")

# Disable warning about
#
Expand Down
3 changes: 2 additions & 1 deletion android/SherpaOnnxTtsEngine/app/src/main/AndroidManifest.xml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">
xmlns:tools="http://schemas.android.com/tools"
package="com.k2fsa.sherpa.onnx.tts.engine">

<application
android:allowBackup="true"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import android.util.Log
import android.widget.Toast
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.viewModels
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
Expand Down Expand Up @@ -43,9 +44,13 @@ import java.lang.NumberFormatException
const val TAG = "sherpa-onnx-tts-engine"

class MainActivity : ComponentActivity() {
// TODO(fangjun): Save settings in ttsViewModel
private val ttsViewModel: TtsViewModel by viewModels()

private var mediaPlayer: MediaPlayer? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
TtsEngine.createTts(this.application)
TtsEngine.createTts(this)
setContent {
SherpaOnnxTtsEngineTheme {
// A surface container using the 'background' color from the theme
Expand Down Expand Up @@ -132,11 +137,12 @@ class MainActivity : ComponentActivity() {
audio.samples.size > 0 && audio.save(filename)

if (ok) {
val mediaPlayer = MediaPlayer.create(
stopMediaPlayer()
mediaPlayer = MediaPlayer.create(
applicationContext,
Uri.fromFile(File(filename))
)
mediaPlayer.start()
mediaPlayer?.start()
} else {
Log.i(TAG, "Failed to generate or save audio")
}
Expand All @@ -162,4 +168,15 @@ class MainActivity : ComponentActivity() {
}
}
}

override fun onDestroy() {
stopMediaPlayer()
super.onDestroy()
}

private fun stopMediaPlayer() {
mediaPlayer?.stop()
mediaPlayer?.release()
mediaPlayer = null
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.k2fsa.sherpa.onnx.tts.engine

import android.app.Application
import android.content.Context
import android.content.res.AssetManager
import android.util.Log
import androidx.compose.runtime.MutableState
Expand All @@ -21,7 +21,6 @@ object TtsEngine {
var lang: String? = null



val speedState: MutableState<Float> = mutableStateOf(1.0F)
val speakerIdState: MutableState<Int> = mutableStateOf(0)

Expand All @@ -44,19 +43,7 @@ object TtsEngine {
private var dataDir: String? = null
private var assets: AssetManager? = null

private var application: Application? = null

fun createTts(application: Application) {
Log.i(TAG, "Init Next-gen Kaldi TTS")
if (tts == null) {
this.application = application
initTts()
}
}

private fun initTts() {
assets = application?.assets

init {
// The purpose of such a design is to make the CI test easier
// Please see
// https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/apk/generate-tts-apk-script.py
Expand Down Expand Up @@ -89,9 +76,21 @@ object TtsEngine {
// ruleFsts = "vits-zh-aishell3/rule.fst"
// lexicon = "lexicon.txt"
// lang = "zho"
}


fun createTts(context: Context) {
Log.i(TAG, "Init Next-gen Kaldi TTS")
if (tts == null) {
initTts(context)
}
}

private fun initTts(context: Context) {
assets = context.assets

if (dataDir != null) {
val newDir = copyDataDir(modelDir!!)
val newDir = copyDataDir(context, modelDir!!)
modelDir = newDir + "/" + modelDir
dataDir = newDir + "/" + dataDir
assets = null
Expand All @@ -107,39 +106,39 @@ object TtsEngine {
}


private fun copyDataDir(dataDir: String): String {
private fun copyDataDir(context: Context, dataDir: String): String {
println("data dir is $dataDir")
copyAssets(dataDir)
copyAssets(context, dataDir)

val newDataDir = application!!.getExternalFilesDir(null)!!.absolutePath
val newDataDir = context.getExternalFilesDir(null)!!.absolutePath
println("newDataDir: $newDataDir")
return newDataDir
}

private fun copyAssets(path: String) {
private fun copyAssets(context: Context, path: String) {
val assets: Array<String>?
try {
assets = application!!.assets.list(path)
assets = context.assets.list(path)
if (assets!!.isEmpty()) {
copyFile(path)
copyFile(context, path)
} else {
val fullPath = "${application!!.getExternalFilesDir(null)}/$path"
val fullPath = "${context.getExternalFilesDir(null)}/$path"
val dir = File(fullPath)
dir.mkdirs()
for (asset in assets.iterator()) {
val p: String = if (path == "") "" else path + "/"
copyAssets(p + asset)
copyAssets(context, p + asset)
}
}
} catch (ex: IOException) {
Log.e(TAG, "Failed to copy $path. ${ex.toString()}")
}
}

private fun copyFile(filename: String) {
private fun copyFile(context: Context, filename: String) {
try {
val istream = application!!.assets.open(filename)
val newFilename = application!!.getExternalFilesDir(null).toString() + "/" + filename
val istream = context.assets.open(filename)
val newFilename = context.getExternalFilesDir(null).toString() + "/" + filename
val ostream = FileOutputStream(newFilename)
// Log.i(TAG, "Copying $filename to $newFilename")
val buffer = ByteArray(1024)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,18 @@ Failed to get default language from engine com.k2fsa.sherpa.chapter5

class TtsService : TextToSpeechService() {
override fun onCreate() {
Log.i(TAG, "onCreate tts service")
super.onCreate()

// see https://github.com/Miserlou/Android-SDK-Samples/blob/master/TtsEngine/src/com/example/android/ttsengine/RobotSpeakTtsService.java#L68
onLoadLanguage(TtsEngine.lang, "", "")
}

override fun onDestroy() {
Log.i(TAG, "onDestroy tts service")
super.onDestroy()
}

// https://developer.android.com/reference/kotlin/android/speech/tts/TextToSpeechService#onislanguageavailable
override fun onIsLanguageAvailable(_lang: String?, _country: String?, _variant: String?): Int {
val lang = _lang ?: ""
Expand All @@ -79,12 +85,15 @@ class TtsService : TextToSpeechService() {

// https://developer.android.com/reference/kotlin/android/speech/tts/TextToSpeechService#onLoadLanguage(kotlin.String,%20kotlin.String,%20kotlin.String)
override fun onLoadLanguage(_lang: String?, _country: String?, _variant: String?): Int {
Log.i(TAG, "onLoadLanguage: $_lang, $_country")
val lang = _lang ?: ""

return if (lang == TtsEngine.lang) {
Log.i(TAG, "creating tts, lang :$lang")
TtsEngine.createTts(application)
TextToSpeech.LANG_AVAILABLE
} else {
Log.i(TAG, "lang $lang not supported, tts engine lang: ${TtsEngine.lang}")
TextToSpeech.LANG_NOT_SUPPORTED
}
}
Expand Down Expand Up @@ -118,7 +127,7 @@ class TtsService : TextToSpeechService() {
return
}

val ttsCallback = {floatSamples: FloatArray ->
val ttsCallback = { floatSamples: FloatArray ->
// convert FloatArray to ByteArray
val samples = floatArrayToByteArray(floatSamples)
val maxBufferSize: Int = callback.maxBufferSize
Expand All @@ -136,7 +145,7 @@ class TtsService : TextToSpeechService() {
text = text,
sid = TtsEngine.speakerId,
speed = TtsEngine.speed,
callback=ttsCallback,
callback = ttsCallback,
)

callback.done()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package com.k2fsa.sherpa.onnx.tts.engine

import android.app.Application
import android.os.FileUtils.ProgressListener
import android.speech.tts.TextToSpeech
import android.speech.tts.TextToSpeech.OnInitListener
import android.speech.tts.UtteranceProgressListener
import android.util.Log
import androidx.lifecycle.ViewModel
import java.util.Locale

class TtsApp : Application() {
companion object {
lateinit var instance: TtsApp
}

override fun onCreate() {
super.onCreate()
instance = this
}

}

class TtsViewModel : ViewModel() {

// https://developer.android.com/reference/kotlin/android/speech/tts/TextToSpeech.OnInitListener
private val onInitListener = object : OnInitListener {
override fun onInit(status: Int) {
when (status) {
TextToSpeech.SUCCESS -> Log.i(TAG, "Init tts succeded")
TextToSpeech.ERROR -> Log.i(TAG, "Init tts failed")
else -> Log.i(TAG, "Unknown status $status")
}
}
}

// https://developer.android.com/reference/kotlin/android/speech/tts/UtteranceProgressListener
private val utteranceProgressListener = object : UtteranceProgressListener() {
override fun onStart(utteranceId: String?) {
Log.i(TAG, "onStart: $utteranceId")
}

override fun onStop(utteranceId: String?, interrupted: Boolean) {
Log.i(TAG, "onStop: $utteranceId, $interrupted")
super.onStop(utteranceId, interrupted)
}

override fun onError(utteranceId: String?, errorCode: Int) {
Log.i(TAG, "onError: $utteranceId, $errorCode")
super.onError(utteranceId, errorCode)
}

override fun onDone(utteranceId: String?) {
Log.i(TAG, "onDone: $utteranceId")
}

@Deprecated("Deprecated in Java")
override fun onError(utteranceId: String?) {
Log.i(TAG, "onError: $utteranceId")
}
}

val tts = TextToSpeech(TtsApp.instance, onInitListener, "com.k2fsa.sherpa.onnx.tts.engine")

init {
tts.setLanguage(Locale(TtsEngine.lang!!))
tts.setOnUtteranceProgressListener(utteranceProgressListener)
}

override fun onCleared() {
super.onCleared()
tts.shutdown()
}
}

0 comments on commit 7ae73e7

Please sign in to comment.