From d81520477466129988c45324caf49be898dd61db Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 7 Feb 2025 14:51:49 +0800 Subject: [PATCH] Add CXX API for Kokoro TTS 1.0 (#1802) --- .github/workflows/cxx-api.yaml | 22 ++++++ c-api-examples/kws-c-api.c | 7 +- cxx-api-examples/CMakeLists.txt | 3 + cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc | 74 ++++++++++++++++++++ cxx-api-examples/kws-cxx-api.cc | 7 +- sherpa-onnx/c-api/cxx-api.cc | 2 + sherpa-onnx/c-api/cxx-api.h | 2 + 7 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 7227dd4273..e5a99fb09b 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -103,6 +103,28 @@ jobs: rm kws-cxx-api rm -rf sherpa-onnx-kws-* + - name: Test Kokoro TTS (zh+en) + shell: bash + run: | + g++ -std=c++17 -o kokoro-tts-zh-en-cxx-api ./cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 + tar xf kokoro-multi-lang-v1_0.tar.bz2 + rm kokoro-multi-lang-v1_0.tar.bz2 + + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH + + ./kokoro-tts-zh-en-cxx-api + + rm kokoro-tts-zh-en-cxx-api + rm -rf kokoro-* + - name: Test Kokoro TTS (en) shell: bash run: | diff --git a/c-api-examples/kws-c-api.c b/c-api-examples/kws-c-api.c index ecd70ccf24..8909809f43 100644 --- a/c-api-examples/kws-c-api.c +++ b/c-api-examples/kws-c-api.c @@ -26,7 +26,7 @@ int32_t main() { memset(&config, 0, sizeof(config)); config.model_config.transducer.encoder = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" - "encoder-epoch-12-avg-2-chunk-16-left-64.onnx"; + "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"; config.model_config.transducer.decoder = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" @@ -34,7 +34,7 @@ int32_t main() { config.model_config.transducer.joiner = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" - "joiner-epoch-12-avg-2-chunk-16-left-64.onnx"; + "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx"; config.model_config.tokens = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" @@ -58,7 +58,8 @@ int32_t main() { "--Test pre-defined keywords from test_wavs/test_keywords.txt--\n"); const char *wav_filename = - "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"; + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" + "test_wavs/3.wav"; float tail_paddings[8000] = {0}; // 0.5 seconds diff --git a/cxx-api-examples/CMakeLists.txt b/cxx-api-examples/CMakeLists.txt index 2250736d78..fe21d580c8 100644 --- a/cxx-api-examples/CMakeLists.txt +++ b/cxx-api-examples/CMakeLists.txt @@ -27,4 +27,7 @@ if(SHERPA_ONNX_ENABLE_TTS) add_executable(kokoro-tts-en-cxx-api ./kokoro-tts-en-cxx-api.cc) target_link_libraries(kokoro-tts-en-cxx-api sherpa-onnx-cxx-api) + + add_executable(kokoro-tts-zh-en-cxx-api ./kokoro-tts-zh-en-cxx-api.cc) + target_link_libraries(kokoro-tts-zh-en-cxx-api sherpa-onnx-cxx-api) endif() diff --git a/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc b/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc new file mode 100644 index 0000000000..a47e1964a2 --- /dev/null +++ b/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc @@ -0,0 +1,74 @@ +// cxx-api-examples/kokoro-tts-zh-en-cxx-api.c +// +// Copyright (c) 2025 Xiaomi Corporation + +// This file shows how to use sherpa-onnx CXX API +// for Chinese TTS with Kokoro. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 +tar xf kokoro-multi-lang-v1_0.tar.bz2 +rm kokoro-multi-lang-v1_0.tar.bz2 + +./kokoro-tts-zh-en-cxx-api + + */ +// clang-format on + +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +static int32_t ProgressCallback(const float *samples, int32_t num_samples, + float progress, void *arg) { + fprintf(stderr, "Progress: %.3f%%\n", progress * 100); + // return 1 to continue generating + // return 0 to stop generating + return 1; +} + +int32_t main(int32_t argc, char *argv[]) { + using namespace sherpa_onnx::cxx; // NOLINT + OfflineTtsConfig config; + + config.model.kokoro.model = "./kokoro-multi-lang-v1_0/model.onnx"; + config.model.kokoro.voices = "./kokoro-multi-lang-v1_0/voices.bin"; + config.model.kokoro.tokens = "./kokoro-multi-lang-v1_0/tokens.txt"; + config.model.kokoro.data_dir = "./kokoro-multi-lang-v1_0/espeak-ng-data"; + config.model.kokoro.dict_dir = "./kokoro-multi-lang-v1_0/dict"; + config.model.kokoro.lexicon = + "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/" + "lexicon-zh.txt"; + + config.model.num_threads = 2; + + // If you don't want to see debug messages, please set it to 0 + config.model.debug = 1; + + std::string filename = "./generated-kokoro-zh-en-cxx.wav"; + std::string text = + "中英文语音合成测试。This is generated by next generation Kaldi using " + "Kokoro without Misaki. 你觉得中英文说的如何呢？"; + + auto tts = OfflineTts::Create(config); + int32_t sid = 50; + float speed = 1.0; // larger -> faster in speech speed + +#if 0 + // If you don't want to use a callback, then please enable this branch + GeneratedAudio audio = tts.Generate(text, sid, speed); +#else + GeneratedAudio audio = tts.Generate(text, sid, speed, ProgressCallback); +#endif + + WriteWave(filename, {audio.samples, audio.sample_rate}); + + fprintf(stderr, "Input text is: %s\n", text.c_str()); + fprintf(stderr, "Speaker ID is is: %d\n", sid); + fprintf(stderr, "Saved to: %s\n", filename.c_str()); + + return 0; +} diff --git a/cxx-api-examples/kws-cxx-api.cc b/cxx-api-examples/kws-cxx-api.cc index 44f7343829..12dc8d9cf0 100644 --- a/cxx-api-examples/kws-cxx-api.cc +++ b/cxx-api-examples/kws-cxx-api.cc @@ -25,7 +25,7 @@ int32_t main() { KeywordSpotterConfig config; config.model_config.transducer.encoder = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" - "encoder-epoch-12-avg-2-chunk-16-left-64.onnx"; + "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"; config.model_config.transducer.decoder = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" @@ -33,7 +33,7 @@ int32_t main() { config.model_config.transducer.joiner = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" - "joiner-epoch-12-avg-2-chunk-16-left-64.onnx"; + "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx"; config.model_config.tokens = "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" @@ -57,7 +57,8 @@ int32_t main() { << "--Test pre-defined keywords from test_wavs/test_keywords.txt--\n"; std::string wave_filename = - "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"; + "./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01-mobile/" + "test_wavs/3.wav"; std::array tail_paddings = {0}; // 0.5 seconds diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 63a47cc6e6..7ce2b63a19 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -343,6 +343,8 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) { c.model.kokoro.tokens = config.model.kokoro.tokens.c_str(); c.model.kokoro.data_dir = config.model.kokoro.data_dir.c_str(); c.model.kokoro.length_scale = config.model.kokoro.length_scale; + c.model.kokoro.dict_dir = config.model.kokoro.dict_dir.c_str(); + c.model.kokoro.lexicon = config.model.kokoro.lexicon.c_str(); c.model.num_threads = config.model.num_threads; c.model.debug = config.model.debug; diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 8416c59453..66133f6d29 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -343,6 +343,8 @@ struct OfflineTtsKokoroModelConfig { std::string voices; std::string tokens; std::string data_dir; + std::string dict_dir; + std::string lexicon; float length_scale = 1.0; // < 1, faster in speed; > 1, slower in speed };