WIP: Begin to add C++ API for streaming zipformer ASR on RK NPU

CMakeLists.txt

@@ -45,6 +45,7 @@ option(SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY "True to link libstdc++ statically.
 option(SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE "True to use pre-installed onnxruntime if available" ON)
 option(SHERPA_ONNX_ENABLE_SANITIZER "Whether to enable ubsan and asan" OFF)
 option(SHERPA_ONNX_BUILD_C_API_EXAMPLES "Whether to enable C API examples" ON)
+option(SHERPA_ONNX_ENABLE_RKNN "Whether to build for RKNN NPU " OFF)
 
 set(SHERPA_ONNX_LINUX_ARM64_GPU_ONNXRUNTIME_VERSION "1.11.0" CACHE STRING "Used only for Linux ARM64 GPU. If you use Jetson nano b01, then please set it to 1.11.0. If you use Jetson Orin NX, then set it to 1.16.0")
 
@@ -198,7 +199,7 @@ if(SHERPA_ONNX_ENABLE_DIRECTML)
   message(STATUS "DirectML is enabled")
   add_definitions(-DSHERPA_ONNX_ENABLE_DIRECTML=1)
 else()
-  message(WARNING "DirectML is disabled")
+  message(STATUS "DirectML is disabled")
   add_definitions(-DSHERPA_ONNX_ENABLE_DIRECTML=0)
 endif()
 
@@ -266,6 +267,10 @@ message(STATUS "C++ Standard version: ${CMAKE_CXX_STANDARD}")
 
 include(CheckIncludeFileCXX)
 
+if(SHERPA_ONNX_ENABLE_RKNN)
+  add_definitions(-DSHERPA_ONNX_ENABLE_RKNN=1)
+endif()
+
 if(UNIX AND NOT APPLE AND NOT SHERPA_ONNX_ENABLE_WASM AND NOT CMAKE_SYSTEM_NAME STREQUAL Android AND NOT CMAKE_SYSTEM_NAME STREQUAL OHOS)
   check_include_file_cxx(alsa/asoundlib.h SHERPA_ONNX_HAS_ALSA)
   if(SHERPA_ONNX_HAS_ALSA)

build-aarch64-linux-gnu.sh

@@ -66,7 +66,7 @@ if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then
 fi
 
 if [[ x"$SHERPA_ONNX_ENABLE_GPU" == x"" ]]; then
-  # By default, use CPU
+  # By default, don't use CPU
   SHERPA_ONNX_ENABLE_GPU=OFF
 fi
 

build-rknn-linux-aarch64.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Before you run this file, make sure you have first cloned
+# https://github.com/airockchip/rknn-toolkit2
+# and set the environment variable SHERPA_ONNX_RKNN_TOOLKIT2_PATH
+
+if [ -z $SHERPA_ONNX_RKNN_TOOLKIT2_PATH ]; then
+  SHERPA_ONNX_RKNN_TOOLKIT2_PATH=/star-fj/fangjun/open-source/rknn-toolkit2
+fi
+
+if [ ! -d $SHERPA_ONNX_RKNN_TOOLKIT2_PATH ]; then
+  echo "Please first clone https://github.com/airockchip/rknn-toolkit2"
+  echo "You can use"
+  echo " git clone --depth 1 https://github.com/airockchip/rknn-toolkit2"
+  echo " export SHERPA_ONNX_RKNN_TOOLKIT2_PATH=$PWD/rknn-toolkit2"
+
+  exit 1
+fi
+
+if [ ! -f  $SHERPA_ONNX_RKNN_TOOLKIT2_PATH/rknpu2/runtime/Linux/librknn_api/include/rknn_api.h ]; then
+  echo "$SHERPA_ONNX_RKNN_TOOLKIT2_PATH/rknpu2/runtime/Linux/librknn_api/include/rknn_api.h does not exist"
+  exit 1
+fi
+
+if [ ! -f $SHERPA_ONNX_RKNN_TOOLKIT2_PATH/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so ]; then
+  echo "$SHERPA_ONNX_RKNN_TOOLKIT2_PATH/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so does not exist"
+  exit 1
+fi
+
+export SHERPA_ONNX_RKNN_TOOLKIT2_LIB_DIR=$SHERPA_ONNX_RKNN_TOOLKIT2_PATH/rknpu2/runtime/Linux/librknn_api/aarch64
+
+export CPLUS_INCLUDE_PATH=$SHERPA_ONNX_RKNN_TOOLKIT2_PATH/rknpu2/runtime/Linux/librknn_api/include:$CPLUS_INCLUDE_PATH
+
+if ! command -v aarch64-linux-gnu-gcc  &> /dev/null; then
+  echo "Please install a toolchain for cross-compiling."
+  echo "You can refer to: "
+  echo "  https://k2-fsa.github.io/sherpa/onnx/install/rknn-linux-aarch64.html"
+  echo "for help."
+  exit 1
+fi
+
+if [ -z $SHERPA_ONNX_RKNN_TARGET_PLATFORM ]; then
+  SHERPA_ONNX_RKNN_TARGET_PLATFORM=rk3588
+fi
+
+if [[ $SHERPA_ONNX_RKNN_TARGET_PLATFORM != rk3562 &&
+     $SHERPA_ONNX_RKNN_TARGET_PLATFORM != rk3566 &&
+     $SHERPA_ONNX_RKNN_TARGET_PLATFORM != rk3568 &&
+     $SHERPA_ONNX_RKNN_TARGET_PLATFORM != rk3576 &&
+     $SHERPA_ONNX_RKNN_TARGET_PLATFORM != rk3588
+   ]]; then
+   echo "Only support rk3562, rk3566, rk3576, rk3588. Given: $SHERPA_ONNX_RKNN_TARGET_PLATFORM"
+   exit 1
+fi
+
+dir=$PWD/build-rknn-linux-aarch64-$SHERPA_ONNX_RKNN_TARGET_PLATFORM
+mkdir -p $dir
+
+cd $dir
+
+if [ ! -f alsa-lib/src/.libs/libasound.so ]; then
+  echo "Start to cross-compile alsa-lib"
+  if [ ! -d alsa-lib ]; then
+    git clone --depth 1 --branch v1.2.12 https://github.com/alsa-project/alsa-lib
+  fi
+  # If it shows:
+  #  ./gitcompile: line 79: libtoolize: command not found
+  # Please use:
+  #  sudo apt-get install libtool m4 automake
+  #
+  pushd alsa-lib
+  CC=aarch64-linux-gnu-gcc ./gitcompile --host=aarch64-linux-gnu
+  popd
+  echo "Finish cross-compiling alsa-lib"
+fi
+
+export CPLUS_INCLUDE_PATH=$PWD/alsa-lib/include:$CPLUS_INCLUDE_PATH
+export SHERPA_ONNX_ALSA_LIB_DIR=$PWD/alsa-lib/src/.libs
+
+if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then
+  # By default, use shared link
+  BUILD_SHARED_LIBS=ON
+fi
+
+cmake \
+  -DBUILD_PIPER_PHONMIZE_EXE=OFF \
+  -DBUILD_PIPER_PHONMIZE_TESTS=OFF \
+  -DBUILD_ESPEAK_NG_EXE=OFF \
+  -DBUILD_ESPEAK_NG_TESTS=OFF \
+  -DCMAKE_INSTALL_PREFIX=./install \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DSHERPA_ONNX_ENABLE_GPU=OFF \
+  -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \
+  -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+  -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+  -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+  -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+  -DSHERPA_ONNX_ENABLE_JNI=OFF \
+  -DSHERPA_ONNX_ENABLE_C_API=ON \
+  -DSHERPA_ONNX_ENABLE_WEBSOCKET=ON \
+  -DSHERPA_ONNX_ENABLE_RKNN=ON \
+  -DCMAKE_TOOLCHAIN_FILE=../toolchains/aarch64-linux-gnu.toolchain.cmake \
+  ..
+
+# make VERBOSE=1 -j4
+make VERBOSE=1 -j15 sherpa-onnx
+# make install/strip
+
+# Enable it if only needed
+# cp -v $SHERPA_ONNX_ALSA_LIB_DIR/libasound.so* ./install/lib/

sherpa-onnx/csrc/CMakeLists.txt

@@ -151,6 +151,13 @@ list(APPEND sources
   online-punctuation-model-config.cc
   online-punctuation.cc
 )
+if(SHERPA_ONNX_ENABLE_RKNN)
+  list(APPEND sources
+    ./rknn/online-stream-rknn.cc
+    ./rknn/online-zipformer-transducer-model-rknn.cc
+  )
+
+endif()
 
 if(SHERPA_ONNX_ENABLE_TTS)
   list(APPEND sources
@@ -230,6 +237,14 @@ if(SHERPA_ONNX_ENABLE_GPU)
   )
 endif()
 
+if(SHERPA_ONNX_ENABLE_RKNN)
+  if(DEFINED ENV{SHERPA_ONNX_RKNN_TOOLKIT2_LIB_DIR})
+    target_link_libraries(sherpa-onnx-core -L$ENV{SHERPA_ONNX_RKNN_TOOLKIT2_LIB_DIR} -lrknnrt)
+  else()
+    target_link_libraries(sherpa-onnx-core rknnrt)
+  endif()
+endif()
+
 if(BUILD_SHARED_LIBS AND NOT DEFINED onnxruntime_lib_files)
   target_link_libraries(sherpa-onnx-core onnxruntime)
 else()

sherpa-onnx/csrc/file-utils.cc

@@ -22,4 +22,61 @@ void AssertFileExists(const std::string &filename) {
   }
 }
 
+std::vector<char> ReadFile(const std::string &filename) {
+  std::ifstream input(filename, std::ios::binary);
+  std::vector<char> buffer(std::istreambuf_iterator<char>(input), {});
+  return buffer;
+}
+
+#if __ANDROID_API__ >= 9
+std::vector<char> ReadFile(AAssetManager *mgr, const std::string &filename) {
+  AAsset *asset = AAssetManager_open(mgr, filename.c_str(), AASSET_MODE_BUFFER);
+  if (!asset) {
+    __android_log_print(ANDROID_LOG_FATAL, "sherpa-onnx",
+                        "Read binary file: Load %s failed", filename.c_str());
+    exit(-1);
+  }
+
+  auto p = reinterpret_cast<const char *>(AAsset_getBuffer(asset));
+  size_t asset_length = AAsset_getLength(asset);
+
+  std::vector<char> buffer(p, p + asset_length);
+  AAsset_close(asset);
+
+  return buffer;
+}
+#endif
+
+#if __OHOS__
+std::vector<char> ReadFile(NativeResourceManager *mgr,
+                           const std::string &filename) {
+  std::unique_ptr<RawFile, decltype(&OH_ResourceManager_CloseRawFile)> fp(
+      OH_ResourceManager_OpenRawFile(mgr, filename.c_str()),
+      OH_ResourceManager_CloseRawFile);
+
+  if (!fp) {
+    std::ostringstream os;
+    os << "Read file '" << filename << "' failed.";
+    SHERPA_ONNX_LOGE("%s", os.str().c_str());
+    return {};
+  }
+
+  auto len = static_cast<int32_t>(OH_ResourceManager_GetRawFileSize(fp.get()));
+
+  std::vector<char> buffer(len);
+
+  int32_t n = OH_ResourceManager_ReadRawFile(fp.get(), buffer.data(), len);
+
+  if (n != len) {
+    std::ostringstream os;
+    os << "Read file '" << filename << "' failed. Number of bytes read: " << n
+       << ". Expected bytes to read: " << len;
+    SHERPA_ONNX_LOGE("%s", os.str().c_str());
+    return {};
+  }
+
+  return buffer;
+}
+#endif
+
 }  // namespace sherpa_onnx

sherpa-onnx/csrc/file-utils.h

@@ -7,6 +7,16 @@
 
 #include <fstream>
 #include <string>
+#include <vector>
+
+#if __ANDROID_API__ >= 9
+#include "android/asset_manager.h"
+#include "android/asset_manager_jni.h"
+#endif
+
+#if __OHOS__
+#include "rawfile/raw_file_manager.h"
+#endif
 
 namespace sherpa_onnx {
 
@@ -23,6 +33,17 @@ bool FileExists(const std::string &filename);
  */
 void AssertFileExists(const std::string &filename);
 
+std::vector<char> ReadFile(const std::string &filename);
+
+#if __ANDROID_API__ >= 9
+std::vector<char> ReadFile(AAssetManager *mgr, const std::string &filename);
+#endif
+
+#if __OHOS__
+std::vector<char> ReadFile(NativeResourceManager *mgr,
+                           const std::string &filename);
+#endif
+
 }  // namespace sherpa_onnx
 
 #endif  // SHERPA_ONNX_CSRC_FILE_UTILS_H_

sherpa-onnx/csrc/hifigan-vocoder.cc

@@ -17,6 +17,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-ced-model.cc

@@ -7,6 +7,7 @@
 #include <string>
 #include <vector>
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"
 #include "sherpa-onnx/csrc/text-utils.h"

sherpa-onnx/csrc/offline-ct-transformer-model.cc

@@ -7,6 +7,7 @@
 #include <string>
 #include <vector>
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"
 #include "sherpa-onnx/csrc/text-utils.h"

sherpa-onnx/csrc/offline-ctc-model.cc

@@ -18,6 +18,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h"
 #include "sherpa-onnx/csrc/offline-tdnn-ctc-model.h"

sherpa-onnx/csrc/offline-fire-red-asr-model.cc

@@ -20,6 +20,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-moonshine-model.cc

@@ -17,6 +17,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.cc

@@ -13,6 +13,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-paraformer-model.cc

@@ -17,6 +17,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-recognizer-impl.cc

@@ -22,6 +22,7 @@
 #include "fst/extensions/far/far.h"
 #include "kaldifst/csrc/kaldi-fst-io.h"
 #include "onnxruntime_cxx_api.h"  // NOLINT
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/offline-recognizer-ctc-impl.h"
 #include "sherpa-onnx/csrc/offline-recognizer-fire-red-asr-impl.h"
@@ -31,7 +32,6 @@
 #include "sherpa-onnx/csrc/offline-recognizer-transducer-impl.h"
 #include "sherpa-onnx/csrc/offline-recognizer-transducer-nemo-impl.h"
 #include "sherpa-onnx/csrc/offline-recognizer-whisper-impl.h"
-#include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/text-utils.h"
 
 namespace sherpa_onnx {

sherpa-onnx/csrc/offline-rnn-lm.cc

@@ -18,6 +18,7 @@
 #endif
 
 #include "onnxruntime_cxx_api.h"  // NOLINT
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-sense-voice-model.cc

@@ -17,6 +17,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-speaker-segmentation-pyannote-model.cc

@@ -17,6 +17,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"
 

sherpa-onnx/csrc/offline-tdnn-ctc-model.cc

@@ -15,6 +15,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"

sherpa-onnx/csrc/offline-telespeech-ctc-model.cc

@@ -13,6 +13,7 @@
 #include "rawfile/raw_file_manager.h"
 #endif
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"