Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add context biasing for mobile #568

Merged
merged 6 commits into from
Feb 1, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,17 @@ class SherpaOnnx(
acceptWaveform(ptr, samples, sampleRate)

fun inputFinished() = inputFinished(ptr)
fun reset(recreate: Boolean = false) = reset(ptr, recreate = recreate)
fun reset(recreate: Boolean = false, hotWords: String = "") = reset(ptr, recreate, hotWords)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you also update

fun reset(recreate: Boolean = false) = reset(ptr, recreate = recreate)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
fun reset(recreate: Boolean = false, hotWords: String = "") = reset(ptr, recreate, hotWords)
fun reset(recreate: Boolean = false, hotwords: String = "") = reset(ptr, recreate, hotwords)

fun decode() = decode(ptr)
fun isEndpoint(): Boolean = isEndpoint(ptr)
fun isReady(): Boolean = isReady(ptr)

val text: String
get() = getText(ptr)

val tokens: Array<String>
get() = getTokens(ptr)

private external fun delete(ptr: Long)

private external fun new(
Expand All @@ -107,10 +110,11 @@ class SherpaOnnx(
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
private external fun inputFinished(ptr: Long)
private external fun getText(ptr: Long): String
private external fun reset(ptr: Long, recreate: Boolean)
private external fun reset(ptr: Long, recreate: Boolean, hotWords: String)
private external fun decode(ptr: Long)
private external fun isEndpoint(ptr: Long): Boolean
private external fun isReady(ptr: Long): Boolean
private external fun getTokens(ptr: Long): Array<String>

companion object {
init {
Expand Down
24 changes: 19 additions & 5 deletions sherpa-onnx/jni/jni.cc
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,22 @@ class SherpaOnnx {

bool IsReady() const { return recognizer_.IsReady(stream_.get()); }

void Reset(bool recreate) {
if (recreate) {
// If keywords is an empty string, it just recreates the decoding stream
// If keywords is not empty, it will create a new decoding stream with
// the given keywords appended to the default keywords.
void Reset(bool recreate, const std::string &keywords = {}) {
if (keywords.empty()) {
if (recreate) {
stream_ = recognizer_.CreateStream();
} else {
} else {
recognizer_.Reset(stream_.get());
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you indent the statements?

You can run

cd /path/to/sherpa-onnx
./scripts/check_style_cpplint.sh 
./scripts/check_style_cpplint.sh 1
./scripts/check_style_cpplint.sh 2

to check your code style.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you indent the statements?

You can run

cd /path/to/sherpa-onnx
./scripts/check_style_cpplint.sh 
./scripts/check_style_cpplint.sh 1
./scripts/check_style_cpplint.sh 2

to check your code style.

Done!
All code style checks passed

} else {
auto stream = recognizer_.CreateStream(keywords);
// Set new keywords failed, the stream_ will not be updated.
if (stream != nullptr) {
stream_ = std::move(stream);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you also use SHERPA_ONNX_LOGE() to print some logs here saying it failed to set the hotwords?

}
}
}

Expand Down Expand Up @@ -1509,9 +1520,12 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnxOffline_delete(

SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_reset(
JNIEnv *env, jobject /*obj*/, jlong ptr, jboolean recreate) {
JNIEnv *env, jobject /*obj*/, jlong ptr, jboolean recreate, jstring keywords) {
auto model = reinterpret_cast<sherpa_onnx::SherpaOnnx *>(ptr);
model->Reset(recreate);
const char *p_keywords = env->GetStringUTFChars(keywords, nullptr);
std::string keywords_str = p_keywords;
model->Reset(recreate, keywords_str);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
std::string keywords_str = p_keywords;
model->Reset(recreate, keywords_str);
model->Reset(recreate, p_keywords);

You can pass p_keywords directly to the function.

Copy link
Author

@ductranminh ductranminh Feb 1, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But p_keywords is const char * while the reset method accept std::string ? It's incorrect type

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But p_keywords is const char * while the reset method accept std::string ? It's incorrect type

Does it cause compiler errors? If yes, please post the error logs.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you're right, it works normally

env->ReleaseStringUTFChars(keywords, p_keywords);
}

SHERPA_ONNX_EXTERN_C
Expand Down
16 changes: 13 additions & 3 deletions swift-api-examples/SherpaOnnx.swift
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ class SherpaOnnxOnlineRecongitionResult {
class SherpaOnnxRecognizer {
/// A pointer to the underlying counterpart in C
let recognizer: OpaquePointer!
let stream: OpaquePointer!
var stream: OpaquePointer!

/// Constructor taking a model config
init(
Expand Down Expand Up @@ -237,8 +237,18 @@ class SherpaOnnxRecognizer {

/// Reset the recognizer, which clears the neural network model state
/// and the state for decoding.
func reset() {
Reset(recognizer, stream)
/// If hotWords is an empty string, it just recreates the decoding stream
/// If hotWords is not empty, it will create a new decoding stream with
/// the given hotWords appended to the default hotWords.
func reset(hotWords: String? = nil) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
func reset(hotWords: String? = nil) {
func reset(hotwords: String? = nil) {

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

guard let words = hotWords, !words.isEmpty else {
Reset(recognizer, stream)
return
}

words.withCString { cString in
stream = CreateOnlineStreamWithHotwords(recognizer, cString)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please free the previous stream to avoid memory leak.

}
}

/// Signal that no more audio samples would be available.
Expand Down
Loading