Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add context biasing for mobile #568

Merged
merged 6 commits into from
Feb 1, 2024
Merged

Add context biasing for mobile #568

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c0320e1
Implement reset with hot words on android
Feb 1, 2024
c646f05
Implement ios reset with hot words
Feb 1, 2024
9b88ea4
Default hot words is empty
Feb 1, 2024
0efe736
Clean up codes
Feb 1, 2024
a471f47
Clean up
Feb 1, 2024
5ed7e79
Correct codestyle
Feb 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,17 @@ class SherpaOnnx(
acceptWaveform(ptr, samples, sampleRate)

fun inputFinished() = inputFinished(ptr)
fun reset(recreate: Boolean = false) = reset(ptr, recreate = recreate)
fun reset(recreate: Boolean = false, hotwords: String = "") = reset(ptr, recreate, hotwords)
fun decode() = decode(ptr)
fun isEndpoint(): Boolean = isEndpoint(ptr)
fun isReady(): Boolean = isReady(ptr)

val text: String
get() = getText(ptr)

val tokens: Array<String>
get() = getTokens(ptr)

private external fun delete(ptr: Long)

private external fun new(
Expand All @@ -107,10 +110,11 @@ class SherpaOnnx(
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
private external fun inputFinished(ptr: Long)
private external fun getText(ptr: Long): String
private external fun reset(ptr: Long, recreate: Boolean)
private external fun reset(ptr: Long, recreate: Boolean, hotwords: String)
private external fun decode(ptr: Long)
private external fun isEndpoint(ptr: Long): Boolean
private external fun isReady(ptr: Long): Boolean
private external fun getTokens(ptr: Long): Array<String>

companion object {
init {
Expand Down
8 changes: 6 additions & 2 deletions android/SherpaOnnx2Pass/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,14 +120,17 @@ class SherpaOnnx(
acceptWaveform(ptr, samples, sampleRate)

fun inputFinished() = inputFinished(ptr)
fun reset(recreate: Boolean = false) = reset(ptr, recreate = recreate)
fun reset(recreate: Boolean = false, hotwords: String = "") = reset(ptr, recreate, hotwords)
fun decode() = decode(ptr)
fun isEndpoint(): Boolean = isEndpoint(ptr)
fun isReady(): Boolean = isReady(ptr)

val text: String
get() = getText(ptr)

val tokens: Array<String>
get() = getTokens(ptr)

private external fun delete(ptr: Long)

private external fun new(
Expand All @@ -142,10 +145,11 @@ class SherpaOnnx(
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
private external fun inputFinished(ptr: Long)
private external fun getText(ptr: Long): String
private external fun reset(ptr: Long, recreate: Boolean)
private external fun reset(ptr: Long, recreate: Boolean, hotwords: String)
private external fun decode(ptr: Long)
private external fun isEndpoint(ptr: Long): Boolean
private external fun isReady(ptr: Long): Boolean
private external fun getTokens(ptr: Long): Array<String>

companion object {
init {
Expand Down
26 changes: 21 additions & 5 deletions sherpa-onnx/jni/jni.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,24 @@ class SherpaOnnx {

bool IsReady() const { return recognizer_.IsReady(stream_.get()); }

void Reset(bool recreate) {
if (recreate) {
// If keywords is an empty string, it just recreates the decoding stream
// If keywords is not empty, it will create a new decoding stream with
// the given keywords appended to the default keywords.
void Reset(bool recreate, const std::string &keywords = {}) {
if (keywords.empty()) {
if (recreate) {
stream_ = recognizer_.CreateStream();
} else {
} else {
recognizer_.Reset(stream_.get());
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you indent the statements?

You can run

cd /path/to/sherpa-onnx
./scripts/check_style_cpplint.sh 
./scripts/check_style_cpplint.sh 1
./scripts/check_style_cpplint.sh 2

to check your code style.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you indent the statements?

You can run

cd /path/to/sherpa-onnx
./scripts/check_style_cpplint.sh 
./scripts/check_style_cpplint.sh 1
./scripts/check_style_cpplint.sh 2

to check your code style.

Done!
All code style checks passed

} else {
auto stream = recognizer_.CreateStream(keywords);
// Set new keywords failed, the stream_ will not be updated.
if (stream != nullptr) {
stream_ = std::move(stream);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you also use SHERPA_ONNX_LOGE() to print some logs here saying it failed to set the hotwords?

} else {
SHERPA_ONNX_LOGE("Failed to set keywords: %s", keywords.c_str());
}
}
}

Expand Down Expand Up @@ -1509,9 +1522,12 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnxOffline_delete(

SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_SherpaOnnx_reset(
JNIEnv *env, jobject /*obj*/, jlong ptr, jboolean recreate) {
JNIEnv *env, jobject /*obj*/, jlong ptr, jboolean recreate, jstring keywords) {
auto model = reinterpret_cast<sherpa_onnx::SherpaOnnx *>(ptr);
model->Reset(recreate);
const char *p_keywords = env->GetStringUTFChars(keywords, nullptr);
std::string keywords_str = p_keywords;
model->Reset(recreate, keywords_str);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
std::string keywords_str = p_keywords;
model->Reset(recreate, keywords_str);
model->Reset(recreate, p_keywords);

You can pass p_keywords directly to the function.

Copy link
Author

@ductranminh ductranminh Feb 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But p_keywords is const char * while the reset method accept std::string ? It's incorrect type

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But p_keywords is const char * while the reset method accept std::string ? It's incorrect type

Does it cause compiler errors? If yes, please post the error logs.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you're right, it works normally

env->ReleaseStringUTFChars(keywords, p_keywords);
}

SHERPA_ONNX_EXTERN_C
Expand Down
21 changes: 18 additions & 3 deletions swift-api-examples/SherpaOnnx.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ class SherpaOnnxOnlineRecongitionResult {
class SherpaOnnxRecognizer {
/// A pointer to the underlying counterpart in C
let recognizer: OpaquePointer!
let stream: OpaquePointer!
var stream: OpaquePointer!

/// Constructor taking a model config
init(
Expand Down Expand Up @@ -237,8 +237,23 @@ class SherpaOnnxRecognizer {

/// Reset the recognizer, which clears the neural network model state
/// and the state for decoding.
func reset() {
Reset(recognizer, stream)
/// If hotWords is an empty string, it just recreates the decoding stream
/// If hotWords is not empty, it will create a new decoding stream with
/// the given hotWords appended to the default hotWords.
func reset(hotWords: String? = nil) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
func reset(hotWords: String? = nil) {
func reset(hotwords: String? = nil) {

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

guard let words = hotWords, !words.isEmpty else {
Reset(recognizer, stream)
return
}

words.withCString { cString in
let newStream = CreateOnlineStreamWithHotwords(recognizer, cString)
// lock while release and replace stream
objc_sync_enter(self)
DestroyOnlineStream(stream)
stream = newStream
objc_sync_enter(self)
}
}

/// Signal that no more audio samples would be available.
Expand Down
Loading