diff --git a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h index 2f49613503..ca0b5522ec 100644 --- a/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-ctc-impl.h @@ -45,8 +45,10 @@ static OfflineRecognitionResult Convert(const OfflineCtcDecoderResult &src, auto sym = sym_table[src.tokens[i]]; text.append(sym); - if (sym.size() == 1 && sym[0] != ' ') { + if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { // for byte bpe models + // (but don't rewrite printable characters 0x20..0x7e, + // which collide with standard BPE units) std::ostringstream os; os << "<0x" << std::hex << std::uppercase << (static_cast(sym[0]) & 0xff) << ">"; diff --git a/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h b/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h index 084d39fe9f..de9f6263b0 100644 --- a/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/offline-recognizer-transducer-impl.h @@ -46,8 +46,10 @@ static OfflineRecognitionResult Convert( auto sym = sym_table[i]; text.append(sym); - if (sym.size() == 1 && sym[0] != ' ') { - // for byte bpe models + if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { + // for byte bpe models, + // (but don't rewrite printable characters 0x20..0x7e, + // which collide with standard BPE units) std::ostringstream os; os << "<0x" << std::hex << std::uppercase << (static_cast(sym[0]) & 0xff) << ">"; diff --git a/sherpa-onnx/csrc/online-recognizer-ctc-impl.h b/sherpa-onnx/csrc/online-recognizer-ctc-impl.h index fa81d6590c..5697a77e8a 100644 --- a/sherpa-onnx/csrc/online-recognizer-ctc-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-ctc-impl.h @@ -38,8 +38,10 @@ static OnlineRecognizerResult Convert(const OnlineCtcDecoderResult &src, r.text.append(sym); - if (sym.size() == 1 && sym[0] != ' ') { + if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { // for byte bpe models + // (but don't rewrite printable characters 0x20..0x7e, + // which collide with standard BPE units) std::ostringstream os; os << "<0x" << std::hex << std::uppercase << (static_cast(sym[0]) & 0xff) << ">"; diff --git a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h index 8b193a67ea..b3f31cdd0e 100644 --- a/sherpa-onnx/csrc/online-recognizer-transducer-impl.h +++ b/sherpa-onnx/csrc/online-recognizer-transducer-impl.h @@ -50,8 +50,10 @@ static OnlineRecognizerResult Convert(const OnlineTransducerDecoderResult &src, r.text.append(sym); - if (sym.size() == 1 && sym[0] != ' ') { + if (sym.size() == 1 && (sym[0] < 0x20 || sym[0] > 0x7e)) { // for byte bpe models + // (but don't rewrite printable characters 0x20..0x7e, + // which collide with standard BPE units) std::ostringstream os; os << "<0x" << std::hex << std::uppercase << (static_cast(sym[0]) & 0xff) << ">";