fix(llmb): minor changes

rustformers · Dec 12, 2023 · 86b068c · 86b068c
1 parent e46a433
commit 86b068c
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 22 deletions.
diff --git a/crates/llm-base/src/tokenizer/embedded.rs b/crates/llm-base/src/tokenizer/embedded.rs
@@ -172,16 +172,10 @@ impl EmbeddedTokenizer {
         match self.model {
             GgufEmbeddedTokenizerModel::Llama => {
                 let text = escape_whitespace(format!(" {}", text).as_bytes());
-
-                let _token_ids: Vec<_> = TokenizerSpm::new(self)
-                    .tokenize(&text)
-                    .into_iter()
-                    .map(|id| {
-                        // TODO: see if this can be made more efficient
-                        output.push((self.id_to_token[id as usize].text.clone(), id));
-                        (self.id_to_token[id as usize].text.clone(), id)
-                    })
-                    .collect();
+                for id in TokenizerSpm::new(self).tokenize(&text) {
+                    // TODO: see if this can be made more efficient
+                    output.push((self.id_to_token[id as usize].text.clone(), id));
+                }
                 Ok(output)
             }
             _ => unimplemented!(),

diff --git a/crates/llm-base/src/tokenizer/mod.rs b/crates/llm-base/src/tokenizer/mod.rs
@@ -79,10 +79,6 @@ impl Display for HuggingFaceTokenizerErrorSource {
     }
 }
 
-/// At the time of writing, the embedded tokenizer is not enabled as it has
-/// some bugs. We're just not enabling the option while it's broken.
-const EMBEDDED_TOKENIZER_ENABLED: bool = true;
-
 #[derive(Clone, Debug, PartialEq)]
 /// The source of a tokenizer.
 pub enum TokenizerSource {
@@ -140,13 +136,7 @@ impl TokenizerSource {
                 if let Ok(hf) = gguf.metadata.get_str("tokenizer.huggingface.json") {
                     Ok(Self::load_huggingface_json(hf)?)
                 } else if EmbeddedTokenizer::is_present_in_metadata(&gguf.metadata) {
-                    if EMBEDDED_TOKENIZER_ENABLED {
-                        Ok(EmbeddedTokenizer::from_metadata(&gguf.metadata)?.into())
-                    } else {
-                        Err(TokenizerLoadError::NoSupportedTokenizersFound {
-                            unsupported_tokenizers: vec!["embedded".to_owned()],
-                        })
-                    }
+                    Ok(EmbeddedTokenizer::from_metadata(&gguf.metadata)?.into())
                 } else {
                     Err(TokenizerLoadError::NoSupportedTokenizersFound {
                         unsupported_tokenizers: vec![],

diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs
@@ -116,7 +116,7 @@ macro_rules! define_models {
 
         impl ModelArchitecture {
             /// All available model architectures
-            pub const ALL: &[Self] = &[
+            pub const ALL: &'static [Self] = &[
                 $(
                     #[cfg(feature = $model_lowercase_str)]
                     Self::$model_pascalcase,