actually use the config passed in to the token_enforcer

noamgat · Nov 28, 2024 · b6abea1 · b6abea1
1 parent 764e05c
commit b6abea1
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/lmformatenforcer/tokenenforcer.py b/lmformatenforcer/tokenenforcer.py
@@ -58,8 +58,7 @@ def __init__(self, tokenizer_data: TokenEnforcerTokenizerData, parser: Character
                 if decoded == '"':
                     self.allowed_token_cache["superfast"] = [token_idx]
 
-        config = CharacterLevelParserConfig(alphabet=tokenizer_data.tokenizer_alphabet)
-        parser.config = config
+        parser.config.alphabet = tokenizer_data.tokenizer_alphabet
 
     def get_allowed_tokens(self, token_sequence: List[int]) -> List[int]:
         """