Skip to content

Commit

Permalink
Merge pull request #11 from WorksApplications/fix/pretokenizer
Browse files Browse the repository at this point in the history
fix import
  • Loading branch information
t-yamamura authored Jul 12, 2021
2 parents 60ef00f + 7d3e2fc commit ced5a47
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from tokenizers.implementations import BertWordPieceTokenizer
from tokenizers.implementations.base_tokenizer import BaseTokenizer

from . import CustomPreTokenizer
from .sudachipy_pretokenizer import CustomPreTokenizer


CPT = TypeVar('CPT', bound=CustomPreTokenizer)
Expand Down
2 changes: 1 addition & 1 deletion sudachitra/pretokenizer/sudachipy_pretokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def __init__(
**kwargs:
Sudachi dictionary parameters.
"""
SudachipyWordTokenizer.__init__(split_mode=split_mode, dict_type=dict_type, **kwargs)
SudachipyWordTokenizer.__init__(self, split_mode=split_mode, dict_type=dict_type, **kwargs)
self.word_form_type = word_form_type
self.word_formatter = WORD_FORM_TYPES[self.word_form_type] if self.word_form_type != "surface" else None

Expand Down

0 comments on commit ced5a47

Please sign in to comment.