dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

legacy to init the slow tokenizer when converting from slow was wrong (#30972)

This commit is contained in:
Arthur 2024-05-22 18:06:50 +02:00 committed by Ita Zaporozhets
parent 774f7295f2
commit d799d6715f
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/models/llama/tokenization_llama_fast.py
View File

@ -151,9 +151,6 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
self.legacy = legacy self.legacy = legacy
if add_prefix_space is not None: if add_prefix_space is not None:
logger.warning_once(
"You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers"
)
kwargs["from_slow"] = True kwargs["from_slow"] = True
super().__init__( super().__init__(
@ -166,6 +163,7 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
add_bos_token=add_bos_token, add_bos_token=add_bos_token,
add_eos_token=add_eos_token, add_eos_token=add_eos_token,
use_default_system_prompt=use_default_system_prompt, use_default_system_prompt=use_default_system_prompt,
legacy=legacy,
**kwargs, **kwargs,
) )
self._add_bos_token = add_bos_token self._add_bos_token = add_bos_token