dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

This commit is contained in:
Ita Zaporozhets 2024-05-31 17:15:56 +02:00
parent d92822e045
commit 74e78f1720
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_utils_fast.py
View File

@ -894,7 +894,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
if getattr(self._tokenizer, "normalizer", None) == None:
return
curr_normalizer = json.loads(self._tokenizer.normalizer.__getstate__().decode('utf-8'))
if not curr_normalizer['normalizers']:
if 'normalizers' not in curr_normalizer:
return
prepend_normalizer = [n for n in curr_normalizer['normalizers'] if n['type'] == 'Prepend']
if prepend_normalizer: