Fix barthez tokenizer (#9562)
This commit is contained in:
parent
247a7b2029
commit
245cdb469d
|
@ -199,7 +199,6 @@ TOKENIZER_MAPPING = OrderedDict(
|
|||
(AlbertConfig, (AlbertTokenizer, AlbertTokenizerFast)),
|
||||
(CamembertConfig, (CamembertTokenizer, CamembertTokenizerFast)),
|
||||
(PegasusConfig, (PegasusTokenizer, PegasusTokenizerFast)),
|
||||
(MBartConfig, (BarthezTokenizer, BarthezTokenizerFast)),
|
||||
(MBartConfig, (MBartTokenizer, MBartTokenizerFast)),
|
||||
(XLMRobertaConfig, (XLMRobertaTokenizer, XLMRobertaTokenizerFast)),
|
||||
(MarianConfig, (MarianTokenizer, None)),
|
||||
|
@ -243,6 +242,7 @@ NO_CONFIG_TOKENIZER = [
|
|||
HerbertTokenizer,
|
||||
HerbertTokenizerFast,
|
||||
PhobertTokenizer,
|
||||
BarthezTokenizer,
|
||||
]
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue