Add `ErnieMTokenizer`

This commit is contained in:
Joshua Lochner 2024-03-24 23:42:22 +02:00
parent 52d65d7443
commit 75d7c09ae1
1 changed files with 3 additions and 0 deletions

View File

@ -4311,6 +4311,8 @@ export class VitsTokenizer extends PreTrainedTokenizer {
export class CohereTokenizer extends PreTrainedTokenizer { }
export class ErnieMTokenizer extends PreTrainedTokenizer { }
/**
* Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.
* The chosen tokenizer class is determined by the type specified in the tokenizer config.
@ -4364,6 +4366,7 @@ export class AutoTokenizer {
GemmaTokenizer,
Grok1Tokenizer,
CohereTokenizer,
ErnieMTokenizer,
// Base case:
PreTrainedTokenizer,