Add `ErnieMTokenizer`
This commit is contained in:
parent
52d65d7443
commit
75d7c09ae1
|
@ -4311,6 +4311,8 @@ export class VitsTokenizer extends PreTrainedTokenizer {
|
||||||
|
|
||||||
export class CohereTokenizer extends PreTrainedTokenizer { }
|
export class CohereTokenizer extends PreTrainedTokenizer { }
|
||||||
|
|
||||||
|
export class ErnieMTokenizer extends PreTrainedTokenizer { }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.
|
* Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.
|
||||||
* The chosen tokenizer class is determined by the type specified in the tokenizer config.
|
* The chosen tokenizer class is determined by the type specified in the tokenizer config.
|
||||||
|
@ -4364,6 +4366,7 @@ export class AutoTokenizer {
|
||||||
GemmaTokenizer,
|
GemmaTokenizer,
|
||||||
Grok1Tokenizer,
|
Grok1Tokenizer,
|
||||||
CohereTokenizer,
|
CohereTokenizer,
|
||||||
|
ErnieMTokenizer,
|
||||||
|
|
||||||
// Base case:
|
// Base case:
|
||||||
PreTrainedTokenizer,
|
PreTrainedTokenizer,
|
||||||
|
|
Loading…
Reference in New Issue