Add warning to `ErnieMTokenizer` class
This commit is contained in:
parent
9bf393304e
commit
62da9cec78
|
@ -4311,7 +4311,12 @@ export class VitsTokenizer extends PreTrainedTokenizer {
|
|||
|
||||
export class CohereTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
export class ErnieMTokenizer extends PreTrainedTokenizer { }
|
||||
export class ErnieMTokenizer extends PreTrainedTokenizer {
|
||||
constructor(tokenizerJSON, tokenizerConfig) {
|
||||
super(tokenizerJSON, tokenizerConfig);
|
||||
console.warn('WARNING: `ErnieMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.
|
||||
|
|
Loading…
Reference in New Issue