diff --git a/src/tokenizers.js b/src/tokenizers.js index b750b04..9b3505b 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -4311,7 +4311,12 @@ export class VitsTokenizer extends PreTrainedTokenizer { export class CohereTokenizer extends PreTrainedTokenizer { } -export class ErnieMTokenizer extends PreTrainedTokenizer { } +export class ErnieMTokenizer extends PreTrainedTokenizer { + constructor(tokenizerJSON, tokenizerConfig) { + super(tokenizerJSON, tokenizerConfig); + console.warn('WARNING: `ErnieMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.') + } +} /** * Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.