dnrops
/
transformers.js
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add warning to `ErnieMTokenizer` class

This commit is contained in:
Joshua Lochner 2024-03-25 00:58:36 +02:00
parent 9bf393304e
commit 62da9cec78
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/tokenizers.js
View File

@ -4311,7 +4311,12 @@ export class VitsTokenizer extends PreTrainedTokenizer {
export class CohereTokenizer extends PreTrainedTokenizer { }
export class ErnieMTokenizer extends PreTrainedTokenizer { }
export class ErnieMTokenizer extends PreTrainedTokenizer {
constructor(tokenizerJSON, tokenizerConfig) {
super(tokenizerJSON, tokenizerConfig);
console.warn('WARNING: `ErnieMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.')
}
}
/**
* Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.