From 62da9cec78debb99f969af534b17ad6e0f971349 Mon Sep 17 00:00:00 2001
From: Joshua Lochner <admin@xenova.com>
Date: Mon, 25 Mar 2024 00:58:36 +0200
Subject: [PATCH] Add warning to `ErnieMTokenizer` class

---
 src/tokenizers.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/tokenizers.js b/src/tokenizers.js
index b750b04..9b3505b 100644
--- a/src/tokenizers.js
+++ b/src/tokenizers.js
@@ -4311,7 +4311,12 @@ export class VitsTokenizer extends PreTrainedTokenizer {
 
 export class CohereTokenizer extends PreTrainedTokenizer { }
 
-export class ErnieMTokenizer extends PreTrainedTokenizer { }
+export class ErnieMTokenizer extends PreTrainedTokenizer {
+    constructor(tokenizerJSON, tokenizerConfig) {
+        super(tokenizerJSON, tokenizerConfig);
+        console.warn('WARNING: `ErnieMTokenizer` is not yet supported by Hugging Face\'s "fast" tokenizers library. Therefore, you may experience slightly inaccurate results.')
+    }
+}
 
 /**
  * Helper class which is used to instantiate pretrained tokenizers with the `from_pretrained` function.