Fix `BertPreTokenizer` regex pattern

2023-05-01 22:34:19 +02:00 · 2023-05-01 22:34:19 +02:00 · 459ac7f4f9
parent 63ccc3dbac
commit 459ac7f4f9
1 changed files with 1 additions and 1 deletions
--- a/src/tokenizers.js
+++ b/src/tokenizers.js
@ -849,7 +849,7 @@ class BertPreTokenizer extends PreTokenizer {
    constructor(config) {
        super();
        // TODO use config
-        this.pattern = /\b\p{L}+\b|[^\s\p{L}]+/gu
+        this.pattern = /\b\p{L}+\b|[^\s\p{L}]/gu;
    }
    /**
     * Tokenizes a single text using the BERT pre-tokenization scheme.