Update tokenizer playground dependencies (#599)

* Update transformers.js version

* Update Token.jsx

examples/tokenizer-playground/package-lock.json

@@ -8,7 +8,7 @@
       "name": "tokenizer-playground",
       "version": "0.0.0",
       "dependencies": {
-        "@xenova/transformers": "^2.15.0",
+        "@xenova/transformers": "^2.15.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0"
       },
@@ -1086,11 +1086,11 @@
       }
     },
     "node_modules/@xenova/transformers": {
-      "version": "2.15.0",
-      "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.15.0.tgz",
-      "integrity": "sha512-e8pt+yLGSmwZnQR5Q/fq1NJ6fPr3+WKqxh/jF2PzfXFZ2KZsDdFQeCVlk8AnOADP4Aimlqy+Wp/xuws96/pX9A==",
+      "version": "2.15.1",
+      "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.15.1.tgz",
+      "integrity": "sha512-HX3kUZbr9v90PS/D2SyffGiFYQ6wQMbzwC1uLuCOA6nRSFOdr0TDnOTxfjS0RB6Phn6ThCTE1vX4n+NUsuobQg==",
       "dependencies": {
-        "@huggingface/jinja": "^0.1.0",
+        "@huggingface/jinja": "^0.1.3",
         "onnxruntime-web": "1.14.0",
         "sharp": "^0.32.0"
       },

examples/tokenizer-playground/package.json

@@ -10,7 +10,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@xenova/transformers": "^2.15.0",
+    "@xenova/transformers": "^2.15.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0"
   },

examples/tokenizer-playground/src/components/Token.jsx

@@ -1,3 +1,5 @@
+import { Fragment } from 'react';
+
 const COLOURS = [
     'bg-purple-300',
     'bg-green-300',
@@ -7,15 +9,15 @@ const COLOURS = [
 ]
 
 export function Token({ text, position, margin }) {
-    const isNormal = text !== '\n'
-
-    return (
-        isNormal ? (
-            <span
-            style={{marginLeft: margin}}
-            
-            className={`leading-5 ${COLOURS[position % COLOURS.length]}`}>
-                {text}
-            </span>) : <br />
-    )
+    const textWithLineBreaks = text.split('\n').map((line, index, array) => (
+        <Fragment key={index}>
+            {line}
+            {index !== array.length - 1 && <br />}
+        </Fragment>
+    ));
+    return (<span
+        style={{ marginLeft: margin }}
+        className={`leading-5 ${textWithLineBreaks.length === 1 ? 'inline-block ' : ''}${COLOURS[position % COLOURS.length]}`}>
+        {textWithLineBreaks}
+    </span>)
 }