dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

let's see

This commit is contained in:
Arthur Zucker 2024-05-17 16:31:55 +02:00
parent d1042545af
commit a7500bc2de
1 changed files with 0 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/models/llama/test_tokenization_llama.py
View File

@ -783,9 +783,5 @@ class CommonSpmIntegrationTests(unittest.TestCase):
with self.subTest(tokenizer=type(tokenizer).__name__, mode="encode "):
self.assertEqual(input_ids, expected_input_ids)
tokens = tokenizer.tokenize(text)
with self.subTest(tokenizer=type(tokenizer).__name__,mode="tokenize"):
self.assertEqual(tokens, expected_tokens)
with self.subTest(tokenizer=type(tokenizer).__name__,mode="decode "):
self.assertEqual(tokenizer.decode(tokenizer.encode(text,add_special_tokens=False),spaces_between_special_tokens=False),text)