dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Test correct tokenizers after default switch (#3003)

This commit is contained in:
Lysandre Debut 2020-02-24 18:45:53 -05:00 committed by GitHub
parent 3716c3d8af
commit b90745c590
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_tokenization_auto.py
View File

@ -101,5 +101,5 @@ class AutoTokenizerTest(unittest.TestCase):
self.assertFalse(issubclass(child_model_fast, parent_model_fast))
def test_from_pretrained_use_fast_toggle(self):
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizerFast)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False), BertTokenizer)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizer)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=True), BertTokenizerFast)