dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix the `bert-base-cased` tokenizer configuration test (#29105) 

Fix test
This commit is contained in:
Lysandre Debut 2024-02-19 13:23:25 +01:00 committed by GitHub
parent 593230f0a1
commit 9830858671
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/models/auto/test_tokenization_auto.py
View File

@ -223,7 +223,7 @@ class AutoTokenizerTest(unittest.TestCase):
config = get_tokenizer_config("google-bert/bert-base-cased")
_ = config.pop("_commit_hash", None)
# If we ever update google-bert/bert-base-cased tokenizer config, this dict here will need to be updated.
self.assertEqual(config, {"do_lower_case": False})
self.assertEqual(config, {"do_lower_case": False, "model_max_length": 512})
# This model does not have a tokenizer_config so we get back an empty dict.
config = get_tokenizer_config(SMALL_MODEL_IDENTIFIER)