dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1296 from danai-antoniou/add-duplicate-tokens-error 

Added ValueError for duplicates in list of added tokens
This commit is contained in:
Thomas Wolf 2019-10-03 17:06:17 -04:00 committed by GitHub
parent e1b2949ae6 a95158518d
commit 1569610f2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
transformers/tokenization_utils.py
View File

@ -512,7 +512,8 @@ class PreTrainedTokenizer(object):
for token in new_tokens:
assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode))
if token != self.unk_token and \
self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token):
self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \
token not in to_add_tokens:
to_add_tokens.append(token)
logger.info("Adding %s to the vocabulary", token)