Fix typo in documentation. toto -> to
This commit is contained in:
parent
26db31e0c0
commit
041a901f32
|
@ -184,7 +184,7 @@ class GPT2Tokenizer(PreTrainedTokenizer):
|
||||||
""" Tokenize a string.
|
""" Tokenize a string.
|
||||||
Args:
|
Args:
|
||||||
- add_prefix_space (boolean, default False):
|
- add_prefix_space (boolean, default False):
|
||||||
Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers.
|
Begin the sentence with at least one space to get invariance to word order in GPT-2 (and RoBERTa) tokenizers.
|
||||||
"""
|
"""
|
||||||
if add_prefix_space:
|
if add_prefix_space:
|
||||||
text = ' ' + text
|
text = ' ' + text
|
||||||
|
|
Loading…
Reference in New Issue