dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests pass on python 2 and 3

This commit is contained in:
thomwolf 2019-02-11 10:47:52 +01:00
parent 2071a9b86e
commit 0a9860daa7
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tests/tokenization_openai_test.py
View File

@ -17,7 +17,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera
import os
import unittest
import json
from io import open
from pytorch_pretrained_bert.tokenization_openai import OpenAIGPTTokenizer
@ -32,10 +31,10 @@ class OpenAIGPTTokenizationTest(unittest.TestCase):
"low</w>", "lowest</w>", "newer</w>", "wider</w>"]
vocab_tokens = dict(zip(vocab, range(len(vocab))))
merges = ["#version: 0.2", "l o", "lo w", "e r</w>", ""]
with open("/tmp/openai_tokenizer_vocab_test.json", "wb") as fp:
with open("/tmp/openai_tokenizer_vocab_test.json", "w") as fp:
json.dump(vocab_tokens, fp)
vocab_file = fp.name
with open("/tmp/openai_tokenizer_merges_test.txt", "w", encoding='utf-8') as fp:
with open("/tmp/openai_tokenizer_merges_test.txt", "w") as fp:
fp.write("\n".join(merges))
merges_file = fp.name