parent
d6eaf4e6d2
commit
1c62e87b34
|
@ -21,7 +21,6 @@ import glob
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from io import open
|
||||
from typing import List
|
||||
|
||||
import tqdm
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
import logging
|
||||
import os
|
||||
from io import open
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
1
setup.py
1
setup.py
|
@ -33,7 +33,6 @@ To create the package for pypi.
|
|||
7. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
|
||||
|
||||
"""
|
||||
from io import open
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
|
|
@ -20,7 +20,6 @@ import copy
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from io import open
|
||||
|
||||
from .file_utils import CONFIG_NAME, cached_path, hf_bucket_url, is_remote_url
|
||||
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
import argparse
|
||||
import logging
|
||||
from io import open
|
||||
|
||||
import torch
|
||||
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
|
||||
import argparse
|
||||
import logging
|
||||
from io import open
|
||||
|
||||
import torch
|
||||
|
||||
|
|
|
@ -20,7 +20,6 @@ import logging
|
|||
import os
|
||||
import pickle
|
||||
import sys
|
||||
from io import open
|
||||
|
||||
import torch
|
||||
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from io import open
|
||||
|
||||
import numpy
|
||||
import torch
|
||||
|
|
|
@ -14,7 +14,6 @@ import logging
|
|||
import math
|
||||
import re
|
||||
import string
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_bert import BasicTokenizer
|
||||
|
||||
|
|
|
@ -14,7 +14,6 @@ import tempfile
|
|||
from contextlib import contextmanager
|
||||
from functools import partial, wraps
|
||||
from hashlib import sha256
|
||||
from io import open
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import boto3
|
||||
|
|
|
@ -19,7 +19,6 @@ import copy
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from io import open
|
||||
|
||||
from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP
|
||||
from .file_utils import (
|
||||
|
|
|
@ -20,7 +20,6 @@ import json
|
|||
import logging
|
||||
import math
|
||||
import os
|
||||
from io import open
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
|
|
@ -19,7 +19,6 @@ import collections
|
|||
import logging
|
||||
import os
|
||||
import unicodedata
|
||||
from io import open
|
||||
|
||||
from .tokenization_utils import PreTrainedTokenizer
|
||||
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from io import open
|
||||
|
||||
import regex as re
|
||||
|
||||
|
|
|
@ -19,7 +19,6 @@ import json
|
|||
import logging
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from io import open
|
||||
|
||||
import regex as re
|
||||
|
||||
|
|
|
@ -19,7 +19,6 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
from io import open
|
||||
|
||||
from .tokenization_bert import BasicTokenizer
|
||||
from .tokenization_utils import PreTrainedTokenizer
|
||||
|
|
|
@ -23,7 +23,6 @@ import logging
|
|||
import os
|
||||
import pickle
|
||||
from collections import Counter, OrderedDict
|
||||
from io import open
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
|
|
@ -21,7 +21,6 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
from io import open
|
||||
|
||||
from .file_utils import cached_path, hf_bucket_url, is_remote_url, is_tf_available, is_torch_available
|
||||
|
||||
|
|
|
@ -21,7 +21,6 @@ import os
|
|||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
from io import open
|
||||
|
||||
import sacremoses as sm
|
||||
|
||||
|
|
|
@ -19,7 +19,6 @@ import collections
|
|||
import json
|
||||
import logging
|
||||
import math
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
|
||||
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_bert import VOCAB_FILES_NAMES, XxxTokenizer
|
||||
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
import collections
|
||||
import logging
|
||||
import os
|
||||
from io import open
|
||||
|
||||
from .tokenization_utils import PreTrainedTokenizer
|
||||
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_bert import (
|
||||
VOCAB_FILES_NAMES,
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_bert import WordpieceTokenizer
|
||||
from transformers.tokenization_bert_japanese import (
|
||||
|
|
|
@ -18,7 +18,6 @@ import os
|
|||
import pickle
|
||||
import shutil
|
||||
import tempfile
|
||||
from io import open
|
||||
|
||||
|
||||
class TokenizerTesterMixin:
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
import json
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer
|
||||
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
import json
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_gpt2 import VOCAB_FILES_NAMES, GPT2Tokenizer
|
||||
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
import json
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers.tokenization_roberta import VOCAB_FILES_NAMES, RobertaTokenizer
|
||||
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
import os
|
||||
import unittest
|
||||
from io import open
|
||||
|
||||
from transformers import is_torch_available
|
||||
|
||||
|
|
Loading…
Reference in New Issue