Patch recursive import (#11812)
This commit is contained in:
parent
bd9871657b
commit
1b652295c5
|
@ -25,7 +25,6 @@ from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers,
|
|||
from tokenizers.models import BPE, Unigram, WordPiece
|
||||
|
||||
from .file_utils import requires_backends
|
||||
from .models.roformer.tokenization_utils import JiebaPreTokenizer
|
||||
|
||||
|
||||
class SentencePieceExtractor:
|
||||
|
@ -299,6 +298,8 @@ class RobertaConverter(Converter):
|
|||
|
||||
class RoFormerConverter(Converter):
|
||||
def converted(self) -> Tokenizer:
|
||||
from .models.roformer.tokenization_utils import JiebaPreTokenizer
|
||||
|
||||
vocab = self.original_tokenizer.vocab
|
||||
tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token)))
|
||||
|
||||
|
|
|
@ -12,6 +12,9 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
isort:skip_file
|
||||
"""
|
||||
import os
|
||||
import pickle
|
||||
import tempfile
|
||||
|
@ -20,12 +23,14 @@ from typing import Callable, Optional
|
|||
|
||||
import numpy as np
|
||||
|
||||
# Ensure there are no circular imports when importing the parent class
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from transformers import (
|
||||
BatchEncoding,
|
||||
BertTokenizer,
|
||||
BertTokenizerFast,
|
||||
PreTrainedTokenizer,
|
||||
PreTrainedTokenizerFast,
|
||||
TensorType,
|
||||
TokenSpan,
|
||||
is_tokenizers_available,
|
||||
|
|
Loading…
Reference in New Issue