dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better dummies (#15148) 

* Better dummies

* See if this fixes the issue

* Fix quality

* Style

* Add doc for DummyObject
This commit is contained in:
Sylvain Gugger 2022-01-14 10:59:41 -05:00 committed by GitHub
parent b212ff9f49
commit 1b730c3d11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 3287 additions and 6876 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/transformers/file_utils.py
View File

@ -831,6 +831,18 @@ def requires_backends(obj, backends):
raise ImportError("".join([BACKENDS_MAPPING[backend][1].format(name) for backend in backends]))
class DummyObject(type):
"""
Metaclass for the dummy objects. Any class inheriting from it will return the ImportError generated by
`requires_backend` each time a user tries to access any method of that class.
"""
def __getattr__(cls, key):
if key.startswith("_"):
return super().__getattr__(cls, key)
requires_backends(cls, cls._backends)
def add_start_docstrings(*docstr):
def docstring_decorator(fn):
fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "")

1188
src/transformers/utils/dummy_flax_objects.py
View File

File diff suppressed because it is too large Load Diff

5109
src/transformers/utils/dummy_pt_objects.py
View File

File diff suppressed because it is too large Load Diff

142
src/transformers/utils/dummy_pytorch_quantization_and_torch_objects.py
View File

@ -1,120 +1,78 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
QDQBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None
class QDQBertForMaskedLM:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForMaskedLM(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForMultipleChoice:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForNextSentencePrediction:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForQuestionAnswering:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForSequenceClassification:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForTokenClassification:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertLayer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertLMHeadModel:
class QDQBertForMultipleChoice(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForNextSentencePrediction(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
class QDQBertModel:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForQuestionAnswering(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
class QDQBertPreTrainedModel:
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["pytorch_quantization", "torch"])
def forward(self, *args, **kwargs):
class QDQBertForSequenceClassification(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertForTokenClassification(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertLayer(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertLMHeadModel(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertModel(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])
class QDQBertPreTrainedModel(metaclass=DummyObject):
_backends = ["pytorch_quantization", "torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["pytorch_quantization", "torch"])

50
src/transformers/utils/dummy_scatter_objects.py
View File

@ -1,69 +1,45 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = None
class TapasForMaskedLM:
class TapasForMaskedLM(metaclass=DummyObject):
_backends = ["scatter"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["scatter"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["scatter"])
def forward(self, *args, **kwargs):
requires_backends(self, ["scatter"])
class TapasForQuestionAnswering(metaclass=DummyObject):
_backends = ["scatter"]
class TapasForQuestionAnswering:
def __init__(self, *args, **kwargs):
requires_backends(self, ["scatter"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["scatter"])
def forward(self, *args, **kwargs):
requires_backends(self, ["scatter"])
class TapasForSequenceClassification(metaclass=DummyObject):
_backends = ["scatter"]
class TapasForSequenceClassification:
def __init__(self, *args, **kwargs):
requires_backends(self, ["scatter"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["scatter"])
def forward(self, *args, **kwargs):
requires_backends(self, ["scatter"])
class TapasModel(metaclass=DummyObject):
_backends = ["scatter"]
class TapasModel:
def __init__(self, *args, **kwargs):
requires_backends(self, ["scatter"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["scatter"])
def forward(self, *args, **kwargs):
requires_backends(self, ["scatter"])
class TapasPreTrainedModel(metaclass=DummyObject):
_backends = ["scatter"]
class TapasPreTrainedModel:
def __init__(self, *args, **kwargs):
requires_backends(self, ["scatter"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["scatter"])
def forward(self, *args, **kwargs):
requires_backends(self, ["scatter"])
def load_tf_weights_in_tapas(*args, **kwargs):
requires_backends(load_tf_weights_in_tapas, ["scatter"])

11
src/transformers/utils/dummy_sentencepiece_and_speech_objects.py
View File

@ -1,11 +1,10 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
class Speech2TextProcessor:
class Speech2TextProcessor(metaclass=DummyObject):
_backends = ["sentencepiece", "speech"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece", "speech"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece", "speech"])

3
src/transformers/utils/dummy_sentencepiece_and_tokenizers_objects.py
View File

@ -1,5 +1,6 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
SLOW_TO_FAST_CONVERTERS = None

137
src/transformers/utils/dummy_sentencepiece_objects.py
View File

@ -1,200 +1,157 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
class AlbertTokenizer:
class AlbertTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class BarthezTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class BarthezTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class BartphoTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class BartphoTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class BertGenerationTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class BertGenerationTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class BigBirdTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class BigBirdTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class CamembertTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class CamembertTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class DebertaV2Tokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class DebertaV2Tokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class LayoutXLMTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class LayoutXLMTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class M2M100Tokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class M2M100Tokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class MarianTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class MarianTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class MBart50Tokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class MBart50Tokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class MBartTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class MBartTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class MLukeTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class MLukeTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class MT5Tokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class MT5Tokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class PegasusTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class PegasusTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class ReformerTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class ReformerTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class RemBertTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class RemBertTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class Speech2TextTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class Speech2TextTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class T5Tokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class T5Tokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class XLMProphetNetTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class XLMProphetNetTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class XLMRobertaTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class XLMRobertaTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])
class XLNetTokenizer(metaclass=DummyObject):
_backends = ["sentencepiece"]
class XLNetTokenizer:
def __init__(self, *args, **kwargs):
requires_backends(self, ["sentencepiece"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["sentencepiece"])

7
src/transformers/utils/dummy_speech_objects.py
View File

@ -1,7 +1,10 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
class Speech2TextFeatureExtractor:
class Speech2TextFeatureExtractor(metaclass=DummyObject):
_backends = ["speech"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["speech"])

3029
src/transformers/utils/dummy_tf_objects.py
View File

File diff suppressed because it is too large Load Diff

41
src/transformers/utils/dummy_timm_and_vision_objects.py
View File

@ -1,53 +1,34 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
DETR_PRETRAINED_MODEL_ARCHIVE_LIST = None
class DetrForObjectDetection:
class DetrForObjectDetection(metaclass=DummyObject):
_backends = ["timm", "vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["timm", "vision"])
def forward(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
class DetrForSegmentation(metaclass=DummyObject):
_backends = ["timm", "vision"]
class DetrForSegmentation:
def __init__(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["timm", "vision"])
def forward(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
class DetrModel(metaclass=DummyObject):
_backends = ["timm", "vision"]
class DetrModel:
def __init__(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["timm", "vision"])
def forward(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
class DetrPreTrainedModel(metaclass=DummyObject):
_backends = ["timm", "vision"]
class DetrPreTrainedModel:
def __init__(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["timm", "vision"])
def forward(self, *args, **kwargs):
requires_backends(self, ["timm", "vision"])

269
src/transformers/utils/dummy_tokenizers_objects.py
View File

@ -1,398 +1,311 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
class AlbertTokenizerFast:
class AlbertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class BartTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class BartTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class BarthezTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class BarthezTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class BertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class BertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class BigBirdTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class BigBirdTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class BlenderbotTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class BlenderbotTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class BlenderbotSmallTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class BlenderbotSmallTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class CamembertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class CamembertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class CLIPTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class CLIPTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class ConvBertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class ConvBertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class DebertaTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class DebertaTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class DistilBertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class DistilBertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class DPRContextEncoderTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class DPRContextEncoderTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class DPRQuestionEncoderTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class DPRQuestionEncoderTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class DPRReaderTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class DPRReaderTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class ElectraTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class ElectraTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class FNetTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class FNetTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class FunnelTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class FunnelTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class GPT2TokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class GPT2TokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class HerbertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class HerbertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class LayoutLMTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class LayoutLMTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class LayoutLMv2TokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class LayoutLMv2TokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class LayoutXLMTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class LayoutXLMTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class LEDTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class LEDTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class LongformerTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class LongformerTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class LxmertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class LxmertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class MBartTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class MBartTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class MBart50TokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class MBart50TokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class MobileBertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class MobileBertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class MPNetTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class MPNetTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class MT5TokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class MT5TokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class OpenAIGPTTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class OpenAIGPTTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class PegasusTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class PegasusTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class ReformerTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class ReformerTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class RemBertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class RemBertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class RetriBertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class RetriBertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class RobertaTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class RobertaTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class RoFormerTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class RoFormerTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class SplinterTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class SplinterTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class SqueezeBertTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class SqueezeBertTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class T5TokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class T5TokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class XLMRobertaTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class XLMRobertaTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class XLNetTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class XLNetTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])
class PreTrainedTokenizerFast(metaclass=DummyObject):
_backends = ["tokenizers"]
class PreTrainedTokenizerFast:
def __init__(self, *args, **kwargs):
requires_backends(self, ["tokenizers"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["tokenizers"])

87
src/transformers/utils/dummy_vision_objects.py
View File

@ -1,79 +1,94 @@
# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..file_utils import requires_backends
# flake8: noqa
from ..file_utils import DummyObject, requires_backends
class ImageFeatureExtractionMixin:
class ImageFeatureExtractionMixin(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class BeitFeatureExtractor:
class BeitFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class CLIPFeatureExtractor:
class CLIPFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class CLIPProcessor:
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class CLIPProcessor(metaclass=DummyObject):
_backends = ["vision"]
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["vision"])
class DeiTFeatureExtractor:
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class DetrFeatureExtractor:
class DeiTFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class ImageGPTFeatureExtractor:
class DetrFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class LayoutLMv2FeatureExtractor:
class ImageGPTFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class LayoutLMv2Processor:
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class LayoutLMv2FeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["vision"])
class LayoutXLMProcessor:
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, ["vision"])
class PerceiverFeatureExtractor:
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class SegformerFeatureExtractor:
class LayoutLMv2Processor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class ViTFeatureExtractor:
class LayoutXLMProcessor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class PerceiverFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class SegformerFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])
class ViTFeatureExtractor(metaclass=DummyObject):
_backends = ["vision"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["vision"])

77
utils/check_dummies.py
View File

@ -33,46 +33,15 @@ DUMMY_CONSTANT = """
{0} = None
"""
DUMMY_PRETRAINED_CLASS = """
class {0}:
def __init__(self, *args, **kwargs):
requires_backends(self, {1})
@classmethod
def from_pretrained(cls, *args, **kwargs):
requires_backends(cls, {1})
"""
PT_DUMMY_PRETRAINED_CLASS = (
DUMMY_PRETRAINED_CLASS
+ """
def forward(self, *args, **kwargs):
requires_backends(self, {1})
"""
)
TF_DUMMY_PRETRAINED_CLASS = (
DUMMY_PRETRAINED_CLASS
+ """
def call(self, *args, **kwargs):
requires_backends(self, {1})
"""
)
FLAX_DUMMY_PRETRAINED_CLASS = (
DUMMY_PRETRAINED_CLASS
+ """
def __call__(self, *args, **kwargs):
requires_backends(self, {1})
"""
)
DUMMY_CLASS = """
class {0}:
class {0}(metaclass=DummyObject):
_backends = {1}
def __init__(self, *args, **kwargs):
requires_backends(self, {1})
"""
DUMMY_FUNCTION = """
def {0}(*args, **kwargs):
requires_backends({0}, {1})
@ -126,45 +95,12 @@ def read_init():
def create_dummy_object(name, backend_name):
"""Create the code for the dummy object corresponding to `name`."""
_models = [
"ForCausalLM",
"ForConditionalGeneration",
"ForMaskedLM",
"ForMultipleChoice",
"ForNextSentencePrediction",
"ForObjectDetection",
"ForQuestionAnswering",
"ForSegmentation",
"ForSequenceClassification",
"ForTokenClassification",
"Model",
]
_pretrained = ["Config", "Tokenizer", "Processor"]
if name.isupper():
return DUMMY_CONSTANT.format(name)
elif name.islower():
return DUMMY_FUNCTION.format(name, backend_name)
else:
is_model = False
for part in _models:
if part in name:
is_model = True
break
if is_model:
if name.startswith("TF"):
return TF_DUMMY_PRETRAINED_CLASS.format(name, backend_name)
if name.startswith("Flax"):
return FLAX_DUMMY_PRETRAINED_CLASS.format(name, backend_name)
return PT_DUMMY_PRETRAINED_CLASS.format(name, backend_name)
is_pretrained = False
for part in _pretrained:
if part in name:
is_pretrained = True
break
if is_pretrained:
return DUMMY_PRETRAINED_CLASS.format(name, backend_name)
else:
return DUMMY_CLASS.format(name, backend_name)
return DUMMY_CLASS.format(name, backend_name)
def create_dummy_files():
@ -176,7 +112,8 @@ def create_dummy_files():
for backend, objects in backend_specific_objects.items():
backend_name = "[" + ", ".join(f'"{b}"' for b in backend.split("_and_")) + "]"
dummy_file = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
dummy_file += "from ..file_utils import requires_backends\n\n"
dummy_file += "# flake8: noqa\n"
dummy_file += "from ..file_utils import DummyObject, requires_backends\n\n"
dummy_file += "\n".join([create_dummy_object(o, backend_name) for o in objects])
dummy_files[backend] = dummy_file

1
utils/check_repo.py
View File

@ -522,6 +522,7 @@ UNDOCUMENTED_OBJECTS = [
"BasicTokenizer", # Internal, should never have been in the main init.
"CharacterTokenizer", # Internal, should never have been in the main init.
"DPRPretrainedReader", # Like an Encoder.
"DummyObject", # Just picked by mistake sometimes.
"MecabTokenizer", # Internal, should never have been in the main init.
"ModelCard", # Internal type.
"SqueezeBertModule", # Internal building block (should have been called SqueezeBertLayer)