2020-04-24 04:27:43 +08:00
import os
import sys
SRC_DIR = os . path . join ( os . path . dirname ( __file__ ) , " src " )
sys . path . append ( SRC_DIR )
from transformers import (
2019-12-21 22:46:46 +08:00
AutoConfig ,
AutoModel ,
AutoModelForQuestionAnswering ,
2019-12-21 22:57:32 +08:00
AutoModelForSequenceClassification ,
AutoModelWithLMHead ,
AutoTokenizer ,
2020-04-24 04:27:43 +08:00
add_start_docstrings ,
2019-08-31 04:34:23 +08:00
)
2019-12-21 22:57:32 +08:00
2020-04-25 22:43:44 +08:00
dependencies = [ " torch " , " numpy " , " tokenizers " , " filelock " , " requests " , " tqdm " , " regex " , " sentencepiece " , " sacremoses " ]
2019-12-21 22:46:46 +08:00
2019-05-31 12:36:58 +08:00
2019-08-31 04:34:23 +08:00
@add_start_docstrings ( AutoConfig . __doc__ )
def config ( * args , * * kwargs ) :
2019-12-22 00:02:36 +08:00
r """
2019-08-31 04:34:23 +08:00
# Using torch.hub !
import torch
2020-11-17 21:58:45 +08:00
config = torch . hub . load ( ' huggingface/transformers ' , ' config ' , ' bert-base-uncased ' ) # Download configuration from huggingface.co and cache.
2019-09-26 16:15:53 +08:00
config = torch . hub . load ( ' huggingface/transformers ' , ' config ' , ' ./test/bert_saved_model/ ' ) # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')`
config = torch . hub . load ( ' huggingface/transformers ' , ' config ' , ' ./test/bert_saved_model/my_configuration.json ' )
2020-09-02 20:11:45 +08:00
config = torch . hub . load ( ' huggingface/transformers ' , ' config ' , ' bert-base-uncased ' , output_attentions = True , foo = False )
assert config . output_attentions == True
config , unused_kwargs = torch . hub . load ( ' huggingface/transformers ' , ' config ' , ' bert-base-uncased ' , output_attentions = True , foo = False , return_unused_kwargs = True )
assert config . output_attentions == True
2019-08-31 04:34:23 +08:00
assert unused_kwargs == { ' foo ' : False }
"""
return AutoConfig . from_pretrained ( * args , * * kwargs )
@add_start_docstrings ( AutoTokenizer . __doc__ )
def tokenizer ( * args , * * kwargs ) :
2019-12-22 00:02:36 +08:00
r """
2019-08-31 04:34:23 +08:00
# Using torch.hub !
import torch
2020-11-17 21:58:45 +08:00
tokenizer = torch . hub . load ( ' huggingface/transformers ' , ' tokenizer ' , ' bert-base-uncased ' ) # Download vocabulary from huggingface.co and cache.
2019-09-26 16:15:53 +08:00
tokenizer = torch . hub . load ( ' huggingface/transformers ' , ' tokenizer ' , ' ./test/bert_saved_model/ ' ) # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`
2019-08-31 04:34:23 +08:00
"""
return AutoTokenizer . from_pretrained ( * args , * * kwargs )
@add_start_docstrings ( AutoModel . __doc__ )
def model ( * args , * * kwargs ) :
r """
# Using torch.hub !
import torch
2020-11-17 21:58:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' model ' , ' bert-base-uncased ' ) # Download model and configuration from huggingface.co and cache.
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' model ' , ' ./test/bert_model/ ' ) # E.g. model was saved using `save_pretrained('./test/saved_model/')`
2020-09-02 20:11:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' model ' , ' bert-base-uncased ' , output_attentions = True ) # Update configuration during loading
assert model . config . output_attentions == True
2019-08-31 04:34:23 +08:00
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
config = AutoConfig . from_json_file ( ' ./tf_model/bert_tf_model_config.json ' )
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' model ' , ' ./tf_model/bert_tf_checkpoint.ckpt.index ' , from_tf = True , config = config )
2019-08-31 04:34:23 +08:00
"""
return AutoModel . from_pretrained ( * args , * * kwargs )
2019-12-21 22:46:46 +08:00
2019-08-31 04:34:23 +08:00
@add_start_docstrings ( AutoModelWithLMHead . __doc__ )
def modelWithLMHead ( * args , * * kwargs ) :
r """
# Using torch.hub !
import torch
2020-11-17 21:58:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelWithLMHead ' , ' bert-base-uncased ' ) # Download model and configuration from huggingface.co and cache.
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelWithLMHead ' , ' ./test/bert_model/ ' ) # E.g. model was saved using `save_pretrained('./test/saved_model/')`
2020-09-02 20:11:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelWithLMHead ' , ' bert-base-uncased ' , output_attentions = True ) # Update configuration during loading
assert model . config . output_attentions == True
2019-08-31 04:34:23 +08:00
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
config = AutoConfig . from_json_file ( ' ./tf_model/bert_tf_model_config.json ' )
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelWithLMHead ' , ' ./tf_model/bert_tf_checkpoint.ckpt.index ' , from_tf = True , config = config )
2019-08-31 04:34:23 +08:00
"""
return AutoModelWithLMHead . from_pretrained ( * args , * * kwargs )
@add_start_docstrings ( AutoModelForSequenceClassification . __doc__ )
def modelForSequenceClassification ( * args , * * kwargs ) :
r """
# Using torch.hub !
import torch
2020-11-17 21:58:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForSequenceClassification ' , ' bert-base-uncased ' ) # Download model and configuration from huggingface.co and cache.
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForSequenceClassification ' , ' ./test/bert_model/ ' ) # E.g. model was saved using `save_pretrained('./test/saved_model/')`
2020-09-02 20:11:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForSequenceClassification ' , ' bert-base-uncased ' , output_attentions = True ) # Update configuration during loading
assert model . config . output_attentions == True
2019-08-31 04:34:23 +08:00
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
config = AutoConfig . from_json_file ( ' ./tf_model/bert_tf_model_config.json ' )
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForSequenceClassification ' , ' ./tf_model/bert_tf_checkpoint.ckpt.index ' , from_tf = True , config = config )
2019-08-31 04:34:23 +08:00
"""
return AutoModelForSequenceClassification . from_pretrained ( * args , * * kwargs )
@add_start_docstrings ( AutoModelForQuestionAnswering . __doc__ )
def modelForQuestionAnswering ( * args , * * kwargs ) :
r """
# Using torch.hub !
import torch
2020-11-17 21:58:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForQuestionAnswering ' , ' bert-base-uncased ' ) # Download model and configuration from huggingface.co and cache.
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForQuestionAnswering ' , ' ./test/bert_model/ ' ) # E.g. model was saved using `save_pretrained('./test/saved_model/')`
2020-09-02 20:11:45 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForQuestionAnswering ' , ' bert-base-uncased ' , output_attentions = True ) # Update configuration during loading
assert model . config . output_attentions == True
2019-08-31 04:34:23 +08:00
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
config = AutoConfig . from_json_file ( ' ./tf_model/bert_tf_model_config.json ' )
2019-09-26 16:15:53 +08:00
model = torch . hub . load ( ' huggingface/transformers ' , ' modelForQuestionAnswering ' , ' ./tf_model/bert_tf_checkpoint.ckpt.index ' , from_tf = True , config = config )
2019-08-31 04:34:23 +08:00
"""
return AutoModelForQuestionAnswering . from_pretrained ( * args , * * kwargs )