Correctly resolve trust_remote_code=None for AutoTokenizer (#28419)
* Correctly resolve trust_remote_code=None for AutoTokenizer * Second attempt at a proper resolution
This commit is contained in:
parent
5509058561
commit
5d4d62d0a2
|
@ -770,7 +770,13 @@ class AutoTokenizer:
|
|||
tokenizer_auto_map = config.auto_map["AutoTokenizer"]
|
||||
|
||||
has_remote_code = tokenizer_auto_map is not None
|
||||
has_local_code = config_tokenizer_class is not None or type(config) in TOKENIZER_MAPPING
|
||||
has_local_code = type(config) in TOKENIZER_MAPPING or (
|
||||
config_tokenizer_class is not None
|
||||
and (
|
||||
tokenizer_class_from_name(config_tokenizer_class) is not None
|
||||
or tokenizer_class_from_name(config_tokenizer_class + "Fast") is not None
|
||||
)
|
||||
)
|
||||
trust_remote_code = resolve_trust_remote_code(
|
||||
trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code
|
||||
)
|
||||
|
|
Loading…
Reference in New Issue