Refactor default chat template warnings (#30551)
* Temporarily silence warnings in apply_chat_template until we can properly deprecate default chat templates * make fixup * Move the default chat template warning into apply_chat_template itself * make fixup
This commit is contained in:
parent
4bc9cb36b7
commit
4b4da18f53
|
@ -411,13 +411,6 @@ class BlenderbotTokenizer(PreTrainedTokenizer):
|
|||
"""
|
||||
A very simple chat template that just adds whitespace between messages.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}"
|
||||
|
|
|
@ -293,13 +293,6 @@ class BlenderbotTokenizerFast(PreTrainedTokenizerFast):
|
|||
"""
|
||||
A very simple chat template that just adds whitespace between messages.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}"
|
||||
|
|
|
@ -224,13 +224,6 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
|
|||
"""
|
||||
A very simple chat template that just adds whitespace between messages.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}"
|
||||
|
|
|
@ -104,13 +104,6 @@ class BlenderbotSmallTokenizerFast(PreTrainedTokenizerFast):
|
|||
"""
|
||||
A very simple chat template that just adds whitespace between messages.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}"
|
||||
|
|
|
@ -155,11 +155,4 @@ class BloomTokenizerFast(PreTrainedTokenizerFast):
|
|||
"""
|
||||
A simple chat template that ignores role information and just concatenates messages with EOS tokens.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"
|
||||
|
|
|
@ -456,13 +456,6 @@ class CodeLlamaTokenizer(PreTrainedTokenizer):
|
|||
snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
|
||||
in the original repository.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
template = (
|
||||
"{% if messages[0]['role'] == 'system' %}"
|
||||
"{% set loop_messages = messages[1:] %}" # Extract system message if it's present
|
||||
|
|
|
@ -369,13 +369,6 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):
|
|||
snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
|
||||
in the original repository.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
template = (
|
||||
"{% if messages[0]['role'] == 'system' %}"
|
||||
"{% set loop_messages = messages[1:] %}" # Extract system message if it's present
|
||||
|
|
|
@ -247,13 +247,6 @@ class CohereTokenizerFast(PreTrainedTokenizerFast):
|
|||
'<BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello, how are you?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'
|
||||
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
default_template = (
|
||||
"{{ bos_token }}"
|
||||
"{% if messages[0]['role'] == 'system' %}"
|
||||
|
|
|
@ -336,11 +336,4 @@ class GPT2Tokenizer(PreTrainedTokenizer):
|
|||
"""
|
||||
A simple chat template that ignores role information and just concatenates messages with EOS tokens.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"
|
||||
|
|
|
@ -147,11 +147,5 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
|
|||
"""
|
||||
A simple chat template that ignores role information and just concatenates messages with EOS tokens.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
|
||||
return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"
|
||||
|
|
|
@ -234,11 +234,4 @@ class GPTNeoXTokenizerFast(PreTrainedTokenizerFast):
|
|||
"""
|
||||
A simple chat template that ignores role information and just concatenates messages with EOS tokens.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"
|
||||
|
|
|
@ -165,13 +165,6 @@ class GPTNeoXJapaneseTokenizer(PreTrainedTokenizer):
|
|||
"""
|
||||
A simple chat template that just adds BOS/EOS tokens around messages while discarding role information.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{{ bos_token + eos_token + message.content + eos_token }}"
|
||||
|
|
|
@ -301,13 +301,6 @@ class GPTSw3Tokenizer(PreTrainedTokenizer):
|
|||
This chat template formats messages like an instant messenger chat log, with "User:" and "Bot:" strings
|
||||
preceding messages. BOS tokens are added between all messages.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{{ eos_token }}{{ bos_token }}"
|
||||
"{% for message in messages %}"
|
||||
|
|
|
@ -246,13 +246,6 @@ class GPTSanJapaneseTokenizer(PreTrainedTokenizer):
|
|||
A simple chat template that adds standard BOS, SEP and EOS tokens between messages while discarding role
|
||||
information.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{% if not loop.first %}{{ bos_token}}{% endif %}"
|
||||
|
|
|
@ -284,8 +284,14 @@ class Idefics2Processor(ProcessorMixin):
|
|||
if self.chat_template is not None:
|
||||
chat_template = self.chat_template
|
||||
else:
|
||||
logger.warning_once(
|
||||
"No chat template is set for this processor, falling back to a default class-level template. This is "
|
||||
"very error-prone, because models are often trained with templates different from the class default! "
|
||||
"Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
chat_template = self.default_chat_template
|
||||
|
||||
return self.tokenizer.apply_chat_template(
|
||||
conversation, chat_template=chat_template, tokenize=tokenize, **kwargs
|
||||
)
|
||||
|
|
|
@ -429,13 +429,6 @@ class LlamaTokenizer(PreTrainedTokenizer):
|
|||
snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
|
||||
in the original repository.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
template = (
|
||||
"{% if messages[0]['role'] == 'system' %}"
|
||||
"{% set loop_messages = messages[1:] %}" # Extract system message if it's present
|
||||
|
|
|
@ -261,13 +261,6 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
|
|||
snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
|
||||
in the original repository.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
template = (
|
||||
"{% if messages[0]['role'] == 'system' %}"
|
||||
"{% set loop_messages = messages[1:] %}" # Extract system message if it's present
|
||||
|
|
|
@ -815,13 +815,6 @@ class WhisperTokenizer(PreTrainedTokenizer):
|
|||
"""
|
||||
A simple chat template that ignores role information and just concatenates messages with EOS tokens.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"
|
||||
|
||||
def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
|
||||
|
|
|
@ -544,13 +544,6 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
|
|||
"""
|
||||
A simple chat template that ignores role information and just concatenates messages with EOS tokens.
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. "
|
||||
"This is very error-prone, because models are often trained with templates different from the class "
|
||||
"default! Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"
|
||||
|
||||
# Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_decoder_prompt_ids
|
||||
|
|
|
@ -1746,16 +1746,27 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
|
|||
if tokenizer_kwargs is None:
|
||||
tokenizer_kwargs = {}
|
||||
|
||||
using_default_template = False
|
||||
|
||||
# First, handle the cases when the model has a dict of multiple templates
|
||||
if isinstance(self.chat_template, dict) or (
|
||||
self.chat_template is None and isinstance(self.default_chat_template, dict)
|
||||
):
|
||||
template_dict = self.chat_template or self.default_chat_template
|
||||
if self.chat_template is not None:
|
||||
template_dict = self.chat_template
|
||||
using_default_dict = False
|
||||
else:
|
||||
template_dict = self.default_chat_template
|
||||
using_default_dict = True
|
||||
if chat_template is not None and chat_template in template_dict:
|
||||
# The user can pass the name of a template to the chat template argument instead of an entire template
|
||||
chat_template = template_dict[chat_template]
|
||||
if using_default_dict:
|
||||
using_default_template = True
|
||||
elif chat_template is None and "default" in template_dict:
|
||||
chat_template = template_dict["default"]
|
||||
if using_default_dict:
|
||||
using_default_template = True
|
||||
elif chat_template is None:
|
||||
raise ValueError(
|
||||
"This model has multiple chat templates with no default specified! Please either pass a chat "
|
||||
|
@ -1769,6 +1780,16 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
|
|||
chat_template = self.chat_template
|
||||
else:
|
||||
chat_template = self.default_chat_template
|
||||
using_default_template = True
|
||||
|
||||
if using_default_template:
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a default class-level template. This is "
|
||||
"very error-prone, because models are often trained with templates different from the class default! "
|
||||
"Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
|
||||
# Compilation function uses a cache to avoid recompiling the same template
|
||||
compiled_template = self._compile_jinja_template(chat_template)
|
||||
|
@ -1840,13 +1861,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
|
|||
This template formats inputs in the standard ChatML format. See
|
||||
https://github.com/openai/openai-python/blob/main/chatml.md
|
||||
"""
|
||||
logger.warning_once(
|
||||
"No chat template is set for this tokenizer, falling back to a ChatML template. "
|
||||
"This is very error-prone, because most models are not trained with a ChatML template!"
|
||||
"Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which "
|
||||
"point any code depending on them will stop working. We recommend setting a valid chat template before "
|
||||
"then to ensure that this model continues working without issues."
|
||||
)
|
||||
return (
|
||||
"{% for message in messages %}"
|
||||
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
|
||||
|
|
Loading…
Reference in New Issue