From 22dab246c5605d2fca0597eed84d16a783bc3e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 28 May 2024 16:02:51 +0200 Subject: [PATCH] Render chat template tojson filter as unicode (#31041) * Render chat template tojson filter as unicode * ruff-- --- src/transformers/tokenization_utils_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index a8d3500328..116fbfdf7b 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1853,6 +1853,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): raise TemplateError(message) jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) + jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template)