diff --git a/src/tokenizers.js b/src/tokenizers.js index 49c3a71..74d179a 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { } export class XLMRobertaTokenizer extends PreTrainedTokenizer { } export class MPNetTokenizer extends PreTrainedTokenizer { } -export class FalconTokenizer extends PreTrainedTokenizer { - /** @type {add_token_types} */ - prepare_model_inputs(inputs) { - return add_token_types(inputs); - } -} +export class FalconTokenizer extends PreTrainedTokenizer { } export class GPTNeoXTokenizer extends PreTrainedTokenizer { } diff --git a/tests/generate_tests.py b/tests/generate_tests.py index 2e81645..adf35b3 100644 --- a/tests/generate_tests.py +++ b/tests/generate_tests.py @@ -10,7 +10,7 @@ from scripts.supported_models import SUPPORTED_MODELS # List of tokenizers where the model isn't yet supported, but the tokenizer is ADDITIONAL_TOKENIZERS_TO_TEST = { - 'RefinedWebModel': [ + 'falcon': [ 'tiiuae/falcon-7b', ], "llama": [