Update falcon tokenizer (#344)
* Update generate_tests.py * Do not add token types for `FalconTokenizer`
This commit is contained in:
parent
5b31129218
commit
b6bd608322
|
@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
|
|||
export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
|
||||
export class MPNetTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
export class FalconTokenizer extends PreTrainedTokenizer {
|
||||
/** @type {add_token_types} */
|
||||
prepare_model_inputs(inputs) {
|
||||
return add_token_types(inputs);
|
||||
}
|
||||
}
|
||||
export class FalconTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
export class GPTNeoXTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ from scripts.supported_models import SUPPORTED_MODELS
|
|||
|
||||
# List of tokenizers where the model isn't yet supported, but the tokenizer is
|
||||
ADDITIONAL_TOKENIZERS_TO_TEST = {
|
||||
'RefinedWebModel': [
|
||||
'falcon': [
|
||||
'tiiuae/falcon-7b',
|
||||
],
|
||||
"llama": [
|
||||
|
|
Loading…
Reference in New Issue