Merge branch 'main' into example-links
This commit is contained in:
commit
d932350751
|
@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
|
|||
export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
|
||||
export class MPNetTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
export class FalconTokenizer extends PreTrainedTokenizer {
|
||||
/** @type {add_token_types} */
|
||||
prepare_model_inputs(inputs) {
|
||||
return add_token_types(inputs);
|
||||
}
|
||||
}
|
||||
export class FalconTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
export class GPTNeoXTokenizer extends PreTrainedTokenizer { }
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ from scripts.supported_models import SUPPORTED_MODELS
|
|||
|
||||
# List of tokenizers where the model isn't yet supported, but the tokenizer is
|
||||
ADDITIONAL_TOKENIZERS_TO_TEST = {
|
||||
'RefinedWebModel': [
|
||||
'falcon': [
|
||||
'tiiuae/falcon-7b',
|
||||
],
|
||||
"llama": [
|
||||
|
|
Loading…
Reference in New Issue