dnrops
/
transformers.js
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Do not add token types for `FalconTokenizer`

This commit is contained in:
Joshua Lochner 2023-10-03 11:53:22 +02:00
parent 965a7154c5
commit 8f30a28246
1 changed files with 1 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/tokenizers.js
View File

@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
export class MPNetTokenizer extends PreTrainedTokenizer { }
export class FalconTokenizer extends PreTrainedTokenizer {
/** @type {add_token_types} */
prepare_model_inputs(inputs) {
return add_token_types(inputs);
}
}
export class FalconTokenizer extends PreTrainedTokenizer { }
export class GPTNeoXTokenizer extends PreTrainedTokenizer { }