Run SQuAD warning when the doc stride may be too high

This commit is contained in:
Lysandre 2020-01-16 13:59:26 -05:00
parent b8f43cb273
commit 6e2c28a14a
1 changed files with 7 additions and 0 deletions

View File

@ -671,6 +671,13 @@ def main():
parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
args = parser.parse_args()
if args.doc_stride >= args.max_seq_length - args.max_query_length:
logger.warning(
"WARNING - You've set a doc stride which may be superior to the document length in some "
"examples. This could result in errors when building features from the examples. Please reduce the doc "
"stride or increase the maximum length to ensure the features are correctly built."
)
if (
os.path.exists(args.output_dir)
and os.listdir(args.output_dir)