Fix to prevent crashing on assert len(tokens_b)>=1

2019-09-09 19:58:08 -04:00 · 2019-09-09 19:58:08 -04:00 · eab980fd68
parent a95ced6260
commit eab980fd68
1 changed files with 2 additions and 1 deletions
--- a/examples/lm_finetuning/pregenerate_training_data.py
+++ b/examples/lm_finetuning/pregenerate_training_data.py
@ -329,7 +329,8 @@ def main():
                    doc = []
                else:
                    tokens = tokenizer.tokenize(line)
-                    doc.append(tokens)
+                    if tokens:
+                        doc.append(tokens)
            if doc:
                docs.add_document(doc)  # If the last doc didn't end on a newline, make sure it still gets added
        if len(docs) <= 1: