Fix to prevent crashing on assert len(tokens_b)>=1
This commit is contained in:
parent
a95ced6260
commit
eab980fd68
|
@ -329,6 +329,7 @@ def main():
|
|||
doc = []
|
||||
else:
|
||||
tokens = tokenizer.tokenize(line)
|
||||
if tokens:
|
||||
doc.append(tokens)
|
||||
if doc:
|
||||
docs.add_document(doc) # If the last doc didn't end on a newline, make sure it still gets added
|
||||
|
|
Loading…
Reference in New Issue