dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixing flaky test (hopefully). (#15154) 

* Fixing flaky test (hopefully).

* tf compliant.
This commit is contained in:
Nicolas Patry 2022-01-14 16:47:03 +01:00 committed by GitHub
parent 7d9a33fb5c
commit b212ff9f49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/transformers/pipelines/question_answering.py
View File

@ -434,11 +434,22 @@ class QuestionAnsweringPipeline(ChunkPipeline):
question_first = bool(self.tokenizer.padding_side == "right")
enc = output["encoding"]
# Encoding was *not* padded, input_ids *might*.
# It doesn't make a difference unless we're padding on
# the left hand side, since now we have different offsets
# everywhere.
if self.tokenizer.padding_side == "left":
offset = (output["input_ids"] == self.tokenizer.pad_token_id).numpy().sum()
else:
offset = 0
# Sometimes the max probability token is in the middle of a word so:
# - we start by finding the right word containing the token with `token_to_word`
# - then we convert this word in a character span with `word_to_chars`
sequence_index = 1 if question_first else 0
for s, e, score in zip(starts, ends, scores):
s = s - offset
e = e - offset
try:
start_word = enc.token_to_word(s)
end_word = enc.token_to_word(e)