dnrops
/
transformers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix computation of attention_probs when head_mask is provided. (#9853) 

* Fix computation of attention_probs when head_mask is provided.

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Apply changes to the template

Co-authored-by: Lysandre <lysandre.debut@reseau.eseo.fr>
This commit is contained in:
Funtowicz Morgan 2021-01-28 12:11:52 +01:00 committed by GitHub
parent b936582f71
commit 2ee9f9b69e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/bert/modeling_tf_bert.py
View File

@ -370,7 +370,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer):
# Mask heads if we want to
if head_mask is not None:
attention_scores = tf.multiply(attention_scores, head_mask)
attention_probs = tf.multiply(attention_probs, head_mask)
attention_output = tf.einsum("acbe,aecd->abcd", attention_probs, value_layer)
outputs = (attention_output, attention_probs) if output_attentions else (attention_output,)

2
src/transformers/models/electra/modeling_tf_electra.py
View File

@ -253,7 +253,7 @@ class TFElectraSelfAttention(tf.keras.layers.Layer):
# Mask heads if we want to
if head_mask is not None:
attention_scores = tf.multiply(attention_scores, head_mask)
attention_probs = tf.multiply(attention_probs, head_mask)
attention_output = tf.einsum("acbe,aecd->abcd", attention_probs, value_layer)
outputs = (attention_output, attention_probs) if output_attentions else (attention_output,)

2
src/transformers/models/roberta/modeling_tf_roberta.py
View File

@ -377,7 +377,7 @@ class TFRobertaSelfAttention(tf.keras.layers.Layer):
# Mask heads if we want to
if head_mask is not None:
attention_scores = tf.multiply(attention_scores, head_mask)
attention_probs = tf.multiply(attention_probs, head_mask)
attention_output = tf.einsum("acbe,aecd->abcd", attention_probs, value_layer)
outputs = (attention_output, attention_probs) if output_attentions else (attention_output,)

2
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py
View File

@ -317,7 +317,7 @@ class TF{{cookiecutter.camelcase_modelname}}SelfAttention(tf.keras.layers.Layer)
# Mask heads if we want to
if head_mask is not None:
attention_scores = tf.multiply(attention_scores, head_mask)
attention_probs = tf.multiply(attention_probs, head_mask)
attention_output = tf.einsum("acbe,aecd->abcd", attention_probs, value_layer)
outputs = (attention_output, attention_probs) if output_attentions else (attention_output,)