162 lines
6.5 KiB
Bash
Executable File
162 lines
6.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# this script evals the following fsmt models
|
|
# it covers:
|
|
# - facebook/wmt19-ru-en
|
|
# - facebook/wmt19-en-ru
|
|
# - facebook/wmt19-de-en
|
|
# - facebook/wmt19-en-de
|
|
|
|
|
|
# this script needs to be run from the top level of the transformers repo
|
|
if [ ! -d "src/transformers" ]; then
|
|
echo "Error: This script needs to be run from the top of the transformers repo"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
# In these scripts you may have to lower BS if you get CUDA OOM (or increase it if you have a large GPU)
|
|
|
|
### a short estimate version for quick testing ###
|
|
|
|
export PAIR=en-ru
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=8
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src | head -10 > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref | head -10 > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
|
|
|
|
### Normal eval ###
|
|
|
|
# ru-en
|
|
|
|
export PAIR=ru-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=50
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
|
|
# (target BLEU: 41.3 http://matrix.statmt.org/matrix/output/1907?run_id=6937)
|
|
|
|
|
|
# en-ru
|
|
|
|
export PAIR=en-ru
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=50
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
# (target BLEU: 36.4 http://matrix.statmt.org/matrix/output/1914?score_id=37605)
|
|
|
|
|
|
|
|
# en-de
|
|
|
|
export PAIR=en-de
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
# (target BLEU: 43.1 http://matrix.statmt.org/matrix/output/1909?run_id=6862)
|
|
|
|
|
|
# de-en
|
|
|
|
export PAIR=de-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=8
|
|
export NUM_BEAMS=50
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
echo $PAIR
|
|
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
|
|
|
|
# (target BLEU: 42.3 http://matrix.statmt.org/matrix/output/1902?run_id=6750)
|
|
|
|
|
|
### Searching hparams eval ###
|
|
|
|
# en-ru
|
|
|
|
export PAIR=ru-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=32
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="0" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1"
|
|
|
|
|
|
# en-ru
|
|
|
|
export PAIR=en-ru
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=16
|
|
mkdir -p $DATA_DIR
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="0" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false"
|
|
|
|
# en-de
|
|
|
|
export PAIR=en-de
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=16
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="1" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false"
|
|
|
|
# de-en
|
|
|
|
export PAIR=de-en
|
|
export DATA_DIR=data/$PAIR
|
|
export SAVE_DIR=data/$PAIR
|
|
export BS=16
|
|
mkdir -p $DATA_DIR
|
|
mkdir -p $DATA_DIR
|
|
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
|
|
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
|
|
CUDA_VISIBLE_DEVICES="1" PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval_search.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --search="num_beams=5:8:11:15 length_penalty=0.6:0.7:0.8:0.9:1.0:1.1 early_stopping=true:false"
|