Add back translations

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 .pyre/
 
 .idea
+.data
+.models
+.bin

config.sh

@@ -10,15 +10,16 @@ TRG=en
 # parallel corpus
 TRAIN_DATASETS="opus_OPUS-ParaCrawl/v7.1"
 DEVTEST_DATASETS="mtdata_newstest2019_ruen mtdata_newstest2017_ruen mtdata_newstest2015_ruen mtdata_newstest2014_ruen"
+# sacrebleu
 TEST_DATASETS="wmt20 wmt18 wmt16 wmt13"
-# mono for source language (ex. paracrawl_paracrawl8  commoncrawl_wmt16)
+# monolingual datasets (ex. paracrawl_paracrawl8, commoncrawl_wmt16,  news-crawl_news.2020)
 MONO_DATASETS_SRC="news-crawl_news.2020"
-MONO_DATASETS_TRG="paracrawl_paracrawl8"
+MONO_DATASETS_TRG="news-crawl_news.2020"
 MONO_MAX_SENTENCES_SRC=100000000
-MONO_MAX_SENTENCES_TRG=10000000
+MONO_MAX_SENTENCES_TRG=20000000
 
 
 # marian --devices parameter for GPUs to use, for example 0 1 2 3
 GPUS=$(seq -s " " 0 $(( $(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)-1 )))
 # for 12 GB GPU
-WORKSPACE=8000
+WORKSPACE=9000

pipeline/data/download-corpus.sh

@@ -26,8 +26,9 @@ if [ ! -e ${trg_corpus} ]; then
 
   for dataset in $datasets; do
     echo "Downloading dataset ${dataset}"
-    name=${dataset#_*}
-    bash ./importers/corpus/${dataset%_*}.sh $SRC $TRG $dir $name
+    name=${dataset#*_}
+    type=${dataset%_*}
+    bash ${WORKDIR}/pipeline/data/importers/corpus/${type}.sh $SRC $TRG $dir $name
   done
 
   cat ${dir}/train-parts/*."${SRC}" | pigz > "$src_corpus"

pipeline/data/download-mono.sh

@@ -24,19 +24,20 @@ if [ ! -e ${file_name} ]; then
 
   for dataset in $datasets; do
     echo "Downloading dataset ${dataset}"
-    name=${dataset#*_}
-    source_path=$dir/$dataset.original.$lang
+    source_prefix=$dir/$dataset.original.$lang
     gz_path=$dir/$dataset.$lang.gz
+    name=${dataset#*_}
+    type=${dataset%_*}
 
-    name=${dataset#_*}
-    bash ./importers/mono/${dataset%_*}.sh $lang $dir $name
+    test -s $source_prefix.gz || \
+    bash ${WORKDIR}/pipeline/data/importers/mono/${type}.sh $lang $source_prefix $name
 
     test -s $gz_path || \
-    zcat $source_path.gz | shuf -n $(bc -l <<< "${max_sent}+${max_sent}*${coef}") | \
+    zcat $source_prefix.gz | shuf -n $(bc -l <<< "${max_sent}+${max_sent}*${coef}") | \
         perl -ne 'print if(split(/\s/, $_) < 100)' | \
         head -n "$max_sent" | pigz > $gz_path
 
-    rm $source_path.*
+    rm $source_prefix*
   done
 
   zcat ${dir}/*.$lang.gz | pigz > $file_name

pipeline/data/importers/mono/commoncrawl.sh

@@ -2,18 +2,16 @@
 # Downloads monolingual data from commoncrawl
 #
 # Usage:
-#   bash commoncrawl.sh lang dir dataset
+#   bash commoncrawl.sh lang output_prefix dataset
 #
 
 set -x
 set -euo pipefail
 
 lang=$1
-dir=$2
+output_prefix=$2
 dataset=$3
 
-source_path=$dir/$dataset.original.$lang
-
-test -s $source_path.xz || \
-wget -O $source_path.xz http://web-language-models.s3-website-us-east-1.amazonaws.com/${name}/deduped/${lang}.xz
-xzcat $source_path.xz | pigz > $source_path.gz
+test -s ${output_prefix}.gz || \
+rm ${output_prefix}.gz && wget -O ${output_prefix}.xz http://web-language-models.s3-website-us-east-1.amazonaws.com/${dataset}/deduped/${lang}.xz
+xzcat $output_prefix.xz | pigz > $output_prefix.gz

pipeline/data/importers/mono/news-crawl.sh

@@ -2,18 +2,17 @@
 # Downloads monolingual data from OPUS
 #
 # Usage:
-#   bash opus.sh lang dir dataset
+#   bash opus.sh lang output_prefix dataset
 #
 
 
 set -x
 set -euo pipefail
 
 lang=$1
-dir=$2
+output_prefix=$2
 dataset=$3
 
-source_path=$dir/$dataset.original.$lang
 
-test -s $source_path.gz || \
-wget -O $source_path.gz http://data.statmt.org/news-crawl/${lang}/${name}.${lang}.shuffled.deduped.gz
+test -s $output_prefix.gz || \
+wget -O $output_prefix.gz http://data.statmt.org/news-crawl/${lang}/${dataset}.${lang}.shuffled.deduped.gz

pipeline/data/importers/mono/paracrawl.sh

@@ -9,14 +9,14 @@ set -x
 set -euo pipefail
 
 lang=$1
-dir=$2
+output_prefix=$2
 dataset=$3
 
 
 if [[ $lang == "en" ]]
 then
-  source_path=$dir/$dataset.original.$lang
-  test -s $source_path.gz || wget -nc -O $source_path.gz https://neural.mt/data/$dataset-mono/en-000.gz
+  test -s $output_prefix.gz || \
+  wget -O $output_prefix.gz https://neural.mt/data/${dataset}-mono/en-000.gz
 else
   echo "Only English language is supported at this time for paracrawl"
   exit 1

pipeline/train/configs/training/s2s.train.yml

@@ -1,4 +1,4 @@
-# https://github.com/marian-nmt/marian-examples/tree/master/wmt2017-uedin
+## https://github.com/marian-nmt/marian-examples/tree/master/wmt2017-uedin
 after-epochs: 10
 beam-size: 12
 cost-type: ce-mean-words
@@ -11,4 +11,4 @@ mini-batch-fit: True
 normalize: 1
 save-freq: 10000
 valid-freq: 10000
-valid-mini-batch: 64
+valid-mini-batch: 64

pipeline/train/configs/training/teacher.transformer-ens.train.yml

@@ -1,4 +1,5 @@
 # https://github.com/marian-nmt/marian-examples/tree/master/wmt2017-uedin
+after-epochs: 8
 beam-size: 12
 clip-norm: 5
 cost-type: ce-mean-words

pipeline/train/eval.sh

@@ -11,12 +11,15 @@ set -euo pipefail
 
 test -v GPUS
 test -v MARIAN
-tests -v WORKSPACE
+test -v WORKSPACE
 
 model_dir=$1
 src="${2:-$SRC}"
 trg="${3:-$TRG}"
-test_datasets=${{@:4}:-$TEST_DATASETS}
+datasets=${@:4}
+test_datasets=${datasets:-$TEST_DATASETS}
+
+mo
 
 
 

pipeline/train/tensorboard/tb_log_parser.py

@@ -196,9 +196,10 @@ def update_all_avg(self):
 
 
 @click.command()
+@click.option('--dir')
 @click.option('--prefix',
-              default='model')
-def run(prefix):
+              default='')
+def run(dir, prefix):
     monitors = {}
 
     while True:

pipeline/train/tensorboard/tesnsorboard.sh

@@ -1,6 +1,5 @@
 
 conda activate bergamot-training-env
 
-python ../marian-tensorboard/tb_log_parser.py --prefix=
-
-tensorboard --logdir=./ --host=0.0.0.0
+python ./tb_log_parser.py --prefix= & \
+tensorboard --logdir=./ --host=0.0.0.0 && fg

pipeline/train/train-teacher-ensemble.sh

@@ -20,7 +20,7 @@ do
     ${WORKDIR}/pipeline/train/configs/training/teacher.transformer-ens.train.yml \
     $SRC \
     $TRG \
-    ${DATA_DIR}/clean/corpus \
+    ${DATA_DIR}/augmented/corpus \
     ${DATA_DIR}/original/devset \
     ${MODELS_DIR}/$SRC-$TRG/teacher-ens$i
 done

pipeline/translate/decoder.yml

@@ -0,0 +1,11 @@
+normalize: 1.0
+word-penalty: 0
+mini-batch: 16
+mini-batch-words: 2000
+maxi-batch: 1000
+maxi-batch-sort: src
+workspace: 8000
+max-length: 200
+max-length-crop: true
+beam-size: 8
+quiet-translation: True

pipeline/translate/translate-mono.sh

@@ -1,35 +1,45 @@
 #!/bin/bash
 
-# Usage: ./translate-mono.sh -d 4 5 6 7
+# Usage: ./translate-mono.sh mono_path model_dir output_path
 
-set -e
+set -x
+set -euo pipefail
 
-# Adjust these variables if needed.
-MARIAN=../../marian-dev/build
-CORPUSSRC=mono.en.gz
-CONFIG=teacher.yml
-DIR=mono
-OUTPUT=$DIR.translated.gz
 
-mkdir -p $DIR
+test -v GPUS
+test -v MARIAN
+
+
+mono_path=$1
+model_dir=$2
+output_path=$3
+
+config=${model_dir}/model.npz.best-ce-mean-words.npz.decoder.yml
+decoder_config=${WORKDIR}/pipeline/translate/decoder.yml
+tmp_dir=$(dirname $output_path)/tmp
+mkdir -p $tmp_dir
 
 
 # Split the corpus into smaller chunks.
-test -s $DIR/file.00 || pigz -dc $CORPUSSRC | split -d -l 2000000 - $DIR/file.
+test -s $tmp_dir/file.00 || pigz -dc $mono_path | split -d -l 2000000 - $tmp_dir/file.
 
 # Translate source sentences with Marian.
 # This can be parallelized across several GPU machines.
-for prefix in `ls $DIR/file.?? | shuf`; do
+for prefix in `ls ${tmp_dir}/file.?? | shuf`; do
     echo "# $prefix"
-    test -e $prefix.out || $MARIAN/marian-decoder -c $CONFIG -i $prefix -o $prefix.out --log $prefix.log -b 4 $@
+    test -e $prefix.out || \
+    $MARIAN/marian-decoder -c $config $decoder_config -i $prefix -o $prefix.out --log $prefix.log \
+    -d $GPUS -w $WORKSPACE
 done
 
 # Collect translations.
-cat $DIR/file.??.out | pigz > $OUTPUT
+cat $tmp_dir/file.??.out | pigz > $output_path
 
 # Source and artificial target files must have the same number of sentences,
 # otherwise collect the data manually.
-echo "# sentences $CORPUSSRC vs $OUTPUT"
-pigz -dc $CORPUSSRC | wc -l
-pigz -dc $OUTPUT | wc -l
+echo "# sentences $mono_path vs $output_path"
+pigz -dc $mono_path | wc -l
+pigz -dc $output_path | wc -l
+
+rm -rf $tmp_dir
 

run.sh

@@ -19,6 +19,9 @@ set -euo pipefail
 #│   │   ├ corpus.en.gz
 #│   │   ├ mono.ru.gz
 #│   │   ├ mono.en.gz
+#│   ├ translated
+#│   │   ├ mono.ru.gz
+#│   │   ├ mono.en.gz
 #│   ├ augmented
 #│   │   ├ corpus.ru.gz
 #│   │   ├ corpus.en.gz
@@ -33,6 +36,7 @@ set -euo pipefail
 
 
 
+
 set -a
 . ./config.sh
 set +a
@@ -47,26 +51,33 @@ original=${DATA_DIR}/original
 . ./pipeline/data/download-corpus.sh ${original}/corpus $TRAIN_DATASETS
 . ./pipeline/data/download-corpus.sh ${original}/devset $DEVTEST_DATASETS
 if [[ ${MONO_DATASETS_SRC} ]]; then
-  . ./pipeline/data/download-mono.sh ${SRC} $MONO_MAX_SENTENCES_SRC ${original}/mono $MONO_DATASETS_SRC
+ . ./pipeline/data/download-mono.sh ${SRC} $MONO_MAX_SENTENCES_SRC ${original}/mono $MONO_DATASETS_SRC
 fi
 if [[ ${MONO_DATASETS_TRG} ]]; then
-  . ./pipeline/data/download-mono.sh ${TRG} $MONO_MAX_SENTENCES_TRG ${original}/mono $MONO_DATASETS_TRG
+ . ./pipeline/data/download-mono.sh ${TRG} $MONO_MAX_SENTENCES_TRG ${original}/mono $MONO_DATASETS_TRG
 fi
 
 clean=${DATA_DIR}/clean
 . ./pipeline/clean/clean-corpus.sh ${original}/corpus ${clean}/corpus
 if [[ -e ${DATA_DIR}/original/mono.${SRC}.gz ]]; then
-  . ./pipeline/clean/clean-mono.sh ${SRC} ${original}/mono ${clean}/mono
+ . ./pipeline/clean/clean-mono.sh ${SRC} ${original}/mono ${clean}/mono
 fi
-if [[ -e ${DATA_DIR}/original/mono.${TRG}.gz ]]; then
-  . ./pipeline/clean/clean-mono.sh ${TRG} ${original}/mono ${clean}/mono
+if [[ -e ${original}/mono.${TRG}.gz ]]; then
+ . ./pipeline/clean/clean-mono.sh ${TRG} ${original}/mono ${clean}/mono
 fi
 
 . ./pipeline/train/train-s2s.sh $TRG $SRC
-. ./pipeline/train/eval.sh ${MODELS_DIR}/teacher-ens $TRG $SRC
+. ./pipeline/train/eval.sh ${MODELS_DIR}/$TRG-$SRC/s2s $TRG $SRC
+
 
-# TODO: backtranslate and augment corpus
+. ./pipeline/translate/translate-mono.sh ${clean}/mono.$TRG.gz ${MODELS_DIR}/$TRG-$SRC/s2s ${DATA_DIR}/translated/mono.$SRC.gz
 
+augmented=${DATA_DIR}/augmented
+mkdir -p $augmented
+test -s $augmented/corpus.$SRC.gz || cat ${DATA_DIR}/translated/mono.$SRC.gz ${DATA_DIR}/clean/corpus.$SRC.gz > $augmented/corpus.$SRC.gz
+test -s $augmented/corpus.$TRG.gz || cat ${clean}/mono.$TRG.gz ${DATA_DIR}/clean/corpus.$TRG.gz > $augmented/corpus.$TRG.gz
+pigz -dc $augmented/corpus.$SRC.gz | wc -l
+pigz -dc $augmented/corpus.$TRG.gz | wc -l
 
-. ./pipeline/train/train-teacher-ens.sh
-. ./pipeline/train/eval.sh ${MODELS_DIR}/teacher-ens
+. ./pipeline/train/train-teacher.sh
+. ./pipeline/train/eval.sh ${MODELS_DIR}/$SRC-$TRG/teacher