Downloading and training fixes

pipeline/clean/clean-corpus.sh

@@ -69,12 +69,12 @@ test -s $output.$SRC$TRG.langid.gz
 ######################################################################
 echo "Removing leading and repetitive white spaces"
 pigz -dc $output.$SRC$TRG.langid.gz | cut -f1 | sed -e 's/^[[:space:]]*//' | tr -s " " \
-    | pigz > $output.$SRC.clean.gz
+    | pigz > $output.$SRC.gz
 pigz -dc $output.$SRC$TRG.langid.gz | cut -f2 | sed -e 's/^[[:space:]]*//' | tr -s " " \
-    | pigz > $output.$TRG.clean.gz
+    | pigz > $output.$TRG.gz
 
-test -s $output.$SRC.clean.gz || exit 1
-test -s $output.$TRG.clean.gz || exit 1
+test -s $output.$SRC.gz || exit 1
+test -s $output.$TRG.gz || exit 1
 
 echo "Remove $data from intermediate steps"
 rm -f $output.*.nrm.gz $output.*.nrm.uniq.gz $output.*.langid.gz $output.*.rule-based.gz

pipeline/clean/clean-mono.sh

@@ -48,13 +48,13 @@ test -s $output.$lang.langid.gz || exit 1
 
 ######################################################################
 echo "Rule-based filtering"
-test -s $output.$lang.clean.gz || \
+test -s $output.$lang.gz || \
 pigz -dc $output.$lang.langid.gz \
     | parallel --no-notice --pipe -k -j$(nproc) --block 50M "python $CLEAN_TOOLS/clean_mono.py -l $lang --debug" \
     2> $output.$lang.clean.debug.txt \
-    | pigz > $output.$lang.clean.gz
+    | pigz > $output.$lang.gz
 
-test -s $output.$lang.clean.gz || exit 1
+test -s $output.$lang.gz || exit 1
 
 echo "Remove data from intermediate steps"
 rm -f $output.*.nrm.gz $output.*.nrm.uniq.gz $output.*.langid.gz

pipeline/data/importers/mono/news-crawl.sh

@@ -13,6 +13,7 @@ lang=$1
 dir=$2
 dataset=$3
 
+source_path=$dir/$dataset.original.$lang
 
 test -s $source_path.gz || \
 wget -O $source_path.gz http://data.statmt.org/news-crawl/${lang}/${name}.${lang}.shuffled.deduped.gz

pipeline/data/importers/mono/paracrawl.sh

@@ -15,7 +15,8 @@ dataset=$3
 
 if [[ $lang == "en" ]]
 then
-  test -s $source_path.gz || wget -nc -O $source_path.gz https://neural.mt/data/${name}-mono/en-000.gz
+  source_path=$dir/$dataset.original.$lang
+  test -s $source_path.gz || wget -nc -O $source_path.gz https://neural.mt/data/$dataset-mono/en-000.gz
 else
   echo "Only English language is supported at this time for paracrawl"
   exit 1

pipeline/setup/compile-marian.sh

@@ -32,13 +32,13 @@ if [ ! -e /usr/local/bin/cmake ]; then
 fi
 
 
-if [ ! -e ${MARIAN}/build/marian ]; then
+if [ ! -e ${MARIAN}/marian ]; then
   echo "--- Compiling marian-dev ---"
-  mkdir -p ${MARIAN}/build
-  cd ${MARIAN}/build
+  mkdir -p ${MARIAN}
+  cd ${MARIAN}
   /usr/local/bin/cmake .. -DUSE_SENTENCEPIECE=on -DUSE_FBGEMM=on -DCOMPILE_CPU=on -DCMAKE_BUILD_TYPE=Release \
             -DCUDA_TOOLKIT_ROOT_DIR="${CUDA_DIR}"
   make -j$(nproc)
   cd ${WORKDIR}
-  test -s ${MARIAN}/build/marian || exit 1
+  test -s ${MARIAN}/marian || exit 1
 fi

pipeline/train/eval.sh

@@ -3,21 +3,22 @@
 # Evaluate a model.
 #
 # Usage:
-#   bash eval.sh model_dir [datasets...]
+#   bash eval.sh model_dir [src] [trg] [datasets...]
 #
 
 set -x
 set -euo pipefail
 
-marian=${MARIAN:-"../../marian-dev/build"}
-workspace=${WORKSPACE:-4000}
-test_datasets=${TEST_DATASETS:-${@:2}}
+test -v GPUS
+test -v MARIAN
+tests -v WORKSPACE
 
 model_dir=$1
+src="${2:-$SRC}"
+trg="${3:-$TRG}"
+test_datasets=${{@:4}:-$TEST_DATASETS}
+
 
-test -v SRC
-test -v TRG
-test -v GPUS
 
 eval_dir=${model_dir}/eval
 
@@ -29,14 +30,14 @@ mkdir -p $eval_dir
 echo "Evaluating a model ${model_dir}"
 
 for prefix in ${test_datasets}; do
-    echo "### Evaluating $prefix $SRC-$TRG"
-    sacrebleu -t $prefix -l $SRC-$TRG --echo src \
-        | tee ${eval_dir}/$prefix.$SRC \
-        | $marian/marian-decoder -c ${model_dir}/model.npz.best-bleu-detok.npz.decoder.yml -w ${workspace} \
+    echo "### Evaluating $prefix $src-$trg"
+    sacrebleu -t $prefix -l $src-$trg --echo src \
+        | tee ${eval_dir}/$prefix.$src \
+        | $MARIAN/marian-decoder -c ${model_dir}/model.npz.best-bleu-detok.npz.decoder.yml -w ${WORKSPACE} \
                                  --quiet  --quiet-translation --log ${eval_dir}/$prefix.log -d $GPUS \
-        | tee ${eval_dir}/$prefix.$TRG \
-        | sacrebleu -d -t $prefix -l $SRC-$TRG \
-        | tee ${eval_dir}/$prefix.$TRG.bleu
+        | tee ${eval_dir}/$prefix.$trg \
+        | sacrebleu -d -t $prefix -l $src-$trg \
+        | tee ${eval_dir}/$prefix.$trg.bleu
 
-    test -e ${eval_dir}/$prefix.$TRG.bleu || exit 1
+    test -e ${eval_dir}/$prefix.$trg.bleu || exit 1
 done

pipeline/train/finetune-student.sh

@@ -16,7 +16,7 @@ bash ./train.sh \
   $TRG \
   ${DATA_DIR}/augmented/corpus \
   ${DATA_DIR}/original/devset \
-  ${MODELS_DIR}/student \
+  ${MODELS_DIR}/$SRC-$TRG/student \
   --guided-alignment ${DATA_DIR}/alignment/corpus.aln.gz
 
 

pipeline/train/train-s2s.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -v
+##
+# Train a shallow s2s model.
+#
+# Usage:
+#   bash train-teacher.sh [src] [trg]
+#
+
+set -x
+set -euo pipefail
+
+src=${1:-SRC}
+trg=${2:-TRG}
+
+
+bash ./train.sh \
+  ${WORKDIR}/pipeline/train/configs/model/s2s.yml \
+  ${WORKDIR}/pipeline/train/configs/training/s2s.train.yml \
+  $src \
+  $trg \
+  ${DATA_DIR}/clean/corpus \
+  ${DATA_DIR}/original/devset \
+  ${MODELS_DIR}/$src-$trg/s2s
+

pipeline/train/train-student.sh

@@ -16,7 +16,7 @@ bash ./train.sh \
   $TRG \
   ${DATA_DIR}/augmented/corpus \
   ${DATA_DIR}/original/devset \
-  ${MODELS_DIR}/student \
+  ${MODELS_DIR}/$SRC-$TRG/student \
   --guided-alignment ${DATA_DIR}/alignment/corpus.aln.gz
 
 

pipeline/train/train-teacher-ensemble.sh

@@ -22,18 +22,5 @@ do
     $TRG \
     ${DATA_DIR}/clean/corpus \
     ${DATA_DIR}/original/devset \
-    ${MODELS_DIR}/teacher-ens$i
-done
-
-for i in $(seq 1 $n)
-do
-  bash ./train.sh \
-    ${WORKDIR}/pipeline/train/configs/model/teacher.transformer.yml \
-    ${WORKDIR}/pipeline/train/configs/training/teacher.transformer-ens.train.yml \
-    $SRC \
-    $TRG \
-    ${DATA_DIR}/clean/corpus \
-    ${DATA_DIR}/original/devset \
-    ${MODELS_DIR}/teacher-ens-rtl$i \
-    --right-left
-done
+    ${MODELS_DIR}/$SRC-$TRG/teacher-ens$i
+done

pipeline/train/train-teacher.sh

@@ -16,5 +16,5 @@ bash ./train.sh \
   $TRG \
   ${DATA_DIR}/clean/corpus \
   ${DATA_DIR}/original/devset \
-  ${MODELS_DIR}/teacher
+  ${MODELS_DIR}/$SRC-$TRG/teacher
 

run.sh

@@ -15,12 +15,22 @@ set -euo pipefail
 #│   │   ├ devset.ru.gz
 #│   │   └ devset.en.gz
 #│   ├ clean
+#│   │   ├ corpus.ru.gz
+#│   │   ├ corpus.en.gz
+#│   │   ├ mono.ru.gz
+#│   │   ├ mono.en.gz
 #│   ├ augmented
+#│   │   ├ corpus.ru.gz
+#│   │   ├ corpus.en.gz
 #│   ├ alignment
+#│   │   ├ corpus.aln.gz
 #├ models
-#│   ├ teacher
-#│   ├ student
-#│   ├ reverse
+#│   ├ ru-en
+#│   │   ├ teacher
+#│   │   ├ student
+#│   ├ en-ru
+#│   │   ├ s2s
+
 
 
 set -a
@@ -46,7 +56,7 @@ if [[ -e ${DATA_DIR}/original/mono.${SRC}.gz ]]; then
   . ./pipeline/clean/clean-mono.sh ${SRC} ${original}/mono ${clean}/mono
 fi
 
-. ./pipeline/train/train-reverse.sh
+. ./pipeline/train/train-s2s.sh $TRG $SRC