Add experiments tracking and caching interface

config.sh

@@ -11,8 +11,10 @@ set -a
 
 WORKDIR=$(pwd)
 CUDA_DIR=/usr/local/cuda-11.2
-DATA_DIR=${DATA_DIR:-${WORKDIR}/data}
-MODELS_DIR=${MODELS_DIR:-${WORKDIR}/models}
+DATA_ROOT_DIR=${DATA_ROOT_DIR:-${WORKDIR}}
+DATA_DIR=${DATA_ROOT_DIR}/data
+MODELS_DIR=${DATA_ROOT_DIR}/models
+EXPERIMENTS_DIR=${DATA_ROOT_DIR}/experiments
 MARIAN=${MARIAN:-${WORKDIR}/3rd_party/marian-dev/build}
 CLEAN_TOOLS=${WORKDIR}/pipeline/clean/tools
 BIN=${WORKDIR}/bin

pipeline/data/download-corpus.sh

@@ -15,6 +15,7 @@ test -v SRC
 test -v TRG
 
 prefix=$1
+cache=$2
 
 src_corpus="${prefix}.${SRC}.gz"
 trg_corpus="${prefix}.${TRG}.gz"

pipeline/data/download-eval.sh

@@ -15,7 +15,7 @@ test -v WORKDIR
 test -v TEST_DATASETS
 
 dir=$1
-
+cache=$2
 
 for dataset in "${@:2}"; do
   name="${dataset//[^A-Za-z0-9_- ]/_}"

pipeline/data/download-mono.sh

@@ -14,6 +14,7 @@ echo "###### Downloading monolingual data"
 lang=$1
 max_sent=$2
 prefix=$3
+cache=$4
 
 file_name="${prefix}.${lang}.gz"
 dir=$(dirname "${prefix}")/mono

pipeline/data/importers/corpus/flores.sh

@@ -23,12 +23,10 @@ mkdir -p "${tmp}"
 test -s "${tmp}/flores101_dataset.tar.gz" ||
   wget -O "${tmp}/flores101_dataset.tar.gz" "https://dl.fbaipublicfiles.com/flores101/dataset/flores101_dataset.tar.gz"
 
-tar -xzf "${tmp}/flores101_dataset.tar.gz"
+tar -xzf "${tmp}/flores101_dataset.tar.gz" -C "${tmp}" --no-same-owner
 
 source "${WORKDIR}/pipeline/setup/activate-python.sh"
 
-trg_flores=$(python -c "from mtdata.iso import iso3_code; print(iso3_code('${trg}', fail_error=True))")
-
 flores_code() {
   code=$1
 
@@ -46,8 +44,8 @@ flores_code() {
 src_flores=$(flores_code "${src}")
 trg_flores=$(flores_code "${trg}")
 
-pigz -c "${tmp}/flores101_dataset/${dataset}/${src_flores}.${dataset}" >"${dir}/flores.${src}"
-pigz -c "${tmp}/flores101_dataset/${dataset}/${trg_flores}.${dataset}" >"${dir}/flores.${trg}"
+cp "${tmp}/flores101_dataset/${dataset}/${src_flores}.${dataset}" "${dir}/flores.${src}"
+cp "${tmp}/flores101_dataset/${dataset}/${trg_flores}.${dataset}" "${dir}/flores.${trg}"
 
 rm -rf "${tmp}"
 

run.sh

@@ -13,9 +13,15 @@ set -euo pipefail
 # Directories structure
 #
 #├ data
-#│   ├ cache TODO
-#│   │  └ opus_wmt20.ru.gz
-#│   │  └ sacrebleu_wmt20.en.gz
+#│   ├ cache
+#│   │  ├ corpus
+#│   │  │  └ opus
+#│   │  │    ├ ada83_v1.en.gz
+#│   │  │    └ ada83_v1.ru.gz
+#│   │  └ mono
+#│   │     └ news-crawl
+#│   │       ├ news.2019.ru.gz
+#│   │       └ news.2019.en.gz
 #│   └ ru-en
 #│      └ test
 #│        ├ original
@@ -62,18 +68,21 @@ set -euo pipefail
 #│   │      ├ speed
 #│   │      └ exported
 #│   ├ en-ru
-#│   │   └ test
-#│   │      └ s2s
+#│      └ test
+#│         └ s2s
+#│
+#├ experiments
+#│   └ ru-en
+#│      └ test
+#│         └ config.sh
 
 echo "###### read config "
 source ./config.sh
 
-echo "######  setup"
-bash ./pipeline/setup/install-all.sh
-
 echo "######  set common variables"
 # data
-data_dir="${DATA_DIR}/${SRC}-${TRG}/${EXPERIMENT}"
+data_dir="${DATA_ROOT_DIR}/data/${SRC}-${TRG}/${EXPERIMENT}"
+cache_dir="${DATA_ROOT_DIR}/cache"
 original="${data_dir}/original"
 evaluation="${data_dir}/evaluation"
 clean="${data_dir}/clean"
@@ -84,22 +93,36 @@ merged="${data_dir}/merged"
 filtered="${data_dir}/filtered"
 align_dir="${data_dir}/alignment"
 # models
-models_dir="${MODELS_DIR}/${SRC}-${TRG}/${EXPERIMENT}"
+models_dir="${DATA_ROOT_DIR}/models/${SRC}-${TRG}/${EXPERIMENT}"
 student_dir="${models_dir}/student"
 student_finetuned_dir="${models_dir}/student-finetuned"
 teacher_dir="${models_dir}/teacher"
-s2s="${MODELS_DIR}/${TRG}-${SRC}/${EXPERIMENT}/s2s"
+s2s="${DATA_ROOT_DIR}/models/${TRG}-${SRC}/${EXPERIMENT}/s2s"
 speed="${models_dir}/speed"
 exported="${models_dir}/exported"
 
+echo "###### save experiment "
+experiment_dir="${EXPERIMENTS_DIR}/${SRC}-${TRG}/${EXPERIMENT}"
+mkdir -p "${experiment_dir}"
+cp ./config.sh "${experiment_dir}/config.sh"
+cp -r ./pipeline/translate/configs "${experiment_dir}/"
+
+echo "######  setup"
+bash ./pipeline/setup/install-all.sh
+
 echo "######  download data"
-bash ./pipeline/data/download-corpus.sh "${original}/corpus" ${TRAIN_DATASETS}
-bash ./pipeline/data/download-corpus.sh "${original}/devset" ${DEVTEST_DATASETS}
-bash ./pipeline/data/download-eval.sh "${evaluation}" ${TEST_DATASETS}
+# shellcheck disable=SC2086
+bash ./pipeline/data/download-corpus.sh "${original}/corpus" "${cache_dir}" ${TRAIN_DATASETS}
+# shellcheck disable=SC2086
+bash ./pipeline/data/download-corpus.sh "${original}/devset" "${cache_dir}" ${DEVTEST_DATASETS}
+# shellcheck disable=SC2086
+bash ./pipeline/data/download-eval.sh "${evaluation}" "${cache_dir}" ${TEST_DATASETS}
+# shellcheck disable=SC2086
 test -n "${MONO_DATASETS_SRC}" &&
-  bash ./pipeline/data/download-mono.sh "${SRC}" "${MONO_MAX_SENTENCES_SRC}" "${original}/mono" ${MONO_DATASETS_SRC}
+  bash ./pipeline/data/download-mono.sh "${SRC}" "${MONO_MAX_SENTENCES_SRC}" "${original}/mono" "${cache_dir}" ${MONO_DATASETS_SRC}
+# shellcheck disable=SC2086
 test -n "${MONO_DATASETS_TRG}" &&
-  bash ./pipeline/data/download-mono.sh "${TRG}" "${MONO_MAX_SENTENCES_TRG}" "${original}/mono" ${MONO_DATASETS_TRG}
+  bash ./pipeline/data/download-mono.sh "${TRG}" "${MONO_MAX_SENTENCES_TRG}" "${original}/mono" "${cache_dir}" ${MONO_DATASETS_TRG}
 
 echo "######  clean data"
 bash ./pipeline/clean/clean-corpus.sh "${original}/corpus" "${clean}/corpus"