Reduce datasets

Snakefile

@@ -106,6 +106,7 @@ best_model_metric = config['experiment']['best-model']
 best_model = f"final.model.npz.best-{best_model_metric}.npz"
 backward_dir = f'{models_dir}/backward'
 spm_sample_size=config['experiment']['spm-sample-size']
+spm_vocab_size=config['experiment'].get('spm-vocab-size',"32000")
 vocab_path=vocab_pretrained or f"{models_dir}/vocab/vocab.spm"
 
 #evaluation
@@ -418,7 +419,7 @@ if not vocab_pretrained:
         output: vocab_path
         params: prefix_train=clean_corpus_prefix,prefix_test=f"{original}/devset"
         shell: '''bash pipeline/train/spm-vocab.sh "{input.corpus_src}" "{input.corpus_trg}" "{output}" {spm_sample_size} \
-                    >> {log} 2>&1'''
+                    {threads} {spm_vocab_size} >> {log} 2>&1'''
 
 if do_train_backward:
     rule train_backward:

configs/config.test.yml

@@ -15,6 +15,7 @@ experiment:
   mono-max-sentences-trg: 200000
   split-length: 100000
   spm-sample-size: 100000
+  spm_vocab_size: 4000
 
   best-model: chrf
 
@@ -30,26 +31,31 @@ marian-args:
     save-freq: 100
     valid-freq: 100
     after: 500u
+    dim-vocabs: [4000, 4000]
   training-teacher-base:
     disp-freq: 10
     save-freq: 100
     valid-freq: 100
     after: 500u
+    dim-vocabs: [4000, 4000]
   training-teacher-finetuned:
     disp-freq: 10
     save-freq: 100
     valid-freq: 100
     after: 500u
+    dim-vocabs: [4000, 4000]
   training-student:
     disp-freq: 10
     save-freq: 100
     valid-freq: 100
     after: 500u
+    dim-vocabs: [4000, 4000]
   training-student-finetuned:
     disp-freq: 10
     save-freq: 100
     valid-freq: 100
     after: 500u
+    dim-vocabs: [4000, 4000]
   decoding-backward:
     mini-batch-words: 2000
   decoding-teacher:
@@ -59,17 +65,16 @@ marian-args:
 datasets:
   train:
     - opus_ada83/v1
-    - opus_GNOME/v1
-    - mtdata_Neulab-tedtalks_train-1-eng-rus
+    - mtdata_ELRC-wikipedia_health-1-eng-rus
   devtest:
     - flores_dev
     - sacrebleu_wmt19
   test:
     - flores_devtest
     - sacrebleu_wmt20
   mono-src:
-    - news-crawl_news.2020
+    - news-crawl_news.2007
   mono-trg:
-    - news-crawl_news.2020
+    - news-crawl_news.2008
 
 

pipeline/train/spm-vocab.sh

@@ -13,6 +13,7 @@ corpus_trg=$2
 vocab_output=$3
 sample_size=$4
 threads=$5
+vocab_size="${6:-32000}"
 
 if [ "$threads" = "auto" ]; then
   threads=$(nproc)
@@ -27,7 +28,7 @@ ${COMPRESSION_CMD} -dc "${corpus_src}" >"${vocab_dir}/data.src.txt"
 ${COMPRESSION_CMD} -dc "${corpus_trg}" >"${vocab_dir}/data.trg.txt"
 
 "${MARIAN}/spm_train" --bos_id=-1 --eos_id=0 --unk_id=1 --user_defined_symbols="" \
-  --model_prefix="${vocab_dir}/vocab" --vocab_size=32000 \
+  --model_prefix="${vocab_dir}/vocab" --vocab_size="${vocab_size}" \
   --input="${vocab_dir}/data.src.txt,${vocab_dir}/data.trg.txt" \
   --input_sentence_size="${sample_size}" --shuffle_input_sentence=true \
   --num_threads "${threads}"