add script to generate all documents for all languages

README.md

@@ -37,6 +37,11 @@ python generate_documents.py \
 --output-filename <path_to_output_file> \
 --rcv-dir <directory_of_rcv_data>
 ```
+To generate documents for all language using a single script:
+```
+bash generate_all_documents.sh <output_directory> <directory_of_rcv_data>
+```
+
 See our paper for benchmark results on this dataset.
 # Evaluate variance through bootstrapping 
 The sampling script generates samples with uniform class priors:

generate_all_documents.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+OUT_DIR=$1
+RCV_DIR=$2
+for filename in mldoc-indices/*; do
+        n="${filename##*/}"
+        lang=$(echo $n | awk -F'.' '{print $1}')
+        echo $lang
+        mkdir -p /vc_data/users/ramitra/data/MLDoc/$lang/
+        if [[ "$lang" == *"english"* ]]; then
+                python generate_documents.py --indices-file $filename --output-filename $OUT_DIR/$lang/$n --rcv-dir $RCV_DIR/rcv1/        
+        else
+                python generate_documents.py --indices-file $filename --output-filename $OUT_DIR/$lang/$n --rcv-dir $RCV_DIR/RCV2_Multilingual_Corpus/$lang/
+        fi
+
+done