Babel s5d recipe

.gitignore

@@ -44,6 +44,9 @@ GRTAGS
 GPATH
 GSYMS
 
+# python compiled sources
+*.pyc
+
 # Make dependencies
 .depend.mk
 
@@ -111,5 +114,13 @@ GSYMS
 /tools/pthreads*.zip
 /tools/sequitur
 /tools/srilm.tgz
+/tools/liblbfgs-1.10.tar.gz
+/tools/liblbfgs-1.10/
+/tools/openfst-1.5.0.tar.gz
+/tools/openfst-1.5.0/
+/tools/srilm-1.7.2-beta.tar.gz
+/tools/liblbfgs/
+/tools/sequitur-g2p/
 
 /kaldiwin_vs*
+

egs/babel/s5/local/make_pitch.sh

@@ -1,4 +1,4 @@
-#!/bin/bash 
+#!/bin/bash
 
 # Copyright 2012-2013  Johns Hopkins University (Author: Daniel Povey)
 #                      Bagher BabaAli
@@ -50,7 +50,7 @@ mkdir -p $expdir/log || exit 1;
 
 scp=$data/wav.scp
 
-[ ! -s $KALDI_ROOT ] && KALDI_ROOT=../../.. 
+[ ! -s $KALDI_ROOT ] && KALDI_ROOT=../../..
 
 ( # this is for back compatiblity:
  cd $KALDI_ROOT/tools
@@ -92,7 +92,7 @@ done
 basename=`basename $data`
 wavdir=$pitchdir/temp_wav_$basename
 mkdir -p $wavdir
-  
+
 if [ -f $data/segments ] || grep '|' $data/wav.scp >/dev/null; then
   wav_scp=$expdir/wav.scp
   cat $data/segments | awk -v dir=$wavdir '{key=$1; printf("%s %s/%s.wav\n", key, dir, key);}' \
@@ -104,7 +104,7 @@ if [ -f $data/segments ] || grep '|' $data/wav.scp >/dev/null; then
   else
     # create a fake segments file that takes the whole file; this is an easy way
     # to copy to static wav files.  Note: probably this has not been tested.
-    cat $data/wav.scp | awk '{print $1, $1, 0.0, -1.0}' > $expdir/fake_segments 
+    cat $data/wav.scp | awk '{print $1, $1, 0.0, -1.0}' > $expdir/fake_segments
     segments=$expdir/fake_segments
   fi
   if [ $stage -le 0 ]; then
@@ -155,11 +155,11 @@ if [ $stage -le 1 ]; then
 fi
 
 # I don't want to put a separate script in svn just for this, so creating a temporary
-# script file in the experimental directory.  Quotes around 'EOF' disable any 
+# script file in the experimental directory.  Quotes around 'EOF' disable any
 # interpretation in the here-doc.
 cat <<'EOF' > $expdir/convert.sh
 #!/bin/bash
-sacc_flist=$1 
+sacc_flist=$1
 scpfile=$2
 [ $# -ne 2 ] && echo "Usage: convert.sh <sacc-flist-in> <scpfile-out>" && exit 1;
 
@@ -247,7 +247,7 @@ exit 0;
 #   rm $expdir/.error 2>/dev/null
 
 # #  for ((n=1; n<=nj; n++)); do
-# #     mkdir -p "$expdir/$n" 
+# #     mkdir -p "$expdir/$n"
 # #  done
 
 # #  $cmd JOB=1:$nj $expdir/make_pitch.JOB.log \
@@ -297,8 +297,8 @@ exit 0;
 
 # rm $expdir/wav.*.scp  $expdir/segments.* 2>/dev/null
 
-# nf=`cat $data/pitchs.scp | wc -l` 
-# nu=`cat $data/utt2spk | wc -l` 
+# nf=`cat $data/pitchs.scp | wc -l`
+# nu=`cat $data/utt2spk | wc -l`
 # if [ $nf -ne $nu ]; then
 #   echo "It seems not all of the feature files were successfully ($nf != $nu);"
 #   echo "consider using utils/fix_data_dir.sh $data"

egs/babel/s5c/local/CHECKPOINT.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
 function GETAPPROVAL {
-  until false ; do 
+  until false ; do
     echo "Do you want to run the command (y/n)?"
     read -n 1 WISH
-    
-    if [ "$WISH" == "y" ]; then 
+
+    if [ "$WISH" == "y" ]; then
       return true;
     elif [ "$WISH" == "n" ]; then
       return false;
@@ -21,11 +21,11 @@ function ESCAPE_PARAMS {
 
       if [[ "$v" == *"<"* ]]; then
          out="$out \"$v\""
-      elif [[ "$v" == *">"* ]] ; then 
+      elif [[ "$v" == *">"* ]] ; then
          out="$out \"$v\""
-      elif [[ "$v" == *"|"* ]] ; then 
+      elif [[ "$v" == *"|"* ]] ; then
          out="$out \'$v\'"
-      elif [[ "$v" == *" "* ]] ; then 
+      elif [[ "$v" == *" "* ]] ; then
          out="$out \"$v\""
       else
          out="$out $v"
@@ -76,7 +76,7 @@ function CHECKPOINT {
 
     if [ !$INTERACTIVE_CHECKPOINT ] ; then
       eval `ESCAPE_PARAMS "$@"`
-    else 
+    else
       APPROVAL=GETAPPROVAL
       if $APPROVAL ; then
         eval `ESCAPE_PARAMS $@`
@@ -87,7 +87,7 @@ function CHECKPOINT {
       echo -e ${COLOR_RED}"CHECKPOINT FAILURE: The command returned non-zero status" >&2
       echo -e "                    rerun the script with the parameter -c $LAST_GOOD_NAME=$COUNTER" >&2
       echo -e "COMMAND">&2
-      echo -e "  " "$@" ${COLOR_RED} >&2 
+      echo -e "  " "$@" ${COLOR_RED} >&2
 
       exit 1
     fi
@@ -97,7 +97,7 @@ function CHECKPOINT {
     echo -e "$@"${COLOR_DEFAULT} >&2
   fi
 
-  COUNTER=$(( $COUNTER + 1 )) 
+  COUNTER=$(( $COUNTER + 1 ))
   eval export $COUNTER_NAME=$COUNTER
 }
 

egs/babel/s5c/local/ali_to_rttm.sh

@@ -42,7 +42,7 @@ if [ $# != 3 ]; then
   exit 1;
 fi
 
-set -e 
+set -e
 set -o pipefail
 set -u
 
@@ -65,7 +65,7 @@ fi
 $cmd $dir/log/align_to_words.log \
   ali-to-phones $dir/final.mdl "ark:gunzip -c $dir/ali.*.gz|" ark,t:- \| \
   phones-to-prons $lang/L_align.fst $wbegin $wend ark:- "ark,s:utils/sym2int.pl -f 2- --map-oov '$oov' $lang/words.txt <$data/text|" ark,t:- \| \
-  prons-to-wordali ark:- "ark:ali-to-phones --write-lengths=true $dir/final.mdl 'ark:gunzip -c $dir/ali.*.gz|' ark,t:- |" ark,t:$dir/align.txt 
+  prons-to-wordali ark:- "ark:ali-to-phones --write-lengths=true $dir/final.mdl 'ark:gunzip -c $dir/ali.*.gz|' ark,t:- |" ark,t:$dir/align.txt
 
 echo "$0: done writing alignments."
 

egs/babel/s5c/local/annotated_kwlist_to_KWs.pl

@@ -26,7 +26,7 @@
 Allowed options:
 EOU
 
-GetOptions(); 
+GetOptions();
 
 @ARGV >= 2 || die $Usage;
 
@@ -77,7 +77,7 @@
       if ($count == 0) {
         $output .= "$value";
         $count ++; next;
-      } 
+      }
       if ($count == 6) {
         $output .= ", ...";
         last;

egs/babel/s5c/local/apply_g2p.sh

@@ -2,7 +2,7 @@
 # Copyright 2014  Johns Hopkins University (Author: Yenda Trmal)
 # Apache 2.0
 
-# Begin configuration section.  
+# Begin configuration section.
 iters=5
 stage=0
 encoding='utf-8'
@@ -82,15 +82,15 @@ cat $output/output.* > $output/output
 #Remap the words from output file back to the original casing
 #Conversion of some of thems might have failed, so we have to be careful
 #and use the transform_map file we generated beforehand
-#Also, because the sequitur output is not readily usable as lexicon (it adds 
+#Also, because the sequitur output is not readily usable as lexicon (it adds
 #one more column with ordering of the pron. variants) convert it into the proper lexicon form
 output_lex=$output/lexicon.lex
 if [ ! -z $icu_transform ] ; then
   #also, the transform is generally N -> 1, i.e. we have to take
   #extra care of words that might have been mapped into the same one
-  perl -e 'open(WORDS, $ARGV[0]) or die "Could not open file $ARGV[0]"; 
-           while(<WORDS>) { chomp; @F=split; 
-             if ($MAP{$F[0]} ) { push @{$MAP{$F[0]}}, $F[1]; } 
+  perl -e 'open(WORDS, $ARGV[0]) or die "Could not open file $ARGV[0]";
+           while(<WORDS>) { chomp; @F=split;
+             if ($MAP{$F[0]} ) { push @{$MAP{$F[0]}}, $F[1]; }
              else { $MAP{$F[0]} = [$F[1]]; }
            }
            close(WORDS);
@@ -101,7 +101,7 @@ if [ ! -z $icu_transform ] ; then
                next;
              }
              foreach $word (@{$MAP{$F[0]}} ) {
-               print "$word\t$F[2]\t$F[3]\n"; 
+               print "$word\t$F[2]\t$F[3]\n";
              }
            }
            close(LEX);

egs/babel/s5c/local/apply_map_tab_preserving.pl

@@ -12,8 +12,8 @@
 # this version preserves tabs.
 
 if (@ARGV > 0 && $ARGV[0] eq "-f") {
-  shift @ARGV; 
-  $field_spec = shift @ARGV; 
+  shift @ARGV;
+  $field_spec = shift @ARGV;
   if ($field_spec =~ m/^\d+$/) {
     $field_begin = $field_spec - 1; $field_end = $field_spec - 1;
   }
@@ -26,7 +26,7 @@
     }
   }
   if (!defined $field_begin && !defined $field_end) {
-    die "Bad argument to -f option: $field_spec"; 
+    die "Bad argument to -f option: $field_spec";
   }
 }
 
@@ -70,20 +70,20 @@
   $field_offset = 0;
   for ($n = 0; $n < @A; $n++) {
     @B = split(" ", $A[$n]);
-    
+
     for ($x = 0; $x < @B; $x++) {
       $y = $x + $field_offset;
       if ( (!defined $field_begin || $y >= $field_begin)
            && (!defined $field_end || $y <= $field_end)) {
         $b = $B[$x];
         if (!defined $map{$b}) {
           if (!$permissive) {
-            die "apply_map.pl: undefined key $a\n"; 
+            die "apply_map.pl: undefined key $a\n";
           } else {
             print STDERR "apply_map.pl: warning! missing key $a\n";
           }
         } else {
-          $B[$x] = $map{$b}; 
+          $B[$x] = $map{$b};
         }
       }
     }

egs/babel/s5c/local/augment_original_stm.pl

@@ -8,7 +8,7 @@
 #As a result, the scoring will be done on per-speaker basis as well
 #As the segment from segment mapping generally do not correspond to
 #the segmentation of the original STM file, it combines the files
-#segments and utt2spk to work out the correct speaker ID for 
+#segments and utt2spk to work out the correct speaker ID for
 #the reference segment
 #In case of overlay, it will either use the previous speaker or
 #prints out an error message

egs/babel/s5c/local/best_path_weights.sh

@@ -16,19 +16,19 @@
 # limitations under the License.
 
 
-# This script combines frame-level posteriors from different decode 
-# directories. The first decode directory is assumed to be the primary 
+# This script combines frame-level posteriors from different decode
+# directories. The first decode directory is assumed to be the primary
 # and is used to get the best path. The posteriors from other decode
-# directories are interpolated with the posteriors of the best path. 
-# The output is a new directory with final.mdl, tree from the primary 
-# decode-dir and the best path alignments and weights in a decode-directory 
+# directories are interpolated with the posteriors of the best path.
+# The output is a new directory with final.mdl, tree from the primary
+# decode-dir and the best path alignments and weights in a decode-directory
 # with the same basename as the primary directory.
 # This is typically used to get better posteriors for semisupervised training
 # of DNN
-# e.g. local/combine_posteriors.sh exp/tri6_nnet/decode_train_unt.seg 
+# e.g. local/combine_posteriors.sh exp/tri6_nnet/decode_train_unt.seg
 # exp/sgmm_mmi_b0.1/decode_fmllr_train_unt.seg_it4 exp/combine_dnn_sgmm
-# Here the final.mdl and tree are copied from exp/tri6_nnet to 
-# exp/combine_dnn_sgmm. best_path_ali.*.gz obtained from the primary dir and 
+# Here the final.mdl and tree are copied from exp/tri6_nnet to
+# exp/combine_dnn_sgmm. best_path_ali.*.gz obtained from the primary dir and
 # the interpolated posteriors in weights.*.gz are placed in
 # exp/combine_dnn_sgmm/decode_train_unt.seg
 
@@ -115,7 +115,7 @@ for i in `seq 0 $[num_sys-1]`; do
       echo $nj > $out_decode/num_jobs
     else
       if [ $nj != `cat $decode_dir/num_jobs` ]; then
-        echo "$0: number of decoding jobs mismatches, $nj versus `cat $decode_dir/num_jobs`" 
+        echo "$0: number of decoding jobs mismatches, $nj versus `cat $decode_dir/num_jobs`"
         exit 1;
       fi
     fi

egs/babel/s5c/local/check_models.sh

@@ -4,7 +4,7 @@
 check_model () {
   model=$1
   if [ -s $model ]; then echo $model
-  else 
+  else
     dir=`dirname $model`
     latest_model=`ls -lt $dir/{?,??}.mdl 2>/dev/null | head -1 | awk '{print $9}'`
     echo "*$model is not there, latest is: $latest_model"

egs/babel/s5c/local/check_wers.sh

@@ -4,7 +4,7 @@
 
 check_wer () {
   dir=$1
-  if [ -d $dir ]; then 
+  if [ -d $dir ]; then
     seen_dir=false
     for ddir in $dir/decode*; do
       if [ -d $ddir ]; then
@@ -34,7 +34,7 @@ for n in `seq 10`; do
   fi
 done
 
-if [ $# != 0 ]; then 
+if [ $# != 0 ]; then
   echo "Usage: local/check_wers.sh [--final] [--char]"
   exit 1;
 fi

egs/babel/s5c/local/cmu_uem2kaldi_dir.sh

@@ -30,12 +30,12 @@ mkdir -p $datadir
 
 echo "Converting `basename $database` to kaldi directory $datadir "
 cat $database | perl -pe 's:.+(BABEL):BABEL:; s:\}\s+\{FROM\s+: :; s:\}\s+\{TO\s+: :; s:\}.+::;' | \
-  perl -ne '@K = split; 
-            $utteranceID = @K[0]; 
-            $utteranceID =~ s:[^_]+_[^_]+_[^_]+_::; 
-            $utteranceID =~ s:([^_]+)_(.+)_(inLine|scripted):${1}_A_${2}:; 
-            $utteranceID =~ s:([^_]+)_(.+)_outLine:${1}_B_${2}:; 
-            $utteranceID .= sprintf ("_%06i", (100*@K[2])); 
+  perl -ne '@K = split;
+            $utteranceID = @K[0];
+            $utteranceID =~ s:[^_]+_[^_]+_[^_]+_::;
+            $utteranceID =~ s:([^_]+)_(.+)_(inLine|scripted):${1}_A_${2}:;
+            $utteranceID =~ s:([^_]+)_(.+)_outLine:${1}_B_${2}:;
+            $utteranceID .= sprintf ("_%06i", (100*@K[2]));
             printf("%s %s %.2f %.2f\n", $utteranceID, @K[0], @K[1], @K[2]);' | sort > $datadir/segments
 
 if [ ! -z $filelist ] ; then
@@ -66,12 +66,12 @@ perl -ne '{chomp; @K=split; $utt{@K[1]}.=" @K[0]";}
 # 4. Create the wav.scp file:
 sph2pipe=`which sph2pipe || which $KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe`
 if [ $? -ne 0 ] ; then
-  echo "Could not find sph2pipe binary. Add it to PATH"  
+  echo "Could not find sph2pipe binary. Add it to PATH"
   exit 1;
 fi
 sox=`which sox`
 if [ $? -ne 0 ] ; then
-  echo "Could not find sox binary. Add it to PATH"  
+  echo "Could not find sox binary. Add it to PATH"
   exit 1;
 fi
 
@@ -84,19 +84,19 @@ echo "Creating the $datadir/wav.scp file"
     elif [ -f $audiopath/audio/$file.wav ] ; then
       echo "$file $sox $audiopath/audio/$file.wav -r 8000 -c 1 -b 16 -t wav - downsample |"
     else
-      echo "Audio file $audiopath/audio/$file.sph does not exist!" >&2 
+      echo "Audio file $audiopath/audio/$file.sph does not exist!" >&2
       exit 1
     fi
-  done | sort -u > $datadir/wav.scp 
-  if [ $? -ne 0 ] ; then 
-    echo "Error producing the wav.scp file" 
+  done | sort -u > $datadir/wav.scp
+  if [ $? -ne 0 ] ; then
+    echo "Error producing the wav.scp file"
     exit 1
   fi
-) || exit 1 
+) || exit 1
 
 l1=`wc -l $datadir/wav.scp | cut -f 1 -d ' ' `
 echo "wav.scp contains $l1 files"
-if [ ! -z $filelist ] ; then 
+if [ ! -z $filelist ] ; then
   l2=`wc -l $filelist | cut -f 1 -d ' '`
   echo "filelist `basename $filelist` contains $l2 files"