train_phone_en.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "\n",
    "from tensorflow.contrib.learn.python.learn.datasets import base\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os,sys\n",
    "sys.path.insert(0, './scripts')\n",
    "import py_compile\n",
    "py_compile.compile('scripts/ivector_tools.py')\n",
    "py_compile.compile('scripts/siamese_model_phone_en.py')\n",
    "import ivector_dataset\n",
    "import siamese_model_phone_en as siamese_model\n",
    "import ivector_tools as it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_dataset_size(dict_file,feat_file):\n",
    "# Counting feature dimension and total number of utterances\n",
    "    f = open(dict_file)\n",
    "    dict_dim = 0\n",
    "    for line in f:\n",
    "        dict_dim+=1\n",
    "    f.close()\n",
    "    feat_len = 0\n",
    "    f = open(feat_file)\n",
    "    for line in f:\n",
    "        feat_len+=1\n",
    "    f.close()\n",
    "    return dict_dim, feat_len\n",
    "\n",
    "def get_feat_label(dict_file, feat_file):\n",
    "# Get feature vectors from files\n",
    "    dict_dim, feat_len = get_dataset_size(dict_file,feat_file)\n",
    "    features = np.zeros((feat_len,dict_dim),dtype='float32')\n",
    "    labels = np.zeros((feat_len),dtype='int8')\n",
    "    names = []\n",
    "    f = open(feat_file)\n",
    "    count = 0\n",
    "    for line in f:\n",
    "        names.append(line.split()[0])\n",
    "        labels[count] = line.split()[1]\n",
    "        line= line.split()[2:]\n",
    "        for iter in range(0,len(line)):\n",
    "            elements = line[iter].split(':')\n",
    "            features[count][ int( elements[0] ) -1 ] = elements[1]\n",
    "        count = count + 1 \n",
    "    f.close()\n",
    "    \n",
    "    return features, labels, names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13825, 33660) (1524, 33660) (1492, 33660)\n"
     ]
    }
   ],
   "source": [
    "context = 3\n",
    "dict_file = 'data/train.vardial2017/dict.phone_en.c'+str(context)\n",
    "feat_file = 'data/train.vardial2017/phone_en.c'+str(context)\n",
    "trn_features, trn_labels, trn_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/dev.vardial2017/phone_en.c'+str(context)\n",
    "dev_features, dev_labels, dev_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/test.MGB3/phone_en.c'+str(context)\n",
    "tst_features, tst_labels, tst_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "print trn_features.shape, dev_features.shape, tst_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "languages = ['EGY','GLF','LAV','MSA','NOR']\n",
    "\n",
    "\n",
    "\n",
    "# load tst.MGB3 labels\n",
    "filename = 'data/test.MGB3/reference'\n",
    "tst_ref_names = np.loadtxt(filename,usecols=[0],dtype='string')\n",
    "tst_ref_labels = np.loadtxt(filename,usecols=[1],dtype='int')\n",
    "\n",
    "tst_labels_index = []\n",
    "for i,name in enumerate(tst_names):\n",
    "    for j, name_ref in enumerate(tst_ref_names):\n",
    "        if name == name_ref:\n",
    "            tst_labels_index = np.append(tst_labels_index,j)\n",
    "\n",
    "tst_labels = np.empty((np.size(tst_labels_index)))\n",
    "for i,j in enumerate(tst_labels_index):\n",
    "    tst_labels[i]=tst_ref_labels[int(j)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# merge trn+dev\n",
    "# trndev_features = np.append(trn_features, dev_features,axis=0)\n",
    "# trndev_labels = np.append(trn_labels,dev_labels)\n",
    "# trndev_names = np.append(trn_names,dev_names)\n",
    "\n",
    "# #center and length norm.\n",
    "# m=np.mean(trn_features,axis=0)\n",
    "# A = np.cov(trn_features.transpose())\n",
    "# [a,D,V] = np.linalg.svd(A)\n",
    "# V= V.transpose()\n",
    "# W= np.dot(V, np.diag(1./( np.sqrt(D) + 0.0000000001 )))\n",
    "\n",
    "\n",
    "# trn_features = np.dot( np.subtract( trn_features, m), W)\n",
    "# trndev_features = np.dot( np.subtract( trndev_features, m), W)\n",
    "# dev_features = np.dot( np.subtract( dev_features, m), W)\n",
    "# tst_features = np.dot( np.subtract( tst_features, m), W)\n",
    "\n",
    "# trn_features = it.length_norm(trn_features.transpose()).transpose()\n",
    "# trndev_features = it.length_norm(trndev_features.transpose()).transpose()\n",
    "# dev_features = it.length_norm(dev_features.transpose()).transpose()\n",
    "# tst_features = it.length_norm(tst_features.transpose()).transpose()\n",
    "\n",
    "# trn_features = it.length_norm(trn_features)\n",
    "# trndev_features = it.length_norm(trndev_features)\n",
    "# dev_features = it.length_norm(dev_features)\n",
    "# tst_features = it.length_norm(tst_features)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((13825, 33660), (1524, 33660), (5, 33660), (1492, 33660))\n"
     ]
    }
   ],
   "source": [
    "#language modeling\n",
    "lang_mean=[]\n",
    "for i, lang in enumerate(languages):\n",
    "     lang_mean.append(np.mean(np.append(trn_features[np.nonzero(trn_labels == i+1)] ,dev_features[np.nonzero(dev_labels == i+1)],axis=0),axis=0))\n",
    "#    lang_mean.append(np.mean( trn_features[np.nonzero(trn_labels == i+1)][:],axis=0 ) )\n",
    "\n",
    "lang_mean = np.array(lang_mean)\n",
    "lang_mean = it.length_norm(lang_mean)\n",
    "\n",
    "print( np.shape(trn_features), np.shape(dev_features), np.shape(lang_mean),np.shape(tst_features) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline accurary on test dataset : 0.345\n",
      "Baseline accurary on dev dataset : 0.331\n"
     ]
    }
   ],
   "source": [
    "# Baseline performance on TST using CDS\n",
    "tst_scores = lang_mean.dot(tst_features.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Baseline accurary on test dataset : %0.3f' %(acc)\n",
    "\n",
    "# Baseline performance on DEV using CDS\n",
    "dev_scores = lang_mean.dot(dev_features.transpose() )\n",
    "hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "temp = ((dev_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(dev_labels))\n",
    "print 'Baseline accurary on dev dataset : %0.3f' %(acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((69125,), (69125,), (69125,))\n",
      "((13825, 33660), (13825, 33660), (13825,), (55300, 33660), (55300, 33660), (55300,))\n"
     ]
    }
   ],
   "source": [
    "# making pair of train i-vector with mean of each language i-vector\n",
    "#  example : for total 3 ivectors\n",
    "#  ivector   ivector_p  label\n",
    "#     1         1         1\n",
    "#     1         2         0\n",
    "#     1         3         0\n",
    "#     2         1         0\n",
    "#     2         2         1\n",
    "#     ...      ...       ...\n",
    "#     3         3         1\n",
    "\n",
    "# preparing pair labels\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(trn_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( trn_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# TRN dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "trn_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "trn_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "trn_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "trn_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((7620,), (7620,), (7620,))\n",
      "((1524, 33660), (1524, 33660), (1524,), (6096, 33660), (6096, 33660), (6096,))\n"
     ]
    }
   ],
   "source": [
    "# preparing pair labels of DEV dataset\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(dev_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( dev_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# DEV dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "dev_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "dev_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "dev_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "dev_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 61200)\n",
      "(?, 1500)\n",
      "(?, 600)\n",
      "(?, 61200)\n",
      "(?, 1500)\n",
      "(?, 600)\n"
     ]
    }
   ],
   "source": [
    "# init variables\n",
    "sess = tf.InteractiveSession()\n",
    "siamese = siamese_model.siamese(np.shape(trn_features)[1]);\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learning_rate = tf.train.exponential_decay(0.01, global_step,\n",
    "                                           5000, 0.99, staircase=True)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(siamese.loss, global_step=global_step)\n",
    "saver = tf.train.Saver()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0: loss 0.999256, Acc.: (DEV)0.264 (TST)0.273, lr : 0.01000\n",
      "Step 100: loss 0.999843, Acc.: (DEV)0.289 (TST)0.306, lr : 0.01000\n",
      "Step 200: loss 0.999942, Acc.: (DEV)0.328 (TST)0.350, lr : 0.01000\n",
      "Step 3100: loss 1.000039, Acc.: (DEV)0.363 (TST)0.355, lr : 0.01000\n",
      "Step 3200: loss 0.999995, Acc.: (DEV)0.360 (TST)0.359, lr : 0.01000\n",
      "Step 4100: loss 0.999634, Acc.: (DEV)0.356 (TST)0.360, lr : 0.01000\n",
      "Step 4300: loss 0.999966, Acc.: (DEV)0.358 (TST)0.363, lr : 0.01000\n",
      "Step 4500: loss 0.999730, Acc.: (DEV)0.365 (TST)0.365, lr : 0.01000\n",
      "Step 6400: loss 1.000143, Acc.: (DEV)0.363 (TST)0.367, lr : 0.00990\n",
      "Step 6600: loss 0.999213, Acc.: (DEV)0.358 (TST)0.369, lr : 0.00990\n",
      "Step 11200: loss 0.993310, Acc.: (DEV)0.354 (TST)0.369, lr : 0.00980\n",
      "Step 13800: loss 1.032185, Acc.: (DEV)0.369 (TST)0.374, lr : 0.00980\n",
      "Step 14300: loss 1.007270, Acc.: (DEV)0.383 (TST)0.379, lr : 0.00980\n",
      "Step 15100: loss 1.003735, Acc.: (DEV)0.387 (TST)0.385, lr : 0.00970\n",
      "Step 15600: loss 0.997513, Acc.: (DEV)0.394 (TST)0.386, lr : 0.00970\n",
      "Step 15700: loss 0.971648, Acc.: (DEV)0.388 (TST)0.396, lr : 0.00970\n",
      "Step 17900: loss 1.015710, Acc.: (DEV)0.416 (TST)0.401, lr : 0.00970\n",
      "Step 18500: loss 1.006037, Acc.: (DEV)0.433 (TST)0.406, lr : 0.00970\n",
      "Step 20100: loss 0.937317, Acc.: (DEV)0.445 (TST)0.410, lr : 0.00961\n",
      "Step 21400: loss 0.931397, Acc.: (DEV)0.488 (TST)0.412, lr : 0.00961\n",
      "Step 24000: loss 0.939118, Acc.: (DEV)0.517 (TST)0.414, lr : 0.00961\n",
      "Step 24700: loss 0.933524, Acc.: (DEV)0.557 (TST)0.416, lr : 0.00961\n",
      "Step 25700: loss 1.019283, Acc.: (DEV)0.579 (TST)0.416, lr : 0.00951\n",
      "Step 25900: loss 0.958286, Acc.: (DEV)0.568 (TST)0.420, lr : 0.00951\n",
      "Step 26700: loss 0.950988, Acc.: (DEV)0.588 (TST)0.426, lr : 0.00951\n",
      "Step 28100: loss 0.943912, Acc.: (DEV)0.639 (TST)0.436, lr : 0.00951\n",
      "Step 28200: loss 0.899750, Acc.: (DEV)0.640 (TST)0.439, lr : 0.00951\n",
      "Step 29700: loss 0.877030, Acc.: (DEV)0.667 (TST)0.440, lr : 0.00951\n",
      "Step 29800: loss 0.871835, Acc.: (DEV)0.686 (TST)0.442, lr : 0.00951\n",
      "Step 29900: loss 0.950870, Acc.: (DEV)0.670 (TST)0.450, lr : 0.00951\n",
      "Step 30300: loss 0.924499, Acc.: (DEV)0.705 (TST)0.457, lr : 0.00941\n",
      "Step 32300: loss 0.878967, Acc.: (DEV)0.750 (TST)0.460, lr : 0.00941\n",
      "Step 32700: loss 0.873276, Acc.: (DEV)0.737 (TST)0.461, lr : 0.00941\n",
      "Step 36400: loss 0.845909, Acc.: (DEV)0.843 (TST)0.462, lr : 0.00932\n",
      "Step 37400: loss 0.877740, Acc.: (DEV)0.854 (TST)0.462, lr : 0.00932\n",
      "Step 37600: loss 0.786367, Acc.: (DEV)0.857 (TST)0.471, lr : 0.00932\n",
      "Step 41600: loss 0.827975, Acc.: (DEV)0.929 (TST)0.481, lr : 0.00923\n",
      "Step 42000: loss 0.797597, Acc.: (DEV)0.935 (TST)0.483, lr : 0.00923\n",
      "Step 43700: loss 0.820040, Acc.: (DEV)0.967 (TST)0.486, lr : 0.00923\n",
      "Step 45800: loss 0.805777, Acc.: (DEV)0.998 (TST)0.489, lr : 0.00914\n"
     ]
    }
   ],
   "source": [
    "#start training\n",
    "batch_size = 40\n",
    "max_acc = 0.0\n",
    "max_step=0\n",
    "saver_folder='snnmodel_phone_en'\n",
    "if not os.path.exists(saver_folder):\n",
    "    os.mkdir(saver_folder)\n",
    "for step in range(500000):\n",
    "  \n",
    "    if step %5 ==0:\n",
    "        batch_x1_a, batch_y1_a = trn_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = trn_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = trn_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = trn_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "    else:\n",
    "        batch_x1_a, batch_y1_a = dev_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = dev_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = dev_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = dev_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "        \n",
    "\n",
    "#     batch_x1,batch_y1 = mgb3_siam1.train.next_batch(120,shuffle=False)\n",
    "#     batch_x2,batch_y2 = mgb3_siam2.train.next_batch(120,shuffle=False)    \n",
    "#     batch_y = (batch_y1==batch_y2).astype('float')\n",
    "    batch_y = batch_y1*2-1\n",
    "#     batch_y = 1-batch_y1\n",
    "    \n",
    "    _, loss_v = sess.run([train_step, siamese.loss], feed_dict={\n",
    "        siamese.x1: batch_x1,\n",
    "        siamese.x2: batch_x2,\n",
    "        siamese.y_: batch_y\n",
    "    })\n",
    "    \n",
    "    if np.isnan(loss_v):\n",
    "        print ('Model diverged with loss = NAN')\n",
    "        quit()\n",
    "        \n",
    "    if step % 100 ==0:\n",
    "        dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "        lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "        tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "        \n",
    "        dev_scores = lang_mean_siam.dot(dev_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "        temp = ((dev_labels-1) - hypo_lang)\n",
    "        acc =1- np.size(np.nonzero(temp)) / float( np.size(dev_labels) )\n",
    "        \n",
    "        tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "        temp = ((tst_labels-1) - hypo_lang)\n",
    "        acc_tst =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "\n",
    "        if max_acc < acc_tst:\n",
    "            max_acc = acc_tst\n",
    "            max_step=step\n",
    "            print ('Step %d: loss %f, Acc.: (DEV)%.3f (TST)%.3f, lr : %.5f' % (step,loss_v,acc,acc_tst,sess.run(learning_rate)))\n",
    "            saver.save(sess, saver_folder+'/model'+str(step)+'.ckpt')\n",
    "        if loss_v<0.6:\n",
    "            break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45800\n",
      "INFO:tensorflow:Restoring parameters from snnmodel_phone_en/model45800.ckpt\n",
      "Final accurary on test dataset : 0.489\n"
     ]
    }
   ],
   "source": [
    "print max_step\n",
    "RESTORE_STEP=max_step\n",
    "saver.restore(sess, saver_folder+'/model'+str(RESTORE_STEP)+'.ckpt')\n",
    "\n",
    "\n",
    "# trn_features_siam = siamese.o1.eval({siamese.x1:trn_features})\n",
    "dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "\n",
    "tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Final accurary on test dataset : %0.3f' %(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix\n",
      "[[ 151.   48.   77.   33.   66.]\n",
      " [  28.  112.   58.   28.   25.]\n",
      " [  57.   44.  111.   16.   33.]\n",
      " [  13.    9.   11.  151.   16.]\n",
      " [  53.   37.   77.   34.  204.]]\n",
      "Precision\n",
      "[ 0.40266667  0.44621514  0.42528736  0.755       0.5037037 ]\n",
      "Recall\n",
      "[ 0.5         0.448       0.33233533  0.57633588  0.59302326]\n",
      "\n",
      "\n",
      "<Performance evaluation on Test dataset>\n",
      "Accurary  : 0.489\n",
      "Precision : 0.507\n",
      "Recall    : 0.490\n"
     ]
    }
   ],
   "source": [
    "confusionmat = np.zeros((5,5))\n",
    "for i,lang in enumerate(languages):\n",
    "    hypo_bylang = hypo_lang[ tst_labels == i+1]\n",
    "    hist_bylang = np.histogram(hypo_bylang,5)\n",
    "    confusionmat[:,i] = hist_bylang[0]\n",
    "\n",
    "precision = np.diag(confusionmat) / np.sum(confusionmat,axis=1) #precision\n",
    "recall = np.diag(confusionmat) / np.sum(confusionmat,axis=0) # recall\n",
    "    \n",
    "print 'Confusion matrix'\n",
    "print confusionmat\n",
    "print 'Precision'\n",
    "print precision\n",
    "print 'Recall'\n",
    "print recall\n",
    "\n",
    "print '\\n\\n<Performance evaluation on Test dataset>'\n",
    "print 'Accurary  : %0.3f' %(acc)\n",
    "print 'Precision : %0.3f' %(np.mean(precision))\n",
    "print 'Recall    : %0.3f' %(np.mean(recall))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}