train_phone_hu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "\n",
    "from tensorflow.contrib.learn.python.learn.datasets import base\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os,sys\n",
    "sys.path.insert(0, './scripts')\n",
    "import py_compile\n",
    "py_compile.compile('scripts/ivector_tools.py')\n",
    "py_compile.compile('scripts/siamese_model_phone_hu.py')\n",
    "import ivector_dataset\n",
    "import siamese_model_phone_hu as siamese_model\n",
    "import ivector_tools as it\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_dataset_size(dict_file,feat_file):\n",
    "# Counting feature dimension and total number of utterances\n",
    "    f = open(dict_file)\n",
    "    dict_dim = 0\n",
    "    for line in f:\n",
    "        dict_dim+=1\n",
    "    f.close()\n",
    "    feat_len = 0\n",
    "    f = open(feat_file)\n",
    "    for line in f:\n",
    "        feat_len+=1\n",
    "    f.close()\n",
    "    return dict_dim, feat_len\n",
    "\n",
    "def get_feat_label(dict_file, feat_file):\n",
    "# Get feature vectors from files\n",
    "    dict_dim, feat_len = get_dataset_size(dict_file,feat_file)\n",
    "    features = np.zeros((feat_len,dict_dim),dtype='float32')\n",
    "    labels = np.zeros((feat_len),dtype='int8')\n",
    "    names = []\n",
    "    f = open(feat_file)\n",
    "    count = 0\n",
    "    for line in f:\n",
    "        names.append(line.split()[0])\n",
    "        labels[count] = line.split()[1]\n",
    "        line= line.split()[2:]\n",
    "        for iter in range(0,len(line)):\n",
    "            elements = line[iter].split(':')\n",
    "            features[count][ int( elements[0] ) -1 ] = elements[1]\n",
    "        count = count + 1 \n",
    "    f.close()\n",
    "    \n",
    "    return features, labels, names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13825, 50320) (1524, 50320) (1492, 50320)\n"
     ]
    }
   ],
   "source": [
    "context = 3\n",
    "dict_file = 'data/train.vardial2017/dict.phone_hu.c'+str(context)\n",
    "feat_file = 'data/train.vardial2017/phone_hu.c'+str(context)\n",
    "trn_features, trn_labels, trn_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/dev.vardial2017/phone_hu.c'+str(context)\n",
    "dev_features, dev_labels, dev_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "feat_file = 'data/test.MGB3/phone_hu.c'+str(context)\n",
    "tst_features, tst_labels, tst_names = get_feat_label(dict_file,feat_file)\n",
    "\n",
    "print trn_features.shape, dev_features.shape, tst_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "languages = ['EGY','GLF','LAV','MSA','NOR']\n",
    "\n",
    "\n",
    "\n",
    "# load tst.MGB3 labels\n",
    "filename = 'data/test.MGB3/reference'\n",
    "tst_ref_names = np.loadtxt(filename,usecols=[0],dtype='string')\n",
    "tst_ref_labels = np.loadtxt(filename,usecols=[1],dtype='int')\n",
    "\n",
    "tst_labels_index = []\n",
    "for i,name in enumerate(tst_names):\n",
    "    for j, name_ref in enumerate(tst_ref_names):\n",
    "        if name == name_ref:\n",
    "            tst_labels_index = np.append(tst_labels_index,j)\n",
    "\n",
    "tst_labels = np.empty((np.size(tst_labels_index)))\n",
    "for i,j in enumerate(tst_labels_index):\n",
    "    tst_labels[i]=tst_ref_labels[int(j)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# merge trn+dev\n",
    "trndev_features = np.append(trn_features, dev_features,axis=0)\n",
    "trndev_labels = np.append(trn_labels,dev_labels)\n",
    "trndev_names = np.append(trn_names,dev_names)\n",
    "\n",
    "# #center and length norm.\n",
    "# m=np.mean(trn_features,axis=0)\n",
    "# A = np.cov(trn_features.transpose())\n",
    "# [a,D,V] = np.linalg.svd(A)\n",
    "# V= V.transpose()\n",
    "# W= np.dot(V, np.diag(1./( np.sqrt(D) + 0.0000000001 )))\n",
    "\n",
    "\n",
    "# trn_features = np.dot( np.subtract( trn_features, m), W)\n",
    "# trndev_features = np.dot( np.subtract( trndev_features, m), W)\n",
    "# dev_features = np.dot( np.subtract( dev_features, m), W)\n",
    "# tst_features = np.dot( np.subtract( tst_features, m), W)\n",
    "\n",
    "# trn_features = it.length_norm(trn_features.transpose()).transpose()\n",
    "# trndev_features = it.length_norm(trndev_features.transpose()).transpose()\n",
    "# dev_features = it.length_norm(dev_features.transpose()).transpose()\n",
    "# tst_features = it.length_norm(tst_features.transpose()).transpose()\n",
    "\n",
    "# trn_features = it.length_norm(trn_features)\n",
    "# trndev_features = it.length_norm(trndev_features)\n",
    "# dev_features = it.length_norm(dev_features)\n",
    "# tst_features = it.length_norm(tst_features)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((13825, 50320), (1524, 50320), (5, 50320), (1492, 50320))\n"
     ]
    }
   ],
   "source": [
    "#language modeling\n",
    "lang_mean=[]\n",
    "for i, lang in enumerate(languages):\n",
    "#     lang_mean.append(np.mean(np.append(trn_features[np.nonzero(trn_labels == i+1)] ,dev_features[np.nonzero(dev_labels == i+1)],axis=0),axis=0))\n",
    "    lang_mean.append(np.mean( trn_features[np.nonzero(trn_labels == i+1)][:],axis=0 ) )\n",
    "\n",
    "lang_mean = np.array(lang_mean)\n",
    "lang_mean = it.length_norm(lang_mean)\n",
    "\n",
    "print( np.shape(trn_features), np.shape(dev_features), np.shape(lang_mean),np.shape(tst_features) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline accurary on test dataset : 0.369\n",
      "Baseline accurary on dev dataset : 0.357\n"
     ]
    }
   ],
   "source": [
    "# Baseline performance on TST using CDS\n",
    "tst_scores = lang_mean.dot(tst_features.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Baseline accurary on test dataset : %0.3f' %(acc)\n",
    "\n",
    "# Baseline performance on DEV using CDS\n",
    "dev_scores = lang_mean.dot(dev_features.transpose() )\n",
    "hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "temp = ((dev_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(dev_labels))\n",
    "print 'Baseline accurary on dev dataset : %0.3f' %(acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((69125,), (69125,), (69125,))\n",
      "((13825, 50320), (13825, 50320), (13825,), (55300, 50320), (55300, 50320), (55300,))\n"
     ]
    }
   ],
   "source": [
    "# making pair of train i-vector with mean of each language i-vector\n",
    "#  example : for total 3 ivectors\n",
    "#  ivector   ivector_p  label\n",
    "#     1         1         1\n",
    "#     1         2         0\n",
    "#     1         3         0\n",
    "#     2         1         0\n",
    "#     2         2         1\n",
    "#     ...      ...       ...\n",
    "#     3         3         1\n",
    "\n",
    "# preparing pair labels\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(trn_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( trn_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# TRN dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "trn_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "trn_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "trn_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "trn_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((7620,), (7620,), (7620,))\n",
      "((1524, 50320), (1524, 50320), (1524,), (6096, 50320), (6096, 50320), (6096,))\n"
     ]
    }
   ],
   "source": [
    "# preparing pair labels of DEV dataset\n",
    "sim = []\n",
    "pair_a_idx = []\n",
    "pair_b_idx = []\n",
    "for i, lang in enumerate(languages):\n",
    "    for j, label in enumerate(dev_labels):\n",
    "#         print i, j, label\n",
    "        pair_a_idx.append(i+1)\n",
    "        pair_b_idx.append(j)\n",
    "        if i+1 == label:\n",
    "            sim.append(1)\n",
    "        else:\n",
    "            sim.append(0)\n",
    "print(np.shape(pair_a_idx),np.shape(pair_b_idx), np.shape(sim))\n",
    "pair_a_idx=np.array(pair_a_idx)\n",
    "pair_b_idx=np.array(pair_b_idx)\n",
    "sim = np.array(sim)\n",
    "\n",
    "#shuffling\n",
    "shuffleidx = np.arange(0,np.size(pair_a_idx))\n",
    "np.random.shuffle(shuffleidx)\n",
    "pair_a_idx = pair_a_idx[shuffleidx]\n",
    "pair_b_idx = pair_b_idx[shuffleidx]\n",
    "sim = sim[shuffleidx]\n",
    "\n",
    "\n",
    "data = []\n",
    "data_p = []\n",
    "    \n",
    "for iter in np.arange(0,np.size(sim)) :\n",
    "    data.append( lang_mean[pair_a_idx[iter]-1] )\n",
    "    data_p.append( dev_features[pair_b_idx[iter]] )\n",
    "data = np.array(data)\n",
    "data_p = np.array(data_p)\n",
    "\n",
    "# DEV dataset loading for feeding \n",
    "tar_data = data[sim==1]\n",
    "tar_data_p = data_p[sim==1]\n",
    "tar_sim = sim[sim==1]\n",
    "non_data = data[sim==0]\n",
    "non_data_p = data_p[sim==0]\n",
    "non_sim = sim[sim==0]\n",
    "print(tar_data.shape, tar_data_p.shape,tar_sim.shape,non_data.shape,non_data_p.shape,non_sim.shape)\n",
    "\n",
    "dev_tar = ivector_dataset.DataSet(tar_data,tar_sim)\n",
    "dev_tar_p = ivector_dataset.DataSet(tar_data_p,tar_sim)\n",
    "\n",
    "dev_non = ivector_dataset.DataSet(non_data,non_sim)\n",
    "dev_non_p = ivector_dataset.DataSet(non_data_p,non_sim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(?, 91520)\n",
      "(?, 1500)\n",
      "(?, 600)\n",
      "(?, 91520)\n",
      "(?, 1500)\n",
      "(?, 600)\n"
     ]
    }
   ],
   "source": [
    "# init variables\n",
    "sess = tf.InteractiveSession()\n",
    "siamese = siamese_model.siamese(np.shape(trn_features)[1]);\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learning_rate = tf.train.exponential_decay(0.01, global_step,\n",
    "                                           5000, 0.99, staircase=True)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(siamese.loss, global_step=global_step)\n",
    "saver = tf.train.Saver()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0: loss 1.000261, Acc.: (DEV)0.282 (TST)0.275, lr : 0.01000\n",
      "Step 2500: loss 0.999950, Acc.: (DEV)0.260 (TST)0.280, lr : 0.01000\n",
      "Step 2700: loss 0.999957, Acc.: (DEV)0.280 (TST)0.295, lr : 0.01000\n",
      "Step 2900: loss 0.999887, Acc.: (DEV)0.287 (TST)0.299, lr : 0.01000\n",
      "Step 3100: loss 1.000019, Acc.: (DEV)0.293 (TST)0.301, lr : 0.01000\n",
      "Step 3300: loss 0.999946, Acc.: (DEV)0.297 (TST)0.308, lr : 0.01000\n",
      "Step 3500: loss 0.999947, Acc.: (DEV)0.302 (TST)0.316, lr : 0.01000\n",
      "Step 4600: loss 0.999893, Acc.: (DEV)0.322 (TST)0.320, lr : 0.01000\n",
      "Step 4700: loss 0.999850, Acc.: (DEV)0.330 (TST)0.328, lr : 0.01000\n",
      "Step 5000: loss 0.999886, Acc.: (DEV)0.332 (TST)0.329, lr : 0.00990\n",
      "Step 5100: loss 0.999977, Acc.: (DEV)0.331 (TST)0.331, lr : 0.00990\n",
      "Step 5200: loss 0.999779, Acc.: (DEV)0.339 (TST)0.334, lr : 0.00990\n",
      "Step 5400: loss 0.999818, Acc.: (DEV)0.348 (TST)0.349, lr : 0.00990\n",
      "Step 5700: loss 1.000127, Acc.: (DEV)0.346 (TST)0.355, lr : 0.00990\n",
      "Step 6000: loss 1.000118, Acc.: (DEV)0.352 (TST)0.356, lr : 0.00990\n",
      "Step 6100: loss 1.000142, Acc.: (DEV)0.355 (TST)0.358, lr : 0.00990\n",
      "Step 6200: loss 0.999486, Acc.: (DEV)0.356 (TST)0.360, lr : 0.00990\n",
      "Step 6300: loss 0.999685, Acc.: (DEV)0.366 (TST)0.364, lr : 0.00990\n",
      "Step 6400: loss 0.999697, Acc.: (DEV)0.373 (TST)0.367, lr : 0.00990\n",
      "Step 6600: loss 1.000206, Acc.: (DEV)0.366 (TST)0.368, lr : 0.00990\n",
      "Step 6700: loss 0.999401, Acc.: (DEV)0.374 (TST)0.370, lr : 0.00990\n",
      "Step 6800: loss 0.999732, Acc.: (DEV)0.382 (TST)0.371, lr : 0.00990\n",
      "Step 7100: loss 0.999670, Acc.: (DEV)0.391 (TST)0.377, lr : 0.00990\n",
      "Step 7300: loss 0.999684, Acc.: (DEV)0.391 (TST)0.379, lr : 0.00990\n",
      "Step 7500: loss 0.999502, Acc.: (DEV)0.398 (TST)0.397, lr : 0.00990\n",
      "Step 8100: loss 0.999331, Acc.: (DEV)0.398 (TST)0.405, lr : 0.00990\n",
      "Step 8400: loss 1.001665, Acc.: (DEV)0.402 (TST)0.405, lr : 0.00990\n",
      "Step 17500: loss 0.959495, Acc.: (DEV)0.433 (TST)0.407, lr : 0.00970\n",
      "Step 17800: loss 0.973635, Acc.: (DEV)0.431 (TST)0.408, lr : 0.00970\n",
      "Step 18000: loss 0.954654, Acc.: (DEV)0.434 (TST)0.411, lr : 0.00970\n",
      "Step 18100: loss 0.979962, Acc.: (DEV)0.438 (TST)0.419, lr : 0.00970\n",
      "Step 18300: loss 0.999485, Acc.: (DEV)0.450 (TST)0.422, lr : 0.00970\n",
      "Step 18400: loss 0.951895, Acc.: (DEV)0.444 (TST)0.423, lr : 0.00970\n",
      "Step 18700: loss 0.965161, Acc.: (DEV)0.442 (TST)0.429, lr : 0.00970\n",
      "Step 19600: loss 0.986405, Acc.: (DEV)0.458 (TST)0.432, lr : 0.00970\n",
      "Step 19900: loss 0.941757, Acc.: (DEV)0.457 (TST)0.436, lr : 0.00970\n",
      "Step 20300: loss 0.955309, Acc.: (DEV)0.478 (TST)0.438, lr : 0.00961\n",
      "Step 20400: loss 0.953324, Acc.: (DEV)0.472 (TST)0.442, lr : 0.00961\n",
      "Step 20800: loss 0.960407, Acc.: (DEV)0.479 (TST)0.442, lr : 0.00961\n",
      "Step 21100: loss 0.980738, Acc.: (DEV)0.489 (TST)0.443, lr : 0.00961\n",
      "Step 21300: loss 0.921135, Acc.: (DEV)0.490 (TST)0.445, lr : 0.00961\n",
      "Step 21600: loss 0.936776, Acc.: (DEV)0.481 (TST)0.446, lr : 0.00961\n",
      "Step 22300: loss 0.970981, Acc.: (DEV)0.498 (TST)0.448, lr : 0.00961\n",
      "Step 23300: loss 0.916188, Acc.: (DEV)0.523 (TST)0.450, lr : 0.00961\n",
      "Step 23400: loss 0.967541, Acc.: (DEV)0.512 (TST)0.456, lr : 0.00961\n",
      "Step 24300: loss 0.895224, Acc.: (DEV)0.536 (TST)0.462, lr : 0.00961\n",
      "Step 27600: loss 0.950614, Acc.: (DEV)0.652 (TST)0.467, lr : 0.00951\n",
      "Step 28200: loss 0.924656, Acc.: (DEV)0.675 (TST)0.477, lr : 0.00951\n",
      "Step 29000: loss 0.902471, Acc.: (DEV)0.690 (TST)0.485, lr : 0.00951\n",
      "Step 29400: loss 0.902886, Acc.: (DEV)0.705 (TST)0.486, lr : 0.00951\n",
      "Step 29800: loss 0.894189, Acc.: (DEV)0.718 (TST)0.489, lr : 0.00951\n",
      "Step 30200: loss 0.942703, Acc.: (DEV)0.741 (TST)0.497, lr : 0.00941\n",
      "Step 31100: loss 0.977978, Acc.: (DEV)0.772 (TST)0.501, lr : 0.00941\n",
      "Step 31300: loss 0.926331, Acc.: (DEV)0.766 (TST)0.505, lr : 0.00941\n",
      "Step 31400: loss 0.874906, Acc.: (DEV)0.756 (TST)0.511, lr : 0.00941\n",
      "Step 31500: loss 0.839176, Acc.: (DEV)0.776 (TST)0.512, lr : 0.00941\n",
      "Step 32200: loss 0.919292, Acc.: (DEV)0.801 (TST)0.514, lr : 0.00941\n",
      "Step 32300: loss 0.917150, Acc.: (DEV)0.804 (TST)0.518, lr : 0.00941\n",
      "Step 33100: loss 0.935891, Acc.: (DEV)0.831 (TST)0.526, lr : 0.00941\n",
      "Step 34600: loss 0.860718, Acc.: (DEV)0.900 (TST)0.527, lr : 0.00941\n",
      "Step 34900: loss 0.910826, Acc.: (DEV)0.900 (TST)0.532, lr : 0.00941\n",
      "Step 38900: loss 0.821969, Acc.: (DEV)0.991 (TST)0.535, lr : 0.00932\n",
      "Step 44400: loss 0.846349, Acc.: (DEV)1.000 (TST)0.541, lr : 0.00923\n",
      "Step 48400: loss 0.837504, Acc.: (DEV)1.000 (TST)0.545, lr : 0.00914\n",
      "Step 49900: loss 0.787843, Acc.: (DEV)1.000 (TST)0.546, lr : 0.00914\n",
      "Step 60400: loss 0.715923, Acc.: (DEV)1.000 (TST)0.548, lr : 0.00886\n"
     ]
    }
   ],
   "source": [
    "#start training\n",
    "batch_size = 40\n",
    "max_acc = 0.0\n",
    "max_step=0\n",
    "saver_folder='snnmodel_phone_hu'\n",
    "if not os.path.exists(saver_folder):\n",
    "    os.mkdir(saver_folder)\n",
    "for step in range(500000):\n",
    "  \n",
    "    if step %5 ==0:\n",
    "        batch_x1_a, batch_y1_a = trn_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = trn_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = trn_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = trn_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "    else:\n",
    "        batch_x1_a, batch_y1_a = dev_tar.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_a, batch_y2_a = dev_tar_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1_b, batch_y1_b = dev_non.next_batch(batch_size,shuffle=False)\n",
    "        batch_x2_b, batch_y2_b = dev_non_p.next_batch(batch_size,shuffle=False)\n",
    "        batch_x1 = np.append(batch_x1_a,batch_x1_b,axis=0)\n",
    "        batch_y1 = np.append(batch_y1_a,batch_y1_b,axis=0)\n",
    "        batch_x2 = np.append(batch_x2_a,batch_x2_b,axis=0)\n",
    "        batch_y2 = np.append(batch_y2_a,batch_y2_b,axis=0)\n",
    "        \n",
    "\n",
    "#     batch_x1,batch_y1 = mgb3_siam1.train.next_batch(120,shuffle=False)\n",
    "#     batch_x2,batch_y2 = mgb3_siam2.train.next_batch(120,shuffle=False)    \n",
    "#     batch_y = (batch_y1==batch_y2).astype('float')\n",
    "    batch_y = batch_y1*2-1\n",
    "#     batch_y = 1-batch_y1\n",
    "    \n",
    "    _, loss_v = sess.run([train_step, siamese.loss], feed_dict={\n",
    "        siamese.x1: batch_x1,\n",
    "        siamese.x2: batch_x2,\n",
    "        siamese.y_: batch_y\n",
    "    })\n",
    "    \n",
    "    if np.isnan(loss_v):\n",
    "        print ('Model diverged with loss = NAN')\n",
    "        quit()\n",
    "        \n",
    "    if step % 100 ==0:\n",
    "        dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "        lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "        tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "        \n",
    "        dev_scores = lang_mean_siam.dot(dev_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(dev_scores,axis = 0)\n",
    "        temp = ((dev_labels-1) - hypo_lang)\n",
    "        acc =1- np.size(np.nonzero(temp)) / float( np.size(dev_labels) )\n",
    "        \n",
    "        tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "        hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "        temp = ((tst_labels-1) - hypo_lang)\n",
    "        acc_tst =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "\n",
    "        if max_acc < acc_tst:\n",
    "            max_acc = acc_tst\n",
    "            max_step=step\n",
    "            print ('Step %d: loss %f, Acc.: (DEV)%.3f (TST)%.3f, lr : %.5f' % (step,loss_v,acc,acc_tst,sess.run(learning_rate)))\n",
    "            saver.save(sess, saver_folder+'/model'+str(step)+'.ckpt')\n",
    "        if loss_v<0.3:\n",
    "            break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60400\n",
      "INFO:tensorflow:Restoring parameters from snnmodel_phone_hu/model60400.ckpt\n",
      "The history saving thread hit an unexpected error (OperationalError('disk I/O error',)).History will not be written to the database.\n",
      "Final accurary on test dataset : 0.548\n"
     ]
    }
   ],
   "source": [
    "print max_step\n",
    "RESTORE_STEP=max_step\n",
    "saver.restore(sess, saver_folder+'/model'+str(RESTORE_STEP)+'.ckpt')\n",
    "\n",
    "\n",
    "# trn_features_siam = siamese.o1.eval({siamese.x1:trn_features})\n",
    "dev_features_siam = siamese.o1.eval({siamese.x1:dev_features})\n",
    "tst_features_siam = siamese.o1.eval({siamese.x1:tst_features})\n",
    "lang_mean_siam = siamese.o1.eval({siamese.x1:lang_mean})\n",
    "\n",
    "tst_scores = lang_mean_siam.dot(tst_features_siam.transpose() )\n",
    "# print(tst_scores.shape)\n",
    "hypo_lang = np.argmax(tst_scores,axis = 0)\n",
    "temp = ((tst_labels-1) - hypo_lang)\n",
    "acc =1- np.size(np.nonzero(temp)) / float(np.size(tst_labels))\n",
    "print 'Final accurary on test dataset : %0.3f' %(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix\n",
      "[[ 170.   34.   73.   19.   50.]\n",
      " [  25.  132.   62.   21.   27.]\n",
      " [  46.   55.  137.   19.   51.]\n",
      " [  17.   11.   15.  187.   25.]\n",
      " [  44.   18.   47.   16.  191.]]\n",
      "Precision\n",
      "[ 0.49132948  0.49438202  0.44480519  0.73333333  0.60443038]\n",
      "Recall\n",
      "[ 0.56291391  0.528       0.41017964  0.71374046  0.55523256]\n",
      "\n",
      "\n",
      "<Performance evaluation on Test dataset>\n",
      "Accurary  : 0.548\n",
      "Precision : 0.554\n",
      "Recall    : 0.554\n"
     ]
    }
   ],
   "source": [
    "confusionmat = np.zeros((5,5))\n",
    "for i,lang in enumerate(languages):\n",
    "    hypo_bylang = hypo_lang[ tst_labels == i+1]\n",
    "    hist_bylang = np.histogram(hypo_bylang,5)\n",
    "    confusionmat[:,i] = hist_bylang[0]\n",
    "\n",
    "precision = np.diag(confusionmat) / np.sum(confusionmat,axis=1) #precision\n",
    "recall = np.diag(confusionmat) / np.sum(confusionmat,axis=0) # recall\n",
    "    \n",
    "print 'Confusion matrix'\n",
    "print confusionmat\n",
    "print 'Precision'\n",
    "print precision\n",
    "print 'Recall'\n",
    "print recall\n",
    "\n",
    "print '\\n\\n<Performance evaluation on Test dataset>'\n",
    "print 'Accurary  : %0.3f' %(acc)\n",
    "print 'Precision : %0.3f' %(np.mean(precision))\n",
    "print 'Recall    : %0.3f' %(np.mean(recall))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}