2.3 how mfcc performs on the small dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Channel-last, i.e., (None, n_freq, n_time, n_ch)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from builtins import range\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import librosa, librosa.display\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Activation, Dense, Flatten, Input, Reshape, Dropout, Permute\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import MaxPooling2D, GlobalAveragePooling2D\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras import backend as K\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\" # the number of the GPU\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.1 # percentage to be used\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "from kapre.time_frequency import Melspectrogram\n",
    "from global_config import *\n",
    "\n",
    "import multiprocessing\n",
    "from multiprocessing import Pool\n",
    "N_JOBS = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_filepaths(df, dataroot=None):\n",
    "    \"\"\"Generate file path (column name 'filepath') from given dataframe \"\"\"\n",
    "    for filepath in df.filepath.values:\n",
    "        yield os.path.join(dataroot, filepath)\n",
    "        \n",
    "def get_mfcc(filename, dataroot):    \n",
    "    csv_filename = 'pedal-{}_npydf_small.csv'.format(filename)\n",
    "    df = pd.read_csv(os.path.join(DIR_PEDAL_METADATA, csv_filename))\n",
    "    training = df.loc[df['category'] == 'train']\n",
    "    validation = df.loc[df['category'] == 'valid']\n",
    "    print('pedal-{}: Dataframe with size {} for training and {} for validation.'.format(filename,len(training),len(validation)))\n",
    "    \n",
    "    for (task_data, task_name) in zip([training, validation], ['train', 'valid']):\n",
    "        print('Getting MFCC features to {}...'.format(task_name))\n",
    "        npy_filename = 'small-{}_{}_mfcc.npy'.format(filename, task_name)\n",
    "        gen_f = gen_filepaths(task_data, dataroot=dataroot)\n",
    "\n",
    "        pool = Pool(N_JOBS)\n",
    "        paths = list(gen_f)\n",
    "        feats = pool.map(_path_to_mfccs, paths)\n",
    "        feats = np.array(feats)\n",
    "        np.save(os.path.join(DIR_SAVE_MODEL, npy_filename), feats)\n",
    "        print('  done!')\n",
    "        pool.close()\n",
    "        pool.join()\n",
    "    \n",
    "def _path_to_mfccs(path):\n",
    "    src = np.load(path)\n",
    "    mfcc = librosa.feature.mfcc(src, SR, n_mfcc=20)\n",
    "    dmfcc = mfcc[:, 1:] - mfcc[:, :-1]\n",
    "    ddmfcc = dmfcc[:, 1:] - dmfcc[:, :-1]\n",
    "    return np.concatenate((np.mean(mfcc, axis=1), np.std(mfcc, axis=1),\n",
    "                           np.mean(dmfcc, axis=1), np.std(dmfcc, axis=1),\n",
    "                           np.mean(ddmfcc, axis=1), np.std(ddmfcc, axis=1)), axis=0)\n",
    "\n",
    "def load_xy(filename, task_name):\n",
    "    \n",
    "    npy_filename = 'small-{}_{}_mfcc.npy'.format(filename, task_name)\n",
    "    x = np.load(os.path.join(DIR_SAVE_MODEL, npy_filename))\n",
    "    \n",
    "    csv_filename = 'pedal-{}_npydf_small.csv'.format(filename)\n",
    "    df = pd.read_csv(os.path.join(DIR_PEDAL_METADATA, csv_filename))\n",
    "    task_data = df.loc[df['category'] == task_name]   \n",
    "    y = task_data.label.values\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get MFCC features from the small datasets\n",
    "\n",
    "Features are saved in `./save-model/small-{onset or segment}_{train or valid}_mfcc.npy`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "filenames = ['onset', 'segment']\n",
    "dataroots = [DIR_PEDAL_ONSET_NPY, DIR_PEDAL_SEGMENT_NPY]\n",
    "for idx, (filename, dr) in enumerate(zip(filenames, dataroots)):\n",
    "    get_mfcc(filename, dataroot=dr)\n",
    "```\n",
    "Print out:\n",
    "```\n",
    "pedal-onset: Dataframe with size 70000 for training and 20000 for validation.\n",
    "Getting MFCC features to train...\n",
    "  done!\n",
    "Getting MFCC features to valid...\n",
    "  done!\n",
    "pedal-segment: Dataframe with size 70000 for training and 20000 for validation.\n",
    "Getting MFCC features to train...\n",
    "  done!\n",
    "Getting MFCC features to valid...\n",
    "  done!\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do grid search cross validation to get the best SVM parameters for binary classification\n",
    "\n",
    "Run `python mfcc_parametergrid.py` to save every performance matrix in `./save-model/small-{onset or segment}_mfcc_svc_performance.npz` and print out the best parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using a small portion of data for test run\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import GridSearchCV, ParameterGrid\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "class OptionalStandardScaler(StandardScaler):\n",
    "    def __init__(self, on=False):\n",
    "        self.on = on  # bool\n",
    "        if self.on:\n",
    "            super(OptionalStandardScaler, self).__init__(with_mean=True, with_std=True)\n",
    "        else:\n",
    "            super(OptionalStandardScaler, self).__init__(with_mean=False, with_std=False)\n",
    "\n",
    "n_cpu = multiprocessing.cpu_count()\n",
    "n_jobs = int(n_cpu * 0.8)\n",
    "print('There are {} cpu available, {} (80%) of them will be used for our jobs.'.format(n_cpu, n_jobs))\n",
    "\n",
    "gps = [{\"C\": [0.1, 2.0], \"kernel\": ['rbf'],\n",
    "        \"gamma\": [0.5 ** i for i in [3, 5]]},\n",
    "       {\"C\": [0.1, 2.0], \"kernel\": ['linear']}\n",
    "      ]\n",
    "classifier = SVC\n",
    "dataroots = [DIR_PEDAL_ONSET_NPY, DIR_PEDAL_SEGMENT_NPY]\n",
    "filenames = ['onset', 'segment']\n",
    "\n",
    "    \n",
    "# tiny version of mfcc_parametergrid.py \n",
    "print('===== Test run of mfcc_parametergrid =====')\n",
    "for filename in filenames:  \n",
    "    print('----- Pedal-{} SVC Performance -----'.format(filename))\n",
    "    x_train, y_train = load_xy(filename, task_name='train')\n",
    "    x_valid, y_valid = load_xy(filename, task_name='valid')\n",
    "    x_train, y_train = [x_train[:30], y_train[:30]]\n",
    "    x_valid, y_valid = [x_valid[:10], y_valid[:10]]\n",
    "    \n",
    "    for parameter in list(ParameterGrid(gps)):\n",
    "        if parameter['kernel']=='linear':\n",
    "            clf = SVC(kernel='linear', C=parameter['C']).fit(x_train, y_train)\n",
    "        else:\n",
    "            clf = SVC(kernel=parameter['kernel'], C=parameter['C'], gamma=parameter['gamma']).fit(x_train, y_train)\n",
    "\n",
    "        y_pred = clf.predict(x_valid)\n",
    "        loss_reg = log_loss(y_valid, y_pred)\n",
    "        acc_reg = clf.score(x_valid, y_valid)\n",
    "        auc_reg = roc_auc_score(y_valid, y_pred)\n",
    "\n",
    "        print('{}'.format(parameter))\n",
    "        print(\"      valid set loss: {}\".format(loss_reg))\n",
    "        print(\"  valid set accuracy: {}\".format(acc_reg))\n",
    "        print(\"       valid set auc: {}\".format(auc_reg))\n",
    "        print('-'*20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the SVM with best parameters to get the scores on the small validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best parameter was chosen from\n",
      " 1: {'kernel': 'rbf', 'C': 0.1, 'gamma': 0.125}\n",
      " 2: {'kernel': 'rbf', 'C': 0.1, 'gamma': 0.03125}\n",
      " 3: {'kernel': 'rbf', 'C': 0.1, 'gamma': 0.0078125}\n",
      " 4: {'kernel': 'rbf', 'C': 0.1, 'gamma': 0.001953125}\n",
      " 5: {'kernel': 'rbf', 'C': 0.1, 'gamma': 0.00048828125}\n",
      " 6: {'kernel': 'rbf', 'C': 0.1, 'gamma': 0.0001220703125}\n",
      " 7: {'kernel': 'rbf', 'C': 0.1, 'gamma': 'auto'}\n",
      " 8: {'kernel': 'rbf', 'C': 2.0, 'gamma': 0.125}\n",
      " 9: {'kernel': 'rbf', 'C': 2.0, 'gamma': 0.03125}\n",
      "10: {'kernel': 'rbf', 'C': 2.0, 'gamma': 0.0078125}\n",
      "11: {'kernel': 'rbf', 'C': 2.0, 'gamma': 0.001953125}\n",
      "12: {'kernel': 'rbf', 'C': 2.0, 'gamma': 0.00048828125}\n",
      "13: {'kernel': 'rbf', 'C': 2.0, 'gamma': 0.0001220703125}\n",
      "14: {'kernel': 'rbf', 'C': 2.0, 'gamma': 'auto'}\n",
      "15: {'kernel': 'rbf', 'C': 8.0, 'gamma': 0.125}\n",
      "16: {'kernel': 'rbf', 'C': 8.0, 'gamma': 0.03125}\n",
      "17: {'kernel': 'rbf', 'C': 8.0, 'gamma': 0.0078125}\n",
      "18: {'kernel': 'rbf', 'C': 8.0, 'gamma': 0.001953125}\n",
      "19: {'kernel': 'rbf', 'C': 8.0, 'gamma': 0.00048828125}\n",
      "20: {'kernel': 'rbf', 'C': 8.0, 'gamma': 0.0001220703125}\n",
      "21: {'kernel': 'rbf', 'C': 8.0, 'gamma': 'auto'}\n",
      "22: {'kernel': 'rbf', 'C': 32.0, 'gamma': 0.125}\n",
      "23: {'kernel': 'rbf', 'C': 32.0, 'gamma': 0.03125}\n",
      "24: {'kernel': 'rbf', 'C': 32.0, 'gamma': 0.0078125}\n",
      "25: {'kernel': 'rbf', 'C': 32.0, 'gamma': 0.001953125}\n",
      "26: {'kernel': 'rbf', 'C': 32.0, 'gamma': 0.00048828125}\n",
      "27: {'kernel': 'rbf', 'C': 32.0, 'gamma': 0.0001220703125}\n",
      "28: {'kernel': 'rbf', 'C': 32.0, 'gamma': 'auto'}\n",
      "29: {'kernel': 'linear', 'C': 0.1}\n",
      "30: {'kernel': 'linear', 'C': 2.0}\n",
      "31: {'kernel': 'linear', 'C': 8.0}\n",
      "32: {'kernel': 'linear', 'C': 32.0}\n"
     ]
    }
   ],
   "source": [
    "print(\"The best parameter was chosen from\")\n",
    "gps = [{\"C\": [0.1, 2.0, 8.0, 32.0], \"kernel\": ['rbf'],\n",
    "        \"gamma\": [0.5 ** i for i in [3, 5, 7, 9, 11, 13]] + ['auto']},\n",
    "       {\"C\": [0.1, 2.0, 8.0, 32.0], \"kernel\": ['linear']}\n",
    "      ]\n",
    "for idx, parameter in enumerate(list(ParameterGrid(gps))):\n",
    "    print(\"{:2d}: {}\".format(idx+1, parameter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===== Pedal-onset SVC Best Parameters =====\n",
      "{'kernel': 'rbf', 'C': 32.0, 'gamma': 0.0001220703125}\n",
      "      valid set loss: 5.28276298084\n",
      "  valid set accuracy: 0.84705\n",
      "       valid set auc: 0.84705\n",
      "===== Pedal-segment SVC Best Parameters =====\n",
      "{'kernel': 'rbf', 'C': 32.0, 'gamma': 0.0001220703125}\n",
      "      valid set loss: 2.85811900893\n",
      "  valid set accuracy: 0.91725\n",
      "       valid set auc: 0.91725\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import GridSearchCV, ParameterGrid\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "filenames = ['onset', 'segment']\n",
    "dataroots = [DIR_PEDAL_ONSET_NPY, DIR_PEDAL_SEGMENT_NPY]\n",
    "\n",
    "for filename in filenames:\n",
    "    print('===== Pedal-{} SVC Best Parameters ====='.format(filename))\n",
    "        \n",
    "    performances = np.load(os.path.join(DIR_SAVE_MODEL,'small-{}_mfcc_svc_performance.npz'.format(filename)))\n",
    "    parameters = performances['parameter']\n",
    "    losss = performances['loss']\n",
    "    accs = performances['acc'] \n",
    "    aucs = performances['auc'] \n",
    "    \n",
    "    parameter_reg, loss_reg, acc_reg, auc_reg = [{}, float('inf'), float('-inf'), float('-inf')]\n",
    "    for (parameter, loss, acc, auc) in zip(parameters, losss, accs, aucs):\n",
    "        if auc > auc_reg:\n",
    "            parameter_reg = parameter\n",
    "            auc_reg = auc\n",
    "            acc_reg = acc\n",
    "            loss_reg = loss\n",
    "        elif auc == auc_reg:\n",
    "            if acc > acc_reg:\n",
    "                arameter_reg = parameter\n",
    "                auc_reg = auc\n",
    "                acc_reg = acc\n",
    "                loss_reg = loss \n",
    "            elif acc == acc_reg:\n",
    "                if loss <= loss_reg:\n",
    "                    arameter_reg = parameter\n",
    "                    auc_reg = auc\n",
    "                    acc_reg = acc\n",
    "                    loss_reg = loss   \n",
    "    \n",
    "    print('{}'.format(parameter_reg))\n",
    "    print(\"      valid set loss: {}\".format(loss_reg))\n",
    "    print(\"  valid set accuracy: {}\".format(acc_reg))\n",
    "    print(\"       valid set auc: {}\".format(auc_reg))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}