1.2 sub-dataset preparation (incl. maestro).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the sub dataset to compare the performance of different CNN models\n",
    "\n",
    "Here the pedal onset/segment data in the sub dataset are from (up to) 1000 samples of each composer. The data are then split into 0.8/0.2 for training/validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Channel-last, i.e., (None, n_freq, n_time, n_ch)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.signal import medfilt\n",
    "from builtins import range\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.metrics import accuracy_score\n",
    "import mir_eval\n",
    "import librosa, librosa.display\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Activation, Dense, Flatten, Input, Reshape, Dropout, Permute\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import MaxPooling2D, GlobalAveragePooling2D\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras import backend as K\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "from keras.layers import concatenate as concat\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\" # the number of the GPU\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.1 # percentage to be used\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "from kapre.time_frequency import Melspectrogram\n",
    "from global_config import *\n",
    "\n",
    "\n",
    "vd_pedal_onset = os.path.join(DIR_PEDAL_METADATA, 'pedal-onset_vd.csv')\n",
    "vd_pedal_segment = os.path.join(DIR_PEDAL_METADATA, 'pedal-segment_vd.csv')\n",
    "DF_ONSET = pd.read_csv(vd_pedal_onset)\n",
    "DF_SEGMENT = pd.read_csv(vd_pedal_segment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF_ONSET_20024689 = DF_ONSET.loc[DF_ONSET['category'] != 'test']\n",
    "DF_SEGMENT_20024689 = DF_SEGMENT.loc[DF_SEGMENT['category'] != 'test']\n",
    "subexcerpts_num = 1000\n",
    "\n",
    "txt_filename = 'filename-20024689-composer.txt'\n",
    "txt_path = os.path.join(DIR_PEDAL_METADATA, txt_filename)\n",
    "filenames = np.genfromtxt(txt_path, dtype=None)\n",
    "\n",
    "list_with_keys, list_with_values = [], []\n",
    "for filename in filenames:\n",
    "    filename_list = filename.split(\"/\")\n",
    "    list_with_keys.append(\"{}/{}\".format(filename_list[0],filename_list[1])) \n",
    "    list_with_values.append(filename_list[2])\n",
    "\n",
    "filename_composer = dict(zip(list_with_keys, list_with_values))\n",
    "\n",
    "author = []\n",
    "for filename in DF_ONSET_20024689.filename.values:\n",
    "    filename_list = filename.split(\"-\")\n",
    "    key_str = \"{}/{}\".format(filename_list[0],filename_list[1])\n",
    "    value_str = filename_composer[key_str]\n",
    "    author.append(value_str)\n",
    "DF_ONSET_20024689 = DF_ONSET_20024689.copy()\n",
    "DF_ONSET_20024689['author'] = author\n",
    "\n",
    "author = []\n",
    "for filename in DF_SEGMENT_20024689.filename.values:\n",
    "    filename_list = filename.split(\"-\")\n",
    "    key_str = \"{}/{}\".format(filename_list[0],filename_list[1])\n",
    "    value_str = filename_composer[key_str]\n",
    "    author.append(value_str) \n",
    "DF_SEGMENT_20024689 = DF_SEGMENT_20024689.copy()\n",
    "DF_SEGMENT_20024689['author'] = author\n",
    "\n",
    "unique_author = np.unique(DF_SEGMENT_20024689.author.values)\n",
    "\n",
    "onset_20024689_author_counter = dict()\n",
    "for authorname in unique_author:\n",
    "    onset_20024689_author_counter[authorname] = len(DF_ONSET_20024689.loc[DF_ONSET_20024689['author'] == authorname])\n",
    "\n",
    "segment_20024689_author_counter = dict()\n",
    "for authorname in unique_author:\n",
    "    segment_20024689_author_counter[authorname] = len(DF_SEGMENT_20024689.loc[DF_SEGMENT_20024689['author'] == authorname])\n",
    "\n",
    "onset_20024689_author_subnum = dict()\n",
    "for k, v in onset_20024689_author_counter.iteritems():\n",
    "    if v<subexcerpts_num:\n",
    "        onset_20024689_author_subnum[k] = v\n",
    "    else:\n",
    "        onset_20024689_author_subnum[k] = subexcerpts_num\n",
    "onset_20024689_author_subnum_sum = np.sum(onset_20024689_author_subnum.values())   \n",
    "\n",
    "segment_20024689_author_subnum = dict()\n",
    "for k, v in segment_20024689_author_counter.iteritems():\n",
    "    if v<subexcerpts_num:\n",
    "        segment_20024689_author_subnum[k] = v\n",
    "    else:\n",
    "        segment_20024689_author_subnum[k] = subexcerpts_num\n",
    "segment_20024689_author_subnum_sum = np.sum(segment_20024689_author_subnum.values())   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split from `onset_20024689_author_counter` and `segment_20024689_author_counter` to train/valid set as 0.8/0.2 (composer wise)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/homes/bl301/tf-env/lib/python2.7/site-packages/pandas/core/indexing.py:537: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "onset_tv_df = pd.DataFrame()\n",
    "for authorname, num in onset_20024689_author_counter.iteritems():\n",
    "    onset_sub = DF_ONSET_20024689.loc[DF_ONSET_20024689['author'] == authorname]\n",
    "    pedal_indx = np.arange(0, len(onset_sub)-1, 2, dtype=int)\n",
    "    pedal_indx_num = random.sample(pedal_indx, int(num/2))\n",
    "    pedal_indx_num_train = random.sample(pedal_indx_num, int(len(pedal_indx_num)*0.8))\n",
    "    pedal_indx_num_valid = [e for e in pedal_indx_num if e not in pedal_indx_num_train]\n",
    "    indx_num_train = np.sort(np.concatenate((np.asarray(pedal_indx_num_train),np.asarray(pedal_indx_num_train)+1), axis=0))\n",
    "    indx_num_valid = np.sort(np.concatenate((np.asarray(pedal_indx_num_valid),np.asarray(pedal_indx_num_valid)+1), axis=0))\n",
    "\n",
    "    onset_sub_train = onset_sub.iloc[indx_num_train]\n",
    "    onset_sub_valid = onset_sub.iloc[indx_num_valid]\n",
    "    onset_sub_train.loc[:,'category'] = 'train'\n",
    "    onset_sub_valid.loc[:,'category'] = 'valid'\n",
    "    onset_sub_tv = onset_sub_train.append(onset_sub_valid)\n",
    "\n",
    "    if len(onset_tv_df) == 0:\n",
    "        onset_tv_df = onset_sub_tv.copy()\n",
    "    else:\n",
    "        onset_tv_df = onset_tv_df.append(onset_sub_tv)\n",
    "\n",
    "onset_tv_df.to_csv(os.path.join(DIR_PEDAL_METADATA, 'pedal-onset_tvdf.csv'))\n",
    "\n",
    "segment_tv_df = pd.DataFrame()\n",
    "for authorname, num in segment_20024689_author_counter.iteritems():\n",
    "    segment_sub = DF_SEGMENT_20024689.loc[DF_SEGMENT_20024689['author'] == authorname]\n",
    "    pedal_indx = np.arange(0, len(segment_sub)-1, 2, dtype=int)\n",
    "    pedal_indx_num = random.sample(pedal_indx, int(num/2))\n",
    "    pedal_indx_num_train = random.sample(pedal_indx_num, int(len(pedal_indx_num)*0.8))\n",
    "    pedal_indx_num_valid = [e for e in pedal_indx_num if e not in pedal_indx_num_train]\n",
    "    indx_num_train = np.sort(np.concatenate((np.asarray(pedal_indx_num_train),np.asarray(pedal_indx_num_train)+1), axis=0))\n",
    "    indx_num_valid = np.sort(np.concatenate((np.asarray(pedal_indx_num_valid),np.asarray(pedal_indx_num_valid)+1), axis=0))\n",
    "\n",
    "    segment_sub_train = segment_sub.iloc[indx_num_train]\n",
    "    segment_sub_valid = segment_sub.iloc[indx_num_valid]\n",
    "    segment_sub_train.loc[:,'category'] = 'train'\n",
    "    segment_sub_valid.loc[:,'category'] = 'valid'\n",
    "    segment_sub_tv = segment_sub_train.append(segment_sub_valid)\n",
    "\n",
    "    if len(segment_tv_df) == 0:\n",
    "        segment_tv_df = segment_sub_tv.copy()\n",
    "    else:\n",
    "        segment_tv_df = segment_tv_df.append(segment_sub_tv)\n",
    "\n",
    "segment_tv_df.to_csv(os.path.join(DIR_PEDAL_METADATA, 'pedal-segment_tvdf.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======== excerpts from 20024689 ========\n",
      "    excerpts composers: 84\n",
      "  onset excerpts total: 1134716\n",
      "segment excerpts total: 903398\n",
      "============= sub excerpts =============\n",
      "  onset sub-excerpts/%: 67540/0.06\n",
      "segment sub-excerpts/%: 62424/0.07\n"
     ]
    }
   ],
   "source": [
    "print(\"======== excerpts from 20024689 ========\")\n",
    "print(\"    excerpts composers: {}\".format(len(np.unique(segment_20024689_author_counter.keys()))))\n",
    "print(\"  onset excerpts total: {}\".format(np.sum(onset_20024689_author_counter.values())))\n",
    "print(\"segment excerpts total: {}\".format(np.sum(segment_20024689_author_counter.values())))\n",
    "\n",
    "print(\"============= sub excerpts =============\")\n",
    "print(\"  onset sub-excerpts/%: {}/{:.2f}\".format(onset_20024689_author_subnum_sum,\n",
    "                                                 onset_20024689_author_subnum_sum/np.sum(onset_20024689_author_counter.values())))\n",
    "print(\"segment sub-excerpts/%: {}/{:.2f}\".format(segment_20024689_author_subnum_sum,\n",
    "                                                 segment_20024689_author_subnum_sum/np.sum(segment_20024689_author_counter.values())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make sub datasets\n",
    "\n",
    "Get excerpts within sub synth datasets of year 20024689 from every composers to form the small version dataset. Train/Valid set is split as 0.8/0.2 from every composer-wise excerpts.\n",
    "\n",
    "Make the sub datasets 10 times for checking the conficence interval."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/homes/bl301/tf-env/lib/python2.7/site-packages/pandas/core/indexing.py:537: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  done!\n",
      "1 ...\n",
      "  done!\n",
      "2 ...\n",
      "  done!\n",
      "3 ...\n",
      "  done!\n",
      "4 ...\n",
      "  done!\n",
      "5 ...\n",
      "  done!\n",
      "6 ...\n",
      "  done!\n",
      "7 ...\n",
      "  done!\n",
      "8 ...\n",
      "  done!\n",
      "9 ...\n",
      "  done!\n",
      "0 ...\n",
      "  done!\n",
      "1 ...\n",
      "  done!\n",
      "2 ...\n",
      "  done!\n",
      "3 ...\n",
      "  done!\n",
      "4 ...\n",
      "  done!\n",
      "5 ...\n",
      "  done!\n",
      "6 ...\n",
      "  done!\n",
      "7 ...\n",
      "  done!\n",
      "8 ...\n",
      "  done!\n",
      "9 ...\n",
      "  done!\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "confidence_interval_times = 10\n",
    "\n",
    "onset_sub_df = pd.DataFrame()\n",
    "for interval_time in range(confidence_interval_times):\n",
    "    print(\"{} ...\".format(interval_time))\n",
    "    for authorname, num in onset_20024689_author_subnum.iteritems():\n",
    "        onset_sub = DF_ONSET_20024689.loc[DF_ONSET_20024689['author'] == authorname]\n",
    "        pedal_indx = np.arange(0, len(onset_sub)-1, 2, dtype=int)\n",
    "        pedal_indx_num = random.sample(pedal_indx, int(num/2))\n",
    "        pedal_indx_num_train = random.sample(pedal_indx_num, int(len(pedal_indx_num)*0.8))\n",
    "        pedal_indx_num_valid = [e for e in pedal_indx_num if e not in pedal_indx_num_train]\n",
    "        indx_num_train = np.sort(np.concatenate((np.asarray(pedal_indx_num_train),np.asarray(pedal_indx_num_train)+1), axis=0))\n",
    "        indx_num_valid = np.sort(np.concatenate((np.asarray(pedal_indx_num_valid),np.asarray(pedal_indx_num_valid)+1), axis=0))\n",
    "\n",
    "        onset_sub_train = onset_sub.iloc[indx_num_train]\n",
    "        onset_sub_valid = onset_sub.iloc[indx_num_valid]\n",
    "        onset_sub_train.loc[:,'category'] = 'train'\n",
    "        onset_sub_valid.loc[:,'category'] = 'valid'\n",
    "        onset_sub_tv = onset_sub_train.append(onset_sub_valid)\n",
    "        onset_sub_tv = onset_sub_tv.copy()\n",
    "        onset_sub_tv['interval_time'] = [interval_time] * len(onset_sub_tv) \n",
    "        \n",
    "        if len(onset_sub_df) == 0:\n",
    "            onset_sub_df = onset_sub_tv.copy()\n",
    "        else:\n",
    "            onset_sub_df = onset_sub_df.append(onset_sub_tv)\n",
    "        \n",
    "    print(\"  done!\")\n",
    "\n",
    "onset_sub_df.to_csv(os.path.join(DIR_PEDAL_METADATA, 'pedal-onset_subdf.csv'))\n",
    "\n",
    "segment_sub_df = pd.DataFrame()\n",
    "for interval_time in range(confidence_interval_times):\n",
    "    print(\"{} ...\".format(interval_time))\n",
    "    for authorname, num in segment_20024689_author_subnum.iteritems():\n",
    "        segment_sub = DF_SEGMENT_20024689.loc[DF_SEGMENT_20024689['author'] == authorname]\n",
    "        pedal_indx = np.arange(0, len(segment_sub)-1, 2, dtype=int)\n",
    "        pedal_indx_num = random.sample(pedal_indx, int(num/2))\n",
    "        pedal_indx_num_train = random.sample(pedal_indx_num, int(len(pedal_indx_num)*0.8))\n",
    "        pedal_indx_num_valid = [e for e in pedal_indx_num if e not in pedal_indx_num_train]\n",
    "        indx_num_train = np.sort(np.concatenate((np.asarray(pedal_indx_num_train),np.asarray(pedal_indx_num_train)+1), axis=0))\n",
    "        indx_num_valid = np.sort(np.concatenate((np.asarray(pedal_indx_num_valid),np.asarray(pedal_indx_num_valid)+1), axis=0))\n",
    "\n",
    "        segment_sub_train = segment_sub.iloc[indx_num_train]\n",
    "        segment_sub_valid = segment_sub.iloc[indx_num_valid]\n",
    "        segment_sub_train.loc[:,'category'] = 'train'\n",
    "        segment_sub_valid.loc[:,'category'] = 'valid'\n",
    "        segment_sub_tv = segment_sub_train.append(segment_sub_valid)\n",
    "        segment_sub_tv = segment_sub_tv.copy()\n",
    "        segment_sub_tv['interval_time'] = [interval_time] * len(segment_sub_tv) \n",
    "        \n",
    "        if len(segment_sub_df) == 0:\n",
    "            segment_sub_df = segment_sub_tv.copy()\n",
    "        else:\n",
    "            segment_sub_df = segment_sub_df.append(segment_sub_tv)\n",
    "            \n",
    "    print(\"  done!\")\n",
    "\n",
    "segment_sub_df.to_csv(os.path.join(DIR_PEDAL_METADATA, 'pedal-segment_subdf.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== sub excerpts ===\n",
      "  onset train: 54004\n",
      "  onset valid: 13536\n",
      "segment train: 49908\n",
      "segment valid: 12516\n"
     ]
    }
   ],
   "source": [
    "print(\"=== sub excerpts ===\")\n",
    "print(\"  onset train: {}\".format(len(onset_sub_df.loc[(onset_sub_df['category'] == 'train') & (onset_sub_df['interval_time'] == 0)])))\n",
    "print(\"  onset valid: {}\".format(len(onset_sub_df.loc[(onset_sub_df['category'] == 'valid') & (onset_sub_df['interval_time'] == 0)])))\n",
    "print(\"segment train: {}\".format(len(segment_sub_df.loc[(segment_sub_df['category'] == 'train') & (segment_sub_df['interval_time'] == 0)])))\n",
    "print(\"segment valid: {}\".format(len(segment_sub_df.loc[(segment_sub_df['category'] == 'valid') & (segment_sub_df['interval_time'] == 0)])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MAESTRO Dataset\n",
    "\n",
    "We will use pieces from year 2013457 as part of the real audio dataset. Train/Valid/Test split follows the maestro setting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  maestro composers: 25\n",
      "maestro piece total: 502\n",
      "              train: 404\n",
      "              valid: 48\n",
      "               test: 50\n"
     ]
    }
   ],
   "source": [
    "MAESTRO_DF = pd.read_csv('/import/c4dm-05/bl301/sustain-pedal-detection/maestro-v1.0.0/maestro-v1.0.0.csv')\n",
    "\n",
    "maestro_df = MAESTRO_DF.loc[MAESTRO_DF['year'] > 2011]\n",
    "\n",
    "author = []\n",
    "for composer in maestro_df.canonical_composer.values:\n",
    "    composer_list = composer.split(\" \")\n",
    "    if \"/\" in composer_list:\n",
    "        surname = composer_list[composer_list.index(\"/\")-1]\n",
    "    else:\n",
    "        surname = composer_list[-1]\n",
    "    \n",
    "    if surname == 'Jan\\xc3\\xa1\\xc4\\x8dek':\n",
    "        surname = 'Janacek'\n",
    "    elif surname == 'Rachmaninoff':\n",
    "        surname = 'Rachmaninov'\n",
    "    author.append(surname) \n",
    "maestro_df = maestro_df.copy()\n",
    "maestro_df['author'] = author\n",
    "\n",
    "print(\"  maestro composers: {}\".format(len(np.unique(maestro_df.author.values))))\n",
    "print(\"maestro piece total: {}\".format(len(maestro_df)))\n",
    "print(\"              train: {}\".format(len(maestro_df.loc[MAESTRO_DF['split']=='train'])))\n",
    "print(\"              valid: {}\".format(len(maestro_df.loc[MAESTRO_DF['split']=='validation'])))\n",
    "print(\"               test: {}\".format(len(maestro_df.loc[MAESTRO_DF['split']=='test'])))\n",
    "# maestro_df.to_csv(os.path.join(DIR_PEDAL_METADATA, 'pedal-maestro_2013457df.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We get the pedal onset and offset times from midi data, and save them into `pedal-times_maestro.npz` with other meta data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pedal-times_maestro.npz is saved!\n"
     ]
    }
   ],
   "source": [
    "import pretty_midi\n",
    "filenames, categories, authors= [], [], []\n",
    "pedal_onsets, pedal_offsets = [], []\n",
    "filename_list = maestro_df.audio_filename.values\n",
    "midiname_list = maestro_df.midi_filename.values\n",
    "category_list = maestro_df.split.values\n",
    "author_list = maestro_df.author.values\n",
    "MAESTRO_PATH = '/import/c4dm-05/bl301/sustain-pedal-detection/maestro-v1.0.0'\n",
    "for indx,filename in enumerate(filename_list):  \n",
    "    midi_path = os.path.join(MAESTRO_PATH, midiname_list[indx])\n",
    "\n",
    "    # get ground truth pedal onset time from midi\n",
    "    pm = pretty_midi.PrettyMIDI(midi_path)\n",
    "    pedal_v = []\n",
    "    pedal_t = []\n",
    "    for control_change in pm.instruments[0].control_changes:\n",
    "        if control_change.number == 64:\n",
    "            pedal_v.append(control_change.value)\n",
    "            pedal_t.append(control_change.time)\n",
    "\n",
    "    pedal_onset = []\n",
    "    pedal_offset = []\n",
    "    for i,v in enumerate(pedal_v):\n",
    "        if i>0 and v>=64 and pedal_v[i-1]<64:\n",
    "            pedal_onset.append(pedal_t[i])   \n",
    "        elif i>0 and v<64 and pedal_v[i-1]>=64:\n",
    "            pedal_offset.append(pedal_t[i])\n",
    "\n",
    "    pedal_offset = [t for t in pedal_offset if t > pedal_onset[0]]\n",
    "    seg_idxs = np.min([len(pedal_onset), len(pedal_offset)])\n",
    "    pedal_offset = pedal_offset[:seg_idxs]\n",
    "    pedal_onset = pedal_onset[:seg_idxs]\n",
    "    for seg_idx, offset in enumerate(pedal_offset):\n",
    "        if offset != pedal_offset[-1] and offset > pedal_onset[seg_idx] and offset < pedal_onset[seg_idx+1]:\n",
    "            correct_pedal_data = True\n",
    "        elif offset == pedal_offset[-1] and offset > pedal_onset[seg_idx]:\n",
    "            correct_pedal_data = True\n",
    "        else:\n",
    "            correct_pedal_data = False\n",
    "\n",
    "    if correct_pedal_data:\n",
    "        filenames.append(filename)\n",
    "        pedal_onsets.append(pedal_onset)\n",
    "        pedal_offsets.append(pedal_offset)\n",
    "        categories.append(category_list[indx])\n",
    "        authors.append(author_list[indx])\n",
    "\n",
    "np.savez(os.path.join(DIR_PEDAL_METADATA, 'pedal-times_maestro.npz'), \n",
    "         filename=filenames, pedal_onset=pedal_onsets, pedal_offset=pedal_offsets, \n",
    "         category=categories, author=authors)\n",
    "print('pedal-times_maestro.npz is saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "npz_path = os.path.join(DIR_PEDAL_METADATA, 'pedal-times_maestro.npz')\n",
    "maestro_tracks = np.load(npz_path)\n",
    "\n",
    "maestro_train_indx = np.where(maestro_tracks['category'] == 'train')[0]\n",
    "maestro_valid_indx = np.where(maestro_tracks['category'] == 'validation')[0]\n",
    "maestro_test_indx = np.where(maestro_tracks['category'] == 'test')[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}