tune_hyperparameters.py

#!/usr/bin/env python
##########################################################
# pytorch-kaldi v.0.1                                      
# Mirco Ravanelli, Titouan Parcollet
# Mila, University of Montreal
# October 2018
#
# Description:
# This scripts generates config files with the random hyperparamters specified by the user.
# python tune_hyperparameters.py cfg_file out_folder N_exp hyperparameters_spec
# e.g., python tune_hyperparameters.py cfg/TIMIT_MLP_mfcc.cfg exp/TIMIT_MLP_mfcc_tuning 10 arch_lr=randfloat(0.001,0.01) batch_size_train=randint(32,256) dnn_act=choose_str{relu,relu,relu,relu,softmax|tanh,tanh,tanh,tanh,softmax}
##########################################################


import random
import re
import os
import sys
from random import randint

if __name__ == '__main__':
    cfg_file = sys.argv[1]
    output_folder = sys.argv[2]
    N_exp = int(sys.argv[3])
    hyperparam_list = sys.argv[4:]
    seed = 1234

    print('Generating config file for hyperparameter tuning...')

    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    random.seed(seed)

    for i in range(N_exp):

        cfg_file_out = output_folder + '/exp' + str(i) + '.cfg'

        with open(cfg_file_out, 'wt') as cfg_out, \
             open(cfg_file, 'rt') as cfg_in:
            for line in cfg_in:

                key = line.split('=')[0]

                if key == 'out_folder':
                    line = 'out_folder=' + output_folder + '/exp' + str(i) + '\n'

                hyper_found = False
                for hyperparam in hyperparam_list:

                    key_hyper = hyperparam.split('=')[0]

                    if key == key_hyper:

                        if "randint" in hyperparam:
                            lower, higher = re.search('randint\((.+?)\)',
                                                      hyperparam).group(1).split(
                                ',')
                            value_hyper = randint(int(lower), int(higher))
                            hyper_found = True

                        if "randfloat" in hyperparam:
                            lower, higher = re.search('randfloat\((.+?)\)',
                                                      hyperparam).group(1).split(
                                ',')
                            value_hyper = random.uniform(float(lower),
                                                         float(higher))
                            hyper_found = True

                        if "choose_str" in hyperparam:
                            value_hyper = random.choice(
                                re.search('\{(.+?)\}', hyperparam).group(1).split(
                                    '|'))
                            hyper_found = True

                        if "choose_int" in hyperparam:
                            value_hyper = int(random.choice(
                                re.search('\{(.+?)\}', hyperparam).group(1).split(
                                    '|')))
                            hyper_found = True

                        if "choose_float" in hyperparam:
                            value_hyper = float(random.choice(
                                re.search('\{(.+?)\}', hyperparam).group(1).split(
                                    '|')))
                            hyper_found = True

                        line_out = key + '=' + str(value_hyper) + '\n'

                if not hyper_found:
                    line_out = line

                cfg_out.write(line_out)

            print('Done %s' % cfg_file_out)