Fixed GPU assignment in hyper param search after upstream fetch

Author: lakshaykc

.gitignore

@@ -1,124 +1,130 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# dotenv
-.env
-
-# virtualenv
-.venv
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-# Test datafiles
-out.dat
-wrds_top_2000_3mo_lag.dat
-sample_data_wrds.dat
-output.txt
-
-# Training checkpoints
-training.ckpt*
-checkpoint
-*.pkl
-
-#pycharm
-.idea/
-
-# Error-Analysis temp files
-/error-analysis/test_EA/
-/error-analysis/errorplotting.py
-/error-analysis/**/*.txt
-/error-analysis/**/*.pyc
-/error-analysis/**/*.png
-/error-analysis/**/*.jpg
-/error-analysis/**/*.csv
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Test datafiles
+out.dat
+wrds_top_2000_3mo_lag.dat
+sample_data_wrds.dat
+output.txt
+
+# Training checkpoints
+training.ckpt*
+checkpoint
+*.pkl
+
+#pycharm
+.idea/
+
+# Error-Analysis temp files
+/error-analysis/test_EA/
+/error-analysis/errorplotting.py
+/error-analysis/**/*.txt
+/error-analysis/**/*.pyc
+/error-analysis/**/*.png
+/error-analysis/**/*.jpg
+/error-analysis/**/*.csv
+
+# Experiments
+/exprmnts/
+
+# Datasets
+/datasets/

scripts/hyper_param_search.py

@@ -53,7 +53,7 @@ def get_search_configs():
     configurations.DEFINE_integer("num_threads",4,"NUmber of parallel threads (Number of parallel executions)")
     configurations.DEFINE_integer("num_gpu",1,"Number of GPU on the machine, Use 0 if there are None")
     configurations.DEFINE_integer("sleep_time",1,"Sleep time")
-    configurations.DEFINE_float("mutate_rate",0.2,"Mutation rate for genetic algorithm")
+    configurations.DEFINE_float("mutate_rate",0.02,"Mutation rate for genetic algorithm")
     configurations.DEFINE_string("init_pop",None,"Specify starting population. Path to the pickle file")
 
     c = configurations.ConfigValues()
@@ -157,16 +157,19 @@ def create_train_scripts(pop,gen):
             m = len(pop)//_NUM_THREADS
             pop_idxs = [thread*m + i for i in range(m)]
             for i in pop_idxs:
+                id_seed = int(17*gen + i)
                 # Add GPU number to the members of the generation
                 if _NUM_GPU!=0:
-                    str = "CUDA_VISIBLE_DEVICES=%d"%thread
+                    str = "CUDA_VISIBLE_DEVICES=%d"%(thread%_NUM_GPU)
                 elif _NUM_GPU==0:
                     str = "CUDA_VISIBLE_DEVICES=''"
                 str += " deep_quant.py"
                 str += " --config=config/"+config_filename(gen,i)
+                #str += " --seed=%i"%id_seed
+                str += " --cache_id=%i"%id_seed
                 str += " > " + output_filename(gen,i) 
                 str += " 2> output/stderr-%s.txt"%get_name(gen,i)
-                str += "; rm -rf chkpts/chkpts-%s"%get_name(gen,i)+";"
+                #str += "; rm -rf chkpts/chkpts-%s"%get_name(gen,i)+";"
                 print(str,file=f)
             donefile = donefile_filename(gen,thread)
             print("echo 'done.' > %s"%donefile,file=f)
@@ -337,7 +340,7 @@ def execute_genetic_search(args):
 
     # Read user specified or latest population
     if args.init_pop:
-        pop = pickle.load(open(str(args.init_pop)),"rb")
+        pop = pickle.load(open(str(args.init_pop),"rb"))
     else:
         pop = init_population(config)