merging some updated code from my fork to a new branch

.gitignore

@@ -4,3 +4,5 @@
 *.pyc
 
 *.pyc
+
+*.DS_Store

eventseg_collector.py

@@ -0,0 +1,3 @@
+#!/usr/bin/python
+
+from eventseg_config import config

config.py → eventseg_config.py

@@ -7,15 +7,16 @@
 # job creation options
 
 #add additional checks for your local machine here...
-if (socket.gethostname() == 'vertex') or (socket.gethostname() == 'vertex.kiewit.dartmouth.edu') or (socket.gethostname() == 'vertex.local'):
-    config['datadir'] = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'testing')
+# ******** check kiewit hostname from eduroam ********
+if (socket.gethostname() == 'Paxtons-MacBook-Pro') or (socket.gethostname() == 'Paxtons-MacBook-Pro.kiewit.dartmouth.edu') or (socket.gethostname() == 'Paxtons-MacBook-Pro.local'):
+    config['datadir'] = '/Users/paxtonfitzpatrick/Documents/Dartmouth/Thesis/memory-dynamics/data/models/participants/trajectories')
     config['workingdir'] = config['datadir']
     config['startdir'] = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))  # directory to start the job in
     config['template'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'run_job_local.sh')
 else:
-    config['datadir'] = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'testing')
-    config['workingdir'] = config['datadir']
-    config['startdir'] = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    config['datadir'] = os.path.join('/dartfs/rc/lab/D/DBIC/CDL/pfitzpatrick/eventseg', 'trajectories')
+    config['workingdir'] = '/dartfs/rc/lab/D/DBIC/CDL/pfitzpatrick/eventseg/'
+    config['startdir'] = '/dartfs/rc/lab/D/DBIC/CDL/pfitzpatrick'
     config['template'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'run_job_cluster.sh')
 
 config['scriptdir'] = os.path.join(config['workingdir'], 'scripts')
@@ -24,8 +25,8 @@
 
 
 # runtime options
-config['jobname'] = "superEEG"  # default job name
-config['q'] = "default"  # options: default, test, largeq
+config['jobname'] = "eventseg"  # default job name
+config['q'] = "largeq"  # options: default, test, largeq
 config['nnodes'] = 1  # how many nodes to use for this one job
 config['ppn'] = 4  # how many processors to use for this one job (assume 4GB of RAM per processor)
 config['walltime'] = '20:00:00'  # maximum runtime, in h:MM:SS

eventseg_cruncher.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+import sys
+import os
+import pickle
+import numpy as np
+import brainiak.eventseg.event as event
+from eventseg_config import config
+
+filepath, k = sys.argv[1], sys.argv[2]
+dir, f_name  = os.path.split(filepath)
+rectype = os.path.split(dir)[1]
+trajectory = np.load(filepath)
+savepath = os.path.join(config['resultsdir'], rectype, os.path.splitext(f_name)[0], 'k'+k+'.npy')
+
+if not os.path.isfile(savepath):
+    ev = event.EventSegment(k)
+    ev.fit(trajectory)
+
+    np.save(savepath, ev.segments_[0])

create_and_submit_jobs.py → eventseg_submit.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 # create a bunch of job scripts
-from config import config
+from eventseg_config import config
 from subprocess import call
 import os
 import socket
@@ -10,12 +10,38 @@
 
 
 # ====== MODIFY ONLY THE CODE BETWEEN THESE LINES ======
+import sys
+
+n_ks = sys.argv[1]
+
 # each job command should be formatted as a string
-job_script = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test.py')
-job_commands = map(lambda x: x[0]+" "+str(x[1]), zip([job_script]*10, range(10)))
+job_script = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'eventseg_cruncher.py')
+
+job_commmands = list()
+job_names = list()
+
+for root, dirs, files in os.walk(config['datadir']):
+    for file in [f for f in files if f.startswith('debug')]:
+        filepath = os.path.join(root,file)
+        rectype = os.path.split(root)[-1]
+        turkid = os.path.splitext(file)[0]
+
+        subjdir = os.path.join(config['resultsdir'], rectype, turkid)
+        if not os.path.isdir(subjdir):
+            os.makedirs(subjdir, exist_ok=True)
+
+        for k in range(2,int(n_ks)+1):
+            if not os.path.isfile(os.path.join(subjdir,'k'+str(k)+'.npy')):
+                job_commands.append(' '.join([job_script, filepath, k]))
+                job_names.append('segment_' + turkid + '_' + rectype + '_k' + str(k) + '.sh')
+
+
+
+
+## job_commands = map(lambda x: x[0]+" "+str(x[1]), zip([job_script]*10, range(10)))
 
 # job_names should specify the file name of each script (as a list, of the same length as job_commands)
-job_names = map(lambda x: str(x)+'.sh', range(len(job_commands)))
+## job_names = map(lambda x: str(x)+'.sh', range(len(job_commands)))
 # ====== MODIFY ONLY THE CODE BETWEEN THESE LINES ======
 
 assert(len(job_commands) == len(job_names))
@@ -114,7 +140,7 @@ def release(lockfile):
             next_job = create_job(n, c)
 
             if (socket.gethostname() == 'discovery') or (socket.gethostname() == 'ndoli'):
-                submit_command = 'echo "[SUBMITTING JOB: ' + next_job + ']"; qsub'
+                submit_command = 'echo "[SUBMITTING JOB: ' + next_job + ']"; mksub'
             else:
                 submit_command = 'echo "[RUNNING JOB: ' + next_job + ']"; sh'
 

run_job_cluster.sh

@@ -20,19 +20,13 @@
 cd <config['startdir']>
 
 echo ----
-echo LOADING PYTHON 2.7.11
-module load python/2.7.11
 
 echo ACTIVATING BRAIN DYNAMICS MODEL VIRTUAL ENVIRONMENT
-source activate hobd
+source activate memdyn
 
-echo UNLOADING PYTHON 2.7.11
-module unload python/2.7.11
-
-echo PYTHON PATH CONFIGURATION COMPLETE
 echo ----
 
 # run the job
 <config['cmd_wrapper']> <job_command> #note: job_command is reserved for the job command; it should not be specified in config.py
 
-source deactivate hobd
+source deactivate memdyn

run_job_local.sh

@@ -5,5 +5,15 @@
 # set the working directory *of the job* to the specified start directory
 cd <config['startdir']>
 
+echo ----
+
+echo ACTIVATING MEMORY DYNAMICS VIRTUAL ENVIRONMENT
+source activate py36
+
+echo ----
+
+
 # run the job
 <config['cmd_wrapper']> <job_command> #note: job_command is reserved for the job command; it should not be specified in config.py
+
+source deactivate py36