minor updates everywhere

Author: smerdov

Analysis/GeneralDataProcessing.py

@@ -22,10 +22,10 @@
 
 eye_tracker_shift_hours = 4
 
-def concat_files4data_source(filenames, path, index_col):
+def concat_files4data_source(filenames, path, index_col, sep=','):
     df_result = pd.DataFrame()
     for filename in filenames:
-        df_addition = pd.read_csv(path + filename)
+        df_addition = pd.read_csv(path + filename, sep=sep)
         df_addition.set_index(index_col, inplace=True)
         df_result = pd.concat([df_result, df_addition])
 
@@ -34,13 +34,14 @@ def concat_files4data_source(filenames, path, index_col):
 data_dict = {}
 
 data_sources = [
-    'arduino_0',
-    'arduino_1',
-    'arduino_2',
+    # 'arduino_0',
+    # 'arduino_1',
+    # 'arduino_2',
     'polar_heart_rate',
-    'face_temperature',
+    # 'face_temperature',
     'eye_tracker',
-    'input',
+    'mouse',
+    'keyboard',
     'EEG',
 ]
 
@@ -58,6 +59,7 @@ def concat_files4data_source(filenames, path, index_col):
 
     # for arduino_id in [0, 1, 2]:
     for data_source in data_sources:
+        sep = ','
         # df4arduino = pd.DataFrame()
         # filename_prefix = f'arduino_{arduino_id}'
         # data_source = f'arduino_{arduino_id}'
@@ -73,18 +75,42 @@ def concat_files4data_source(filenames, path, index_col):
         path2data_source = data_path4player + data_source + '/'
         filenames = sorted(os.listdir(path2data_source))
         filenames_filtered = [filename for filename in filenames if not (filename.startswith('game') or filename.startswith('.'))]
-        df_data_source = concat_files4data_source(filenames_filtered, path2data_source, index_col='Timestamp')
+        if data_source == 'eye_tracker':
+            filenames_filtered = [filename for filename in filenames_filtered if filename.startswith('tobii_pro_gaze')]
+            index_col = '#timestamp'
+        elif data_source in ['mouse', 'keyboard', 'EEG']:
+            index_col = 'time'
+            sep = ';'
+        else:
+            index_col = 'Timestamp'
+
+        # pd.read_csv(path2data_source + filenames_filtered[0], sep=';')
+        df_data_source = concat_files4data_source(filenames_filtered, path2data_source, index_col=index_col, sep=sep)
+        df_data_source.index.name = 'Timestamp'
+
+        if data_source == 'eye_tracker':
+            # break
+            df_data_source.index = pd.to_datetime(df_data_source.index + 3600 * 4 * 1000, unit='ms')
+            df_data_source.index = df_data_source.index.astype(str)
+            df_data_source.index = [index.replace(' ', '-') for index in df_data_source.index]
+            # df_data_source.index = df_data_source.index.apply(lambda x: x.timestamp())
+
+        # df_data_source.rename(columns={'#timestamp': 'Timestamp'}, inplace=True)
+        # df_data_source.set_index(['Timestamp'], inplace=True)
         # df_data_source.index = pd.to_datetime(df_data_source.index)
         # df_data_source.index = df_data_source.index.apply(lambda x: x.timestamp())
         df_data_source.sort_index(inplace=True)
 
+
         for n_row, row in games_start_end_times.iterrows():
             game_num = row['game_num']
             game_start_time = row['game_start']
             game_end_time = row['game_end']
             path4game_folder = data_path4player + data_source + '/' + f'game_{game_num}'
             if not os.path.exists(path4game_folder):
                 os.mkdir(path4game_folder)
+
+            # break
             mask4game = (game_start_time < df_data_source.index) & (df_data_source.index < game_end_time)
             df_data_source.loc[mask4game].to_csv(data_path4player + data_source + '/' + f'game_{game_num}/' + data_source + '.csv')
 
@@ -97,14 +123,14 @@ def concat_files4data_source(filenames, path, index_col):
         # player_data_dict[data_source] = df_data_source
 
     # ### Eye tracker
-    # filenames_with_prefix = [filename for filename in filenames if filename.startswith('tobii')]
+    # # filenames_with_prefix = [filename for filename in filenames if filename.startswith('tobii')]
     # df_eyetracker = concat_files4data_source(filenames_with_prefix, index_col='#timestamp')
     # df_eyetracker.index = pd.to_datetime(df_eyetracker.index + 3600 * 4 * 1000, unit='ms')
-    # # df_eyetracker.index = df_eyetracker.index.apply(lambda x: x.timestamp())
+    # df_eyetracker.index = df_eyetracker.index.apply(lambda x: x.timestamp())
     # df_eyetracker.sort_index(inplace=True)
     # df_eyetracker = df_eyetracker.resample('1s').mean()
     # player_data_dict['eye_tracker'] = df_eyetracker
-
+    #
     # data_dict[player_id] = player_data_dict
 
 

EEG/cortex-v2-example/python/example.py

@@ -31,6 +31,9 @@ async def do_stuff(cortex):
         # session_ids = []
         # session_id2player_id = {}
         for headset_id in cortex.headsets:
+            if not headset_id.startswith('INSIGHT'):
+                continue
+
             await cortex.create_session(activate=True, headset_id=headset_id)
             # session_ids.append(new_session_id)
 

InputLogger/InputLogger.py

@@ -7,6 +7,7 @@
 
 TIME_FORMAT4FILES = '%Y-%m-%d-%H-%M-%S'
 TIME_FORMAT = '%Y-%m-%d-%H:%M:%S.%f'
+sep = ';'
 
 queue_mouse = Queue(maxsize=10000)
 queue_keyboard = Queue(maxsize=10000)
@@ -18,8 +19,8 @@
 
 filename_mouse = f'data/mouse_{current_datetime4files}.csv'
 filename_keyboard = f'data/keyboard_{current_datetime4files}.csv'
-header_mouse = 'time,event,x,y,params'
-header_keyboard = 'time,event,button'
+header_mouse = f'time{sep}event{sep}x{sep}y{sep}params'
+header_keyboard = f'time{sep}event{sep}button'
 
 def get_current_time():
     return datetime.now().strftime(TIME_FORMAT)
@@ -35,7 +36,7 @@ def mouse_on_move(x, y):
     params = {}
     params_str = json.dumps(params)
 
-    msg = f'{current_time},{event},{x},{y},{params_str}'
+    msg = f'{current_time}{sep}{event}{sep}{x}{sep}{y}{sep}{params_str}'
     queue_mouse.put(msg)
 
 
@@ -53,7 +54,7 @@ def mouse_on_click(x, y, button, pressed):
     params['button'] = button.name
     params_str = json.dumps(params)
 
-    msg = f'{current_time},{event},{x},{y},{params_str}'
+    msg = f'{current_time}{sep}{event}{sep}{x}{sep}{y}{sep}{params_str}'
     queue_mouse.put(msg)
 
 def mouse_on_scroll(x, y, dx, dy):
@@ -67,7 +68,7 @@ def mouse_on_scroll(x, y, dx, dy):
     params['dy'] = dy
     params_str = json.dumps(params)
 
-    msg = f'{current_time},{event},{x},{y},{params_str}'
+    msg = f'{current_time}{sep}{event}{sep}{x}{sep}{y}{sep}{params_str}'
     queue_mouse.put(msg)
 
 
@@ -79,10 +80,10 @@ def keyboard_on_press(key):
     else:
         button_name = key.name
 
-    if button_name == ',':  # We won't lose much but simplify the format
+    if button_name == sep:  # We won't lose much but simplify the format
         return
 
-    msg = f'{current_time},{event},{button_name}'
+    msg = f'{current_time}{sep}{event}{sep}{button_name}'
     queue_keyboard.put(msg)
 
 
@@ -94,10 +95,10 @@ def keyboard_on_release(key):
     else:
         button_name = key.name
 
-    if button_name == ',':  # We won't lose much but simplify the format
+    if button_name == sep:  # We won't lose much but simplify the format
         return
 
-    msg = f'{current_time},{event},{button_name}'
+    msg = f'{current_time}{sep}{event}{sep}{button_name}'
     queue_keyboard.put(msg)
 
 

PolarDataCollection/file_parsing.py

@@ -3,7 +3,8 @@
 import os
 import json
 
-TIME_FORMAT = "%Y-%m-%d-%H-%M-%S"
+TIME_FORMAT4FILES = "%Y-%m-%d-%H-%M-%S"
+TIME_FORMAT = "%Y-%m-%d-%H:%M:%S"
 TIME_FORMAT_FIT = "%Y-%m-%dT%H:%M:%SZ"
 players_list = json.load(open('players_credentials.json'))
 
@@ -34,14 +35,16 @@
 
         df2add = pd.DataFrame(data2add)[['hr']]
         df2add.index = pd.to_datetime(start_time) + df2add.index
+        df2add.index = [index_value.strftime(TIME_FORMAT) for index_value in df2add.index]
         df2add.rename(columns={'hr': 'heart_rate'}, inplace=True)
         df2add.index.name = 'Timestamp'
 
         df4player = pd.concat([df4player, df2add])
 
     if len(df4player):
         df4player = df4player.sort_index()
-        suffix = df4player.index[0].strftime(TIME_FORMAT)
+        # suffix = df4player.index[0].strftime(TIME_FORMAT4FILES)
+        suffix = df4player.index[0].replace(':', '-')
 
         df4player.to_csv(f'output/player_{player_id}/heart_rate_{suffix}.csv')
     else: