Initial Commit

feature_collection.py

@@ -0,0 +1,118 @@
+import numpy as np
+import pandas as pd
+import os
+
+class FeatureCollection():
+	def __init__(self,features_dir,n_segments=15, alpha=0.5):
+		self.n_segments = n_segments
+		self.alpha = alpha
+		self.ld_no_ext = []
+		self.ld_csv = []
+
+		ld_with_ext = os.listdir(features_dir)
+		ld_no_ext = [x[:-4] for x in ld_with_ext if x.endswith('.csv')]
+
+		for x in ld_no_ext:
+			self.ld_csv	.append(os.path.join(features_dir, x + '.csv'))
+			self.ld_no_ext.append(x)
+
+		self.ld_csv = np.array(self.ld_csv)
+		self.ld_no_ext = np.array(self.ld_no_ext)
+
+	def get_gaze_features(self, raw_input):
+		"""
+		Get gaze features from raw input
+		:param raw_input:
+		:return:
+		"""
+
+		gaze_direction = raw_input[:, 5:11]
+		gaze_angle = raw_input[:, 11: 13]
+		eye_landmark2D = raw_input[:, 13: 125]
+		eye_landmark3D = raw_input[:, 125: 293]
+		pose_direction = raw_input[:, 293: 299]
+
+		gaze_direction_std = np.std(gaze_direction, axis=0)
+		gaze_direction_mean = np.mean(gaze_direction, axis=0)
+
+		gaze_angle_std = np.std(gaze_angle, axis=0)
+		gaze_angle_mean = np.mean(gaze_angle, axis=0)
+
+		eye_landmark2D_shape_0 = np.abs(eye_landmark2D[:, 56 + 9: 56 + 14] - eye_landmark2D[:, 56 + 19: 56 + 14: -1])
+		eye_landmark2D_shape_1 = np.abs(eye_landmark2D[:, 56 + 37: 56 + 42] - eye_landmark2D[:, 56 + 47: 56 + 42: -1])
+		eye_landmark2D_shape = np.hstack((eye_landmark2D_shape_0, eye_landmark2D_shape_1))
+		eye_landmark2D_shape_cov = np.divide(np.std(eye_landmark2D_shape, axis=0),
+											 np.mean(eye_landmark2D_shape, axis=0))
+
+		eye_distance = 0.5 * (eye_landmark3D[:, 56 * 2 + 8] + eye_landmark3D[:, 56 * 2 + 42])
+		eye_distance_cov = np.std(eye_distance) / np.mean(eye_distance)
+		eye_distance_ratio = np.min(eye_distance) / np.max(eye_distance)
+		eye_distance_fea = np.array([eye_distance_cov, eye_distance_ratio])
+
+		eye_location2D = []
+		for idx in range(4):
+			cur_mean = np.mean(eye_landmark2D[:, 28 * idx: 28 * (idx + 1)], axis=1)
+			eye_location2D.append(cur_mean)
+
+		eye_location2D = np.vstack(eye_location2D).T
+		eye_location2D_mean = np.mean(eye_location2D, axis=0)
+		eye_location2D_std = np.std(eye_location2D, axis=0)
+
+		eye_location3D = []
+		for idx in range(6):
+			cur_mean = np.mean(eye_landmark3D[:, 28 * idx: 28 * (idx + 1)], axis=1)
+			eye_location3D.append(cur_mean)
+		eye_location3D = np.vstack(eye_location3D).T
+		eye_location3D_mean = np.mean(eye_location3D, axis=0)
+		eye_location3D_std = np.std(eye_location3D, axis=0)
+
+		pose_direction_mean = np.mean(pose_direction, axis=0)
+		pose_direction_std = np.std(pose_direction, axis=0)
+		ret_features = np.hstack((gaze_direction_std, gaze_direction_mean, gaze_angle_mean, gaze_angle_std,
+								  eye_landmark2D_shape_cov, eye_location2D_mean, eye_location2D_std,
+								  eye_location3D_mean,
+								  eye_location3D_std, eye_distance_fea, pose_direction_mean, pose_direction_std))
+
+		return ret_features
+
+	def parse_gaze_features(self, txt_path):
+		try:
+			df = pd.read_csv(txt_path, header=0, sep=',').values
+			seq_length = df.shape[0]
+			indexing = int((self.n_segments - 1) * (1 - self.alpha))
+			k_value = seq_length // (1 + indexing)  # In some case, we will ignore some last frames
+
+			ret = []
+			index_st = 0
+			for idx in range(self.n_segments):
+				index_ed = k_value + int(k_value * (1 - self.alpha) * idx)
+				try:
+					index_features = self.get_gaze_features(df[index_st: index_ed, :])
+				except ValueError:
+					index_features = np.zeros(shape=(60,))
+				ret.append(index_features)
+				index_st = index_ed - int((1 - self.alpha) * k_value)
+
+			ret = np.vstack(ret)
+		except:
+			print('IO error')
+			ret = None
+		return ret
+
+	def get_item(self, idx, ft_type=0):
+
+		if ft_type == 0:
+			txt_name = self.ld_csv[idx]
+			X = self.parse_gaze_features(txt_name)
+		else:
+			raise "Do not support ft_type = {}".format(ft_type)
+		return X
+
+	def get_all_data(self):
+
+		ld_features = []
+		for ix in range(len(self.ld_csv)):
+			z = self.get_item(ix)
+			ld_features.append(z)
+
+		return ld_features

md_config.py

@@ -0,0 +1,10 @@
+import numpy as np
+
+lr_config_001 = np.logspace(-2, -4, base=10, num=6)
+
+md_cfg = []
+of_cfg_001 = {'NAME': 'OF_001', 'model_path':'f841b83091a34af3a086a6d2acb88f19', 'n_segments':15, 'alpha':0.5, 'augment_size':0, 'EPOCHS':10000, 'NUM_LSTM_LAYERS': 2, 'LSTM_UNITS':[128, 128], 'NUM_DENSE_LAYERS':2, 'DENSE_UNITS':[128, 128], 'DROPOUT_RATES':0, 'FC1':[0, 0], 'BATCH_SIZE':32, 'learning_rate':lr_config_001[4]}
+of_cfg_002 = {'NAME': 'OF_002', 'model_path':'d53865dd121f47b4af481c070ae2c62b', 'n_segments':15, 'alpha':0.5, 'augment_size':0, 'EPOCHS':10000, 'NUM_LSTM_LAYERS': 2, 'LSTM_UNITS':[128, 128], 'NUM_DENSE_LAYERS':2, 'DENSE_UNITS':[48, 128], 'DROPOUT_RATES':0, 'FC1':[60, 100], 'BATCH_SIZE':40, 'learning_rate':lr_config_001[5]}
+
+md_cfg.append(of_cfg_001)
+md_cfg.append(of_cfg_002)

predict.py

@@ -0,0 +1,134 @@
+import threading
+import matplotlib.pyplot as plt
+import os, time
+import shutil
+import numpy as np
+import random
+import tensorflow as tf
+import copy
+import time
+
+import md_config as cfg
+from feature_collection import FeatureCollection
+
+
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import CuDNNLSTM, Dense, TimeDistributed, GlobalAveragePooling1D, Activation, Concatenate, \
+	InputLayer, PReLU
+
+
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+seed = 0
+np.random.seed(seed)
+random.seed(0)
+
+
+config = tf.ConfigProto()
+config.gpu_options.per_process_gpu_memory_fraction = 0.2
+session = tf.Session(config=config)
+
+
+tf.set_random_seed(seed)
+
+interval_duration = 10.0
+
+def define_model(hparams, model_name):
+	current_n_lstms = hparams['NUM_LSTM_LAYERS']
+	current_lstm_units = hparams['LSTM_UNITS']
+	current_n_denses = hparams['NUM_DENSE_LAYERS']
+	current_dense_units = hparams['DENSE_UNITS']
+	current_dropout_rates = hparams['DROPOUT_RATES']
+	current_time_step = hparams['TIME_STEP']
+	current_input_units = hparams['INPUT_UNITS']
+	current_densen_act = hparams['ACTIVATION_F']
+
+	model = Sequential()
+	if hparams['FC1'][1] > 0:
+		model.add(TimeDistributed(Dense(hparams['FC1'][1], activation='relu'),
+								  input_shape=(current_time_step, hparams['FC1'][0])))
+
+	model.add(
+		CuDNNLSTM(current_lstm_units[0], return_sequences=True, input_shape=(current_time_step, current_input_units),
+				  stateful=False))
+
+	if current_n_lstms > 1:
+		for idx in range(1, current_n_lstms):
+			model.add(CuDNNLSTM(current_lstm_units[idx], return_sequences=True))
+
+	for idx in range(current_n_denses):
+		model.add(TimeDistributed(Dense(current_dense_units[idx], activation='relu')))
+
+	model.add(TimeDistributed(Dense(1, activation=current_densen_act)))
+	model.add(GlobalAveragePooling1D())
+
+	return model
+
+def get_model(model_index, n_segments=15, input_units=60):
+    """
+    Make prediction for data_npy
+    :param data_npy:
+    :return:
+    """
+    ld_cfg = cfg.md_cfg
+    hparams = copy.deepcopy(ld_cfg[model_index])
+    ft_type = 'of'
+
+
+    hparams['TIME_STEP'] = n_segments
+    hparams['INPUT_UNITS'] = hparams['FC1'][1] if hparams['FC1'][1] > 0 else input_units
+    hparams['optimizer'] = 'adam'
+    hparams['ACTIVATION_F'] = 'tanh'
+    hparams['CLSW'] = 1
+
+    cur_model = define_model(hparams,hparams['NAME'])
+    cur_model.build()
+    cur_model.load_weights(
+            './models/{}_{}_models_{}_{}_0_epochs{}_best_weight.h5'.format(hparams['model_path'], ft_type,
+                                                                           hparams['n_segments'], hparams['alpha'],
+                                                                           hparams['EPOCHS']))
+
+    return cur_model
+
+def periodic_function():
+	if os.path.isdir("../../OpenFace/build/processed"):
+		eye_gaze_v1 = get_model(model_index=0)
+		eye_gaze_v2 = get_model(model_index=1)
+		feature_extraction = FeatureCollection('../../OpenFace/build/processed')
+		ft = np.array(feature_extraction.get_all_data())
+		v1 = eye_gaze_v1.predict(ft[0].reshape(1,15,60))
+		v2 = eye_gaze_v2.predict(ft[0].reshape(1,15,60))
+		print('{} {}'.format(v1,v2))
+		enga_score = 0.5 * (v1 + v2)
+		print(enga_score)
+		duration = time.strftime("%M:%S", time.gmtime(int(time.time() - start_time)))
+		x.append(duration)
+		if enga_score<0.4:
+			y.append(0)
+		elif enga_score<0.6:
+			y.append(1)
+		elif enga_score<0.83:
+			y.append(2)
+		else:
+			y.append(3)
+		print(x)
+		print(y)
+		shutil.rmtree('../../OpenFace/build/processed', ignore_errors=True)
+
+def startTimer():
+	threading.Timer(interval_duration,startTimer).start()
+	periodic_function()
+def main():
+	startTimer()
+	while True:
+		plt.yticks(np.arange(4),('Disengaged','Barely Engaged','Engaged','Highly Engaged'))
+		plt.xticks(rotation=90)
+		plt.step(x, y,'b')
+		# plt.xlabel('Duration')
+		# plt.ylabel('Engagement Level')
+		plt.pause(1)
+if __name__ == '__main__':
+	x = []
+	y = []
+	start_time = time.time()
+	main()