lip reading

ndkamath · Jul 16, 2017 · 8bcf0c2 · 8bcf0c2
1 parent c33ea03
commit 8bcf0c2
Show file tree

Hide file tree

Showing 142 changed files with 187 additions and 3 deletions.
diff --git a/README.rst b/README.rst
@@ -60,6 +60,30 @@ Table of Contents
   :local:
   :depth: 3
 
+
+-----
+DEMO
+-----
+
+~~~~~~~~~~~~~~~~~~~~~~~~
+Training/Evaluation DEMO
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+|training|
+
+.. |training| image:: readme_images/liptrackingdemo.png
+    :target: https://asciinema.org/a/kXIDzZt1UzRioL1gDPzOy9VkZ
+
+~~~~~~~~~~~~~
+Lip Tracking
+~~~~~~~~~~~~~
+
+|liptrackingdemo|
+
+.. |Asciinema| image:: readme_images/liptrackingdemo.png
+    :target: https://asciinema.org/a/RiZtscEJscrjLUIhZKkoG3GVm
+.. https://asciinema.org/a/m1r1OaoUXsEECNZKzpkfAXg7y
+
 --------------
 General View
 --------------

diff --git a/activation b/activation
@@ -0,0 +1 @@
+results/activation
diff --git a/code/training_evaluation/auxiliary/__init__.pyc b/code/training_evaluation/auxiliary/__init__.pyc
diff --git a/code/training_evaluation/auxiliary/losses.pyc b/code/training_evaluation/auxiliary/losses.pyc
diff --git a/code/training_evaluation/nets/__init__.pyc b/code/training_evaluation/nets/__init__.pyc
diff --git a/code/training_evaluation/nets/lipread_mouth.pyc b/code/training_evaluation/nets/lipread_mouth.pyc
diff --git a/code/training_evaluation/nets/lipread_speech.pyc b/code/training_evaluation/nets/lipread_speech.pyc
diff --git a/code/training_evaluation/nets/nets_factory.pyc b/code/training_evaluation/nets/nets_factory.pyc
diff --git a/code/training_evaluation/roc_curve/__init__.pyc b/code/training_evaluation/roc_curve/__init__.pyc
diff --git a/code/training_evaluation/roc_curve/calculate_roc.pyc b/code/training_evaluation/roc_curve/calculate_roc.pyc
diff --git a/dlib b/dlib
@@ -0,0 +1 @@
+data/
diff --git a/lip_tracking_demo.sh b/lip_tracking_demo.sh
@@ -1,7 +1,10 @@
+# Sample input
+input_filename="data/sample_video.mp4"
+
 # visualizing (using pretrained model)
+mkdir results
 ln -s data/ dlib
-mkdir results/action && ln -s results/activation ./activation
-python -u ./code/lip_tracking/VisualizeLip.py --input $input_filename --output ../results/output_video.mp4
+python -u ./code/lip_tracking/VisualizeLip.py --input $input_filename --output results/output_video.mp4
 
 # create gif from mouth frames
-ffmpeg -i ./mouth/frame_%*.png ../results/mouth.gif
+ffmpeg -i ./results/mouth/frame_%*.png results/mouth.gif
diff --git a/readme_images/liptrackingdemo.png b/readme_images/liptrackingdemo.png
diff --git a/results/TRAIN_CNN_3D/checkpoint b/results/TRAIN_CNN_3D/checkpoint
@@ -0,0 +1,2 @@
+model_checkpoint_path: "train_logs-62"
+all_model_checkpoint_paths: "train_logs-62"
diff --git a/results/TRAIN_CNN_3D/test_logs/events.out.tfevents.1500232350.sina-Alienware-Area-51-R2 b/results/TRAIN_CNN_3D/test_logs/events.out.tfevents.1500232350.sina-Alienware-Area-51-R2
diff --git a/results/TRAIN_CNN_3D/train_logs-62.data-00000-of-00001 b/results/TRAIN_CNN_3D/train_logs-62.data-00000-of-00001
diff --git a/results/TRAIN_CNN_3D/train_logs-62.index b/results/TRAIN_CNN_3D/train_logs-62.index
diff --git a/results/TRAIN_CNN_3D/train_logs-62.meta b/results/TRAIN_CNN_3D/train_logs-62.meta
diff --git a/results/TRAIN_CNN_3D/train_logs/events.out.tfevents.1500232331.sina-Alienware-Area-51-R2 b/results/TRAIN_CNN_3D/train_logs/events.out.tfevents.1500232331.sina-Alienware-Area-51-R2
diff --git a/results/activation b/results/activation
@@ -0,0 +1,153 @@
+(lp0
+I0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI0
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI0
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI0
+aI0
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+aI1
+a.
diff --git a/results/mouth.gif b/results/mouth.gif
diff --git a/results/mouth/frame_100.png b/results/mouth/frame_100.png
diff --git a/results/mouth/frame_101.png b/results/mouth/frame_101.png
diff --git a/results/mouth/frame_102.png b/results/mouth/frame_102.png
diff --git a/results/mouth/frame_103.png b/results/mouth/frame_103.png
diff --git a/results/mouth/frame_104.png b/results/mouth/frame_104.png
diff --git a/results/mouth/frame_105.png b/results/mouth/frame_105.png
diff --git a/results/mouth/frame_106.png b/results/mouth/frame_106.png
diff --git a/results/mouth/frame_107.png b/results/mouth/frame_107.png
diff --git a/results/mouth/frame_108.png b/results/mouth/frame_108.png
diff --git a/results/mouth/frame_109.png b/results/mouth/frame_109.png
diff --git a/results/mouth/frame_110.png b/results/mouth/frame_110.png
diff --git a/results/mouth/frame_111.png b/results/mouth/frame_111.png
diff --git a/results/mouth/frame_112.png b/results/mouth/frame_112.png
diff --git a/results/mouth/frame_113.png b/results/mouth/frame_113.png
diff --git a/results/mouth/frame_114.png b/results/mouth/frame_114.png
diff --git a/results/mouth/frame_115.png b/results/mouth/frame_115.png
diff --git a/results/mouth/frame_116.png b/results/mouth/frame_116.png
diff --git a/results/mouth/frame_117.png b/results/mouth/frame_117.png
diff --git a/results/mouth/frame_118.png b/results/mouth/frame_118.png
diff --git a/results/mouth/frame_119.png b/results/mouth/frame_119.png
diff --git a/results/mouth/frame_120.png b/results/mouth/frame_120.png
diff --git a/results/mouth/frame_121.png b/results/mouth/frame_121.png
diff --git a/results/mouth/frame_122.png b/results/mouth/frame_122.png
diff --git a/results/mouth/frame_123.png b/results/mouth/frame_123.png
diff --git a/results/mouth/frame_124.png b/results/mouth/frame_124.png
diff --git a/results/mouth/frame_125.png b/results/mouth/frame_125.png
diff --git a/results/mouth/frame_126.png b/results/mouth/frame_126.png
diff --git a/results/mouth/frame_127.png b/results/mouth/frame_127.png
diff --git a/results/mouth/frame_128.png b/results/mouth/frame_128.png
diff --git a/results/mouth/frame_129.png b/results/mouth/frame_129.png
diff --git a/results/mouth/frame_130.png b/results/mouth/frame_130.png
diff --git a/results/mouth/frame_131.png b/results/mouth/frame_131.png
diff --git a/results/mouth/frame_132.png b/results/mouth/frame_132.png
diff --git a/results/mouth/frame_133.png b/results/mouth/frame_133.png
diff --git a/results/mouth/frame_134.png b/results/mouth/frame_134.png
diff --git a/results/mouth/frame_136.png b/results/mouth/frame_136.png
diff --git a/results/mouth/frame_137.png b/results/mouth/frame_137.png
diff --git a/results/mouth/frame_138.png b/results/mouth/frame_138.png
diff --git a/results/mouth/frame_139.png b/results/mouth/frame_139.png
diff --git a/results/mouth/frame_140.png b/results/mouth/frame_140.png
diff --git a/results/mouth/frame_141.png b/results/mouth/frame_141.png
diff --git a/results/mouth/frame_144.png b/results/mouth/frame_144.png
diff --git a/results/mouth/frame_145.png b/results/mouth/frame_145.png
diff --git a/results/mouth/frame_146.png b/results/mouth/frame_146.png
diff --git a/results/mouth/frame_147.png b/results/mouth/frame_147.png
diff --git a/results/mouth/frame_148.png b/results/mouth/frame_148.png
diff --git a/results/mouth/frame_149.png b/results/mouth/frame_149.png
diff --git a/results/mouth/frame_150.png b/results/mouth/frame_150.png
diff --git a/results/mouth/frame_28.png b/results/mouth/frame_28.png
diff --git a/results/mouth/frame_29.png b/results/mouth/frame_29.png
diff --git a/results/mouth/frame_30.png b/results/mouth/frame_30.png
diff --git a/results/mouth/frame_31.png b/results/mouth/frame_31.png
diff --git a/results/mouth/frame_32.png b/results/mouth/frame_32.png
diff --git a/results/mouth/frame_33.png b/results/mouth/frame_33.png
diff --git a/results/mouth/frame_34.png b/results/mouth/frame_34.png
diff --git a/results/mouth/frame_35.png b/results/mouth/frame_35.png
diff --git a/results/mouth/frame_36.png b/results/mouth/frame_36.png
diff --git a/results/mouth/frame_37.png b/results/mouth/frame_37.png
diff --git a/results/mouth/frame_38.png b/results/mouth/frame_38.png
diff --git a/results/mouth/frame_39.png b/results/mouth/frame_39.png
diff --git a/results/mouth/frame_40.png b/results/mouth/frame_40.png
diff --git a/results/mouth/frame_41.png b/results/mouth/frame_41.png
diff --git a/results/mouth/frame_42.png b/results/mouth/frame_42.png
diff --git a/results/mouth/frame_43.png b/results/mouth/frame_43.png
diff --git a/results/mouth/frame_44.png b/results/mouth/frame_44.png
diff --git a/results/mouth/frame_45.png b/results/mouth/frame_45.png
diff --git a/results/mouth/frame_46.png b/results/mouth/frame_46.png
diff --git a/results/mouth/frame_47.png b/results/mouth/frame_47.png
diff --git a/results/mouth/frame_48.png b/results/mouth/frame_48.png
diff --git a/results/mouth/frame_49.png b/results/mouth/frame_49.png
diff --git a/results/mouth/frame_50.png b/results/mouth/frame_50.png
diff --git a/results/mouth/frame_51.png b/results/mouth/frame_51.png
diff --git a/results/mouth/frame_52.png b/results/mouth/frame_52.png
diff --git a/results/mouth/frame_53.png b/results/mouth/frame_53.png
diff --git a/results/mouth/frame_54.png b/results/mouth/frame_54.png
diff --git a/results/mouth/frame_55.png b/results/mouth/frame_55.png
diff --git a/results/mouth/frame_56.png b/results/mouth/frame_56.png
diff --git a/results/mouth/frame_57.png b/results/mouth/frame_57.png
diff --git a/results/mouth/frame_58.png b/results/mouth/frame_58.png
diff --git a/results/mouth/frame_59.png b/results/mouth/frame_59.png
diff --git a/results/mouth/frame_60.png b/results/mouth/frame_60.png
diff --git a/results/mouth/frame_61.png b/results/mouth/frame_61.png
diff --git a/results/mouth/frame_62.png b/results/mouth/frame_62.png
diff --git a/results/mouth/frame_63.png b/results/mouth/frame_63.png
diff --git a/results/mouth/frame_64.png b/results/mouth/frame_64.png
diff --git a/results/mouth/frame_65.png b/results/mouth/frame_65.png
diff --git a/results/mouth/frame_66.png b/results/mouth/frame_66.png
diff --git a/results/mouth/frame_67.png b/results/mouth/frame_67.png
diff --git a/results/mouth/frame_68.png b/results/mouth/frame_68.png
diff --git a/results/mouth/frame_69.png b/results/mouth/frame_69.png
diff --git a/results/mouth/frame_70.png b/results/mouth/frame_70.png
diff --git a/results/mouth/frame_71.png b/results/mouth/frame_71.png
diff --git a/results/mouth/frame_72.png b/results/mouth/frame_72.png
diff --git a/results/mouth/frame_73.png b/results/mouth/frame_73.png
diff --git a/results/mouth/frame_74.png b/results/mouth/frame_74.png
diff --git a/results/mouth/frame_75.png b/results/mouth/frame_75.png
diff --git a/results/mouth/frame_76.png b/results/mouth/frame_76.png
diff --git a/results/mouth/frame_77.png b/results/mouth/frame_77.png
diff --git a/results/mouth/frame_78.png b/results/mouth/frame_78.png
diff --git a/results/mouth/frame_79.png b/results/mouth/frame_79.png
diff --git a/results/mouth/frame_80.png b/results/mouth/frame_80.png
diff --git a/results/mouth/frame_81.png b/results/mouth/frame_81.png
diff --git a/results/mouth/frame_82.png b/results/mouth/frame_82.png
diff --git a/results/mouth/frame_83.png b/results/mouth/frame_83.png
diff --git a/results/mouth/frame_84.png b/results/mouth/frame_84.png
diff --git a/results/mouth/frame_85.png b/results/mouth/frame_85.png
diff --git a/results/mouth/frame_86.png b/results/mouth/frame_86.png
diff --git a/results/mouth/frame_87.png b/results/mouth/frame_87.png
diff --git a/results/mouth/frame_88.png b/results/mouth/frame_88.png
diff --git a/results/mouth/frame_89.png b/results/mouth/frame_89.png
diff --git a/results/mouth/frame_90.png b/results/mouth/frame_90.png
diff --git a/results/mouth/frame_91.png b/results/mouth/frame_91.png
diff --git a/results/mouth/frame_92.png b/results/mouth/frame_92.png
diff --git a/results/mouth/frame_93.png b/results/mouth/frame_93.png
diff --git a/results/mouth/frame_94.png b/results/mouth/frame_94.png
diff --git a/results/mouth/frame_95.png b/results/mouth/frame_95.png
diff --git a/results/mouth/frame_96.png b/results/mouth/frame_96.png
diff --git a/results/mouth/frame_97.png b/results/mouth/frame_97.png
diff --git a/results/mouth/frame_98.png b/results/mouth/frame_98.png
diff --git a/results/mouth/frame_99.png b/results/mouth/frame_99.png
diff --git a/results/output_video.mp4 b/results/output_video.mp4
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		model_checkpoint_path: "train_logs-62"
		all_model_checkpoint_paths: "train_logs-62"
-Original file line number
+Diff line change
@@ -0,0 +1,153 @@
+    (lp0
+    I0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI0
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI0
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI0
+    aI0
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    aI1
+    a.