Some plumbing:

- support for 16 bit images
- preventing copy constructors for certain utilities classes

README.md

@@ -80,5 +80,5 @@ For inquiries about the commercial licensing of the OpenFace toolkit please cont
 
 # Final remarks
 
-I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at Tadas.Baltrusaitis@cl.cam.ac.uk for any bug reports/questions/suggestions. 
+I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at Tadas.Baltrusaitis@cl.cam.ac.uk for any bug reports/questions/suggestions. I prefer questions and bug reports on github as that provides visibility to others who might be encountering same issues or who have the same questions.
 

exe/FaceLandmarkImg/FaceLandmarkImg.cpp

@@ -79,69 +79,6 @@ vector<string> get_arguments(int argc, char **argv)
 	return arguments;
 }
 
-// TODO rem
-void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkDetector::CLNF& clnf_model)
-{
-
-	// Draw head pose if present and draw eye gaze as well
-
-	// preparing the visualisation image
-	display_image = orig.clone();		
-
-	// Creating a display image			
-	cv::Mat xs = clnf_model.detected_landmarks(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows/2));
-	cv::Mat ys = clnf_model.detected_landmarks(cv::Rect(0, clnf_model.detected_landmarks.rows/2, 1, clnf_model.detected_landmarks.rows/2));
-	double min_x, max_x, min_y, max_y;
-
-	cv::minMaxLoc(xs, &min_x, &max_x);
-	cv::minMaxLoc(ys, &min_y, &max_y);
-
-	double width = max_x - min_x;
-	double height = max_y - min_y;
-
-	int minCropX = max((int)(min_x-width/3.0),0);
-	int minCropY = max((int)(min_y-height/3.0),0);
-
-	int widthCrop = min((int)(width*5.0/3.0), display_image.cols - minCropX - 1);
-	int heightCrop = min((int)(height*5.0/3.0), display_image.rows - minCropY - 1);
-
-	double scaling = 350.0/widthCrop;
-
-	// first crop the image
-	display_image = display_image(cv::Rect((int)(minCropX), (int)(minCropY), (int)(widthCrop), (int)(heightCrop)));
-
-	// now scale it
-	cv::resize(display_image.clone(), display_image, cv::Size(), scaling, scaling);
-
-	// Make the adjustments to points
-	xs = (xs - minCropX)*scaling;
-	ys = (ys - minCropY)*scaling;
-
-	cv::Mat shape = clnf_model.detected_landmarks.clone();
-
-	xs.copyTo(shape(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows/2)));
-	ys.copyTo(shape(cv::Rect(0, clnf_model.detected_landmarks.rows/2, 1, clnf_model.detected_landmarks.rows/2)));
-
-	// Do the shifting for the hierarchical models as well
-	for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part)
-	{
-		cv::Mat xs = clnf_model.hierarchical_models[part].detected_landmarks(cv::Rect(0, 0, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2));
-		cv::Mat ys = clnf_model.hierarchical_models[part].detected_landmarks(cv::Rect(0, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2));
-
-		xs = (xs - minCropX)*scaling;
-		ys = (ys - minCropY)*scaling;
-
-		cv::Mat shape = clnf_model.hierarchical_models[part].detected_landmarks.clone();
-
-		xs.copyTo(shape(cv::Rect(0, 0, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2)));
-		ys.copyTo(shape(cv::Rect(0, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2, 1, clnf_model.hierarchical_models[part].detected_landmarks.rows / 2)));
-
-	}
-
-	//LandmarkDetector::Draw(display_image, clnf_model);
-
-}
-
 int main (int argc, char **argv)
 {
 
@@ -271,7 +208,10 @@ int main (int argc, char **argv)
 			open_face_rec.WriteObservation();
 
 		}
-		visualizer.ShowObservation();
+		if(face_detections.size() > 0)
+		{
+			visualizer.ShowObservation();
+		}
 
 		// Grabbing the next frame in the sequence
 		captured_image = image_reader.GetNextImage();

lib/local/Utilities/Utilities.vcxproj

@@ -159,6 +159,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="include\ImageCapture.h" />
+    <ClInclude Include="include\ImageManipulationHelpers.h" />
     <ClInclude Include="include\RecorderCSV.h" />
     <ClInclude Include="include\RecorderHOG.h" />
     <ClInclude Include="include\RecorderOpenFace.h" />

lib/local/Utilities/Utilities.vcxproj.filters

@@ -68,5 +68,8 @@
     <ClInclude Include="include\ImageCapture.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="include\ImageManipulationHelpers.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

lib/local/Utilities/include/ImageCapture.h

@@ -57,8 +57,6 @@ namespace Utilities
 		// Default constructor
 		ImageCapture() {};
 
-		// TODO block copy, move etc.
-
 		// Opening based on command line arguments
 		bool Open(std::vector<std::string>& arguments);
 
@@ -94,9 +92,15 @@ namespace Utilities
 
 	private:
 
+		// Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source
+		ImageCapture & operator= (const ImageCapture& other);
+		ImageCapture & operator= (const ImageCapture&& other);
+		ImageCapture(const ImageCapture&& other);
+		ImageCapture(const ImageCapture& other);
+
 		// Storing the latest captures
 		cv::Mat latest_frame;
-		cv::Mat latest_gray_frame;
+		cv::Mat_<uchar> latest_gray_frame;
 
 		// Keeping track of how many files are read and the filenames
 		size_t  frame_num;
@@ -110,7 +114,6 @@ namespace Utilities
 		bool image_focal_length_set;
 		bool image_optical_center_set;
 
-		// TODO make sure the error message makes sense
 		bool no_input_specified;
 
 	};

lib/local/Utilities/include/ImageManipulationHelpers.h

@@ -0,0 +1,124 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2017, Tadas Baltrusaitis all rights reserved.
+//
+// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY
+//
+// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT.  
+// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE.
+//
+// License can be found in OpenFace-license.txt
+//
+//     * Any publications arising from the use of this software, including but
+//       not limited to academic journal and conference publications, technical
+//       reports and manuals, must cite at least one of the following works:
+//
+//       OpenFace: an open source facial behavior analysis toolkit
+//       Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency
+//       in IEEE Winter Conference on Applications of Computer Vision, 2016  
+//
+//       Rendering of Eyes for Eye-Shape Registration and Gaze Estimation
+//       Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling 
+//       in IEEE International. Conference on Computer Vision (ICCV),  2015 
+//
+//       Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection
+//       Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson 
+//       in Facial Expression Recognition and Analysis Challenge, 
+//       IEEE International Conference on Automatic Face and Gesture Recognition, 2015 
+//
+//       Constrained Local Neural Fields for robust facial landmark detection in the wild.
+//       Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. 
+//       in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013.    
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef __IMAGE_MANIPULATION_HELPERS_h_
+#define __IMAGE_MANIPULATION_HELPERS_h_
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/imgproc.hpp>
+
+namespace Utilities
+{
+	//===========================================================================
+	// Converting between color spaces and bit depths
+	//===========================================================================
+
+	// Perform the conversion inplace
+	static void ConvertToRGB_8bit(cv::Mat& in_out)
+	{
+		if (in_out.channels() == 3 && in_out.depth() == CV_16U)
+		{
+			in_out = in_out / 256;
+		}
+		else if (in_out.channels() == 4)
+		{
+			if (in_out.depth() == CV_16U)
+			{
+				in_out = in_out / 256;
+				cv::cvtColor(in_out, in_out, CV_BGRA2BGR);
+			}
+			else
+			{
+				cv::cvtColor(in_out, in_out, CV_BGRA2BGR);
+			}
+		}
+		else if (in_out.channels() == 1)
+		{
+			if (in_out.depth() == CV_16U)
+			{
+				in_out = in_out / 256;
+				cv::cvtColor(in_out, in_out, CV_GRAY2BGR);
+			}
+			else
+			{
+				cv::cvtColor(in_out, in_out, CV_GRAY2BGR);
+			}
+		}
+	}
+
+	static void ConvertToGrayscale_8bit(const cv::Mat& in, cv::Mat& out)
+	{
+		if (in.channels() == 3)
+		{
+			// Make sure it's in a correct format
+			if (in.depth() == CV_16U)
+			{
+				cv::Mat tmp = in / 256;
+				tmp.convertTo(out, CV_8U);
+				cv::cvtColor(out, out, CV_BGR2GRAY);
+			}
+			else
+			{
+				cv::cvtColor(in, out, CV_BGR2GRAY);
+			}
+		}
+		else if (in.channels() == 4)
+		{
+			if (in.depth() == CV_16U)
+			{
+				cv::Mat tmp = in / 256;
+				tmp.convertTo(out, CV_8U);
+				cv::cvtColor(out, out, CV_BGRA2GRAY);
+			}
+			else
+			{
+				cv::cvtColor(in, out, CV_BGRA2GRAY);
+			}
+		}
+		else
+		{
+			if (in.depth() == CV_16U)
+			{
+				cv::Mat tmp = in / 256;
+				tmp.convertTo(out, CV_8U);
+			}
+			else if (in.depth() == CV_8U)
+			{
+				out = in.clone();
+			}
+		}
+	}
+
+
+}
+#endif

lib/local/Utilities/include/RecorderCSV.h

@@ -68,10 +68,14 @@ namespace Utilities
 			const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d,
 			const std::vector<std::pair<std::string, double> >& au_intensities, const std::vector<std::pair<std::string, double> >& au_occurences);
 
-		// TODO have set functions?
-
 	private:
 
+		// Blocking copy and move, as it doesn't make sense to read to write to the same file
+		RecorderCSV & operator= (const RecorderCSV& other);
+		RecorderCSV & operator= (const RecorderCSV&& other);
+		RecorderCSV(const RecorderCSV&& other);
+		RecorderCSV(const RecorderCSV& other);
+
 		// The actual output file stream that will be written
 		std::ofstream output_file;
 

lib/local/Utilities/include/RecorderHOG.h

@@ -68,6 +68,12 @@ namespace Utilities
 
 	private:
 
+		// Blocking copy and move, as it doesn't make sense to read to write to the same file
+		RecorderHOG & operator= (const RecorderHOG& other);
+		RecorderHOG & operator= (const RecorderHOG&& other);
+		RecorderHOG(const RecorderHOG&& other);
+		RecorderHOG(const RecorderHOG& other);
+
 		std::ofstream hog_file;
 
 		// Internals for recording

lib/local/Utilities/include/RecorderOpenFace.h

@@ -61,8 +61,6 @@ namespace Utilities
 
 		~RecorderOpenFace();
 
-		// TODO copy, assignment and move operators? Do not allow
-
 		// Closing and cleaning up the recorder
 		void Close();
 
@@ -100,6 +98,12 @@ namespace Utilities
 
 	private:
 
+		// Blocking copy, assignment and move operators, as it does not make sense to save to the same location
+		RecorderOpenFace & operator= (const RecorderOpenFace& other);
+		RecorderOpenFace & operator= (const RecorderOpenFace&& other);
+		RecorderOpenFace(const RecorderOpenFace&& other);
+		RecorderOpenFace(const RecorderOpenFace& other);
+
 		// Keeping track of what to output and how to output it
 		const RecorderOpenFaceParameters params;
 

lib/local/Utilities/include/SequenceCapture.h

@@ -57,7 +57,8 @@ namespace Utilities
 		// Default constructor
 		SequenceCapture() {};
 
-		// TODO block copy, move etc.
+		// Destructor
+		~SequenceCapture();
 
 		// Opening based on command line arguments
 		bool Open(std::vector<std::string>& arguments);
@@ -104,12 +105,18 @@ namespace Utilities
 
 	private:
 
+		// Blocking copy and move, as it doesn't make sense to have several readers pointed at the same source, and this would cause issues, especially with webcams
+		SequenceCapture & operator= (const SequenceCapture& other);
+		SequenceCapture & operator= (const SequenceCapture&& other);
+		SequenceCapture(const SequenceCapture&& other);
+		SequenceCapture(const SequenceCapture& other);
+
 		// Used for capturing webcam and video
 		cv::VideoCapture capture;
 
 		// Storing the latest captures
 		cv::Mat latest_frame;
-		cv::Mat latest_gray_frame;
+		cv::Mat_<uchar> latest_gray_frame;
 
 		// Keeping track of frame number and the files in the image sequence
 		size_t  frame_num;

lib/local/Utilities/include/Visualizer.h

@@ -55,11 +55,7 @@ namespace Utilities
 		// The constructor for the visualizer that specifies what to visualize
 		Visualizer(std::vector<std::string> arguments);
 		Visualizer(bool vis_track, bool vis_hog, bool vis_align);
-
-		//~Visualizer();
-
-		// TODO copy, assignment and move operators? Do not allow
-
+
 		// Adding observations to the visualizer
 
 		// Pose related observations