diff --git a/Kinect2Sample/Kinect2Sample.csproj b/Kinect2Sample/Kinect2Sample.csproj index 2409758..6f8d299 100644 --- a/Kinect2Sample/Kinect2Sample.csproj +++ b/Kinect2Sample/Kinect2Sample.csproj @@ -92,6 +92,9 @@ + + Libraries\x64\DirectXSceneStore.winmd + Libraries\x64\KinectFaceStore.dll diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore.dll b/Kinect2Sample/Libraries/x64/DirectXSceneStore.dll new file mode 100644 index 0000000..2c91ab2 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore.dll differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore.pri b/Kinect2Sample/Libraries/x64/DirectXSceneStore.pri new file mode 100644 index 0000000..bafa749 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore.pri differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore.winmd b/Kinect2Sample/Libraries/x64/DirectXSceneStore.winmd new file mode 100644 index 0000000..5f5cdda Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore.winmd differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/bounce.wav b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/bounce.wav new file mode 100644 index 0000000..8267695 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/bounce.wav differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellceiling.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellceiling.dds new file mode 100644 index 0000000..dd24b59 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellceiling.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellfloor.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellfloor.dds new file mode 100644 index 0000000..790df24 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellfloor.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellwall.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellwall.dds new file mode 100644 index 0000000..7bfc4af Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/cellwall.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/dayceiling.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/dayceiling.dds new file mode 100644 index 0000000..6148ae3 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/dayceiling.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/dayfloor.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/dayfloor.dds new file mode 100644 index 0000000..22f11fa Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/dayfloor.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/daywall.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/daywall.dds new file mode 100644 index 0000000..6148ae3 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/daywall.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/hit.wav b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/hit.wav new file mode 100644 index 0000000..f4a78ac Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/hit.wav differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/mediumTile-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/mediumTile-sdk.png new file mode 100644 index 0000000..1c9f68e Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/mediumTile-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/metal_texture.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/metal_texture.dds new file mode 100644 index 0000000..af2e355 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/metal_texture.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightceiling.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightceiling.dds new file mode 100644 index 0000000..291a923 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightceiling.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightfloor.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightfloor.dds new file mode 100644 index 0000000..04e424b Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightfloor.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightwall.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightwall.dds new file mode 100644 index 0000000..291a923 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/nightwall.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/seafloor.dds b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/seafloor.dds new file mode 100644 index 0000000..3c8eed9 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/seafloor.dds differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/smallTile-phone-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/smallTile-phone-sdk.png new file mode 100644 index 0000000..4476cda Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/smallTile-phone-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/smallTile-windows-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/smallTile-windows-sdk.png new file mode 100644 index 0000000..e39b01f Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/smallTile-windows-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/splash-phone-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/splash-phone-sdk.png new file mode 100644 index 0000000..435f872 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/splash-phone-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/splash-windows-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/splash-windows-sdk.png new file mode 100644 index 0000000..e00df02 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/splash-windows-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/squareTile-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/squareTile-sdk.png new file mode 100644 index 0000000..2a8cf87 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/squareTile-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/storeLogo-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/storeLogo-sdk.png new file mode 100644 index 0000000..5c397de Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/storeLogo-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/tile-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/tile-sdk.png new file mode 100644 index 0000000..f72683f Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/tile-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/windows-sdk.png b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/windows-sdk.png new file mode 100644 index 0000000..6726802 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/Assets/windows-sdk.png differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/GameUIStyles.xbf b/Kinect2Sample/Libraries/x64/DirectXSceneStore/GameUIStyles.xbf new file mode 100644 index 0000000..faae63d Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/GameUIStyles.xbf differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/PixelShader.cso b/Kinect2Sample/Libraries/x64/DirectXSceneStore/PixelShader.cso new file mode 100644 index 0000000..96f56e3 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/PixelShader.cso differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/PixelShaderFlat.cso b/Kinect2Sample/Libraries/x64/DirectXSceneStore/PixelShaderFlat.cso new file mode 100644 index 0000000..fc57594 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/PixelShaderFlat.cso differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/ScenePanel.xbf b/Kinect2Sample/Libraries/x64/DirectXSceneStore/ScenePanel.xbf new file mode 100644 index 0000000..e75cb98 Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/ScenePanel.xbf differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/VertexShader.cso b/Kinect2Sample/Libraries/x64/DirectXSceneStore/VertexShader.cso new file mode 100644 index 0000000..c4b50be Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/VertexShader.cso differ diff --git a/Kinect2Sample/Libraries/x64/DirectXSceneStore/VertexShaderFlat.cso b/Kinect2Sample/Libraries/x64/DirectXSceneStore/VertexShaderFlat.cso new file mode 100644 index 0000000..bfffc0f Binary files /dev/null and b/Kinect2Sample/Libraries/x64/DirectXSceneStore/VertexShaderFlat.cso differ diff --git a/Kinect2Sample/MainPage.xaml b/Kinect2Sample/MainPage.xaml index 58960c8..c2ce693 100644 --- a/Kinect2Sample/MainPage.xaml +++ b/Kinect2Sample/MainPage.xaml @@ -5,6 +5,7 @@ xmlns:local="using:Kinect2Sample" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:dx="using:DirectXSceneStore" mc:Ignorable="d"> @@ -53,6 +54,10 @@ + @@ -90,9 +95,13 @@ + diff --git a/Kinect2Sample/MainPage.xaml.cs b/Kinect2Sample/MainPage.xaml.cs index 3c5d326..7d609ec 100644 --- a/Kinect2Sample/MainPage.xaml.cs +++ b/Kinect2Sample/MainPage.xaml.cs @@ -34,13 +34,17 @@ public enum DisplayFrameType BodyJoints, BackgroundRemoved, FaceOnColor, - FaceOnInfrared + FaceOnInfrared, + FaceGame } public sealed partial class MainPage : Page, INotifyPropertyChanged { private const DisplayFrameType DEFAULT_DISPLAYFRAMETYPE = DisplayFrameType.Infrared; + private const double FACE_AIMING_ACCURACY= 1.0; + private const double FACE_AIMING_SENSITIVITY = 0.01; + /// /// The highest value that can be returned in the InfraredFrame. /// It is cast to a float for readability in the visualization code. @@ -113,6 +117,10 @@ public sealed partial class MainPage : Page, INotifyPropertyChanged //Cat assets private Image[] catEyeRightOpen, catEyeRightClosed, catEyeLeftOpen, catEyeLeftClosed, catNose; + //Face Orientation Shaping + private double prevPitch = 0.0f; + private double prevYaw = 0.0f; + public event PropertyChangedEventHandler PropertyChanged; public string StatusText { @@ -391,6 +399,13 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType) this.FacePointsCanvas.Height = infraredFrameDescription.Height; break; + case DisplayFrameType.FaceGame: + colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription; + this.CurrentFrameDescription = colorFrameDescription; + this.FacePointsCanvas.Width = colorFrameDescription.Width; + this.FacePointsCanvas.Height = colorFrameDescription.Height; + break; + default: break; } @@ -540,16 +555,69 @@ private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, Multi DrawFaceOnInfrared(); } break; + case DisplayFrameType.FaceGame: + FaceGameLookUpdate(); + break; default: break; } } + private void FaceGameLookUpdate() + { + this.FacePointsCanvas.Children.Clear(); + FaceFrameResult[] results = faceManager.GetLatestFaceFrameResults(); + for (int i = 0; i < results.Count(); i++) + { + if (results[i] != null) + { + foreach (KeyValuePair facePointKVP in + results[i].FacePointsInColorSpace) + { + if (facePointKVP.Value.X == 0.0 || facePointKVP.Value.Y == 0.0) + { + break; + } + Size ellipseSize = new Size(10, 10); + Ellipse ellipse = new Ellipse(); + ellipse.Width = ellipseSize.Width; + ellipse.Height = ellipseSize.Height; + ellipse.Fill = new SolidColorBrush(Colors.Red); + Canvas.SetLeft(ellipse, facePointKVP.Value.X - (ellipseSize.Width / 2)); + Canvas.SetTop(ellipse, facePointKVP.Value.Y - (ellipseSize.Height / 2)); + this.FacePointsCanvas.Children.Add(ellipse); + } + + double pitch, roll, yaw = 0; + + ExtractFaceRotationInDegrees(results[i].FaceRotationQuaternion, out pitch, out yaw, out roll); + + double pitchDiff = Math.Abs(pitch - prevPitch); + double yawDiff = Math.Abs(yaw - prevYaw); + if (pitchDiff > FACE_AIMING_ACCURACY || + yawDiff > FACE_AIMING_ACCURACY) + { + this.DXScenePanel.SetYawPitch( + -(float)(yaw * FACE_AIMING_SENSITIVITY), + (float)(pitch * FACE_AIMING_SENSITIVITY)); + prevPitch = pitch; + prevYaw = yaw; + } + + if (results[i].FaceProperties[FaceProperty.MouthOpen] == DetectionResult.Yes) + { + this.DXScenePanel.Fire(); + } + break; + } + } + } + private void DrawFaceOnInfrared() { FacePointsCanvas.Children.Clear(); FaceFrameResult[] results = faceManager.GetLatestFaceFrameResults(); - for (int i = 0; i < results.Count(); i++ ) + for (int i = 0; i < results.Count(); i++) { if (results[i] != null) { @@ -869,6 +937,19 @@ private void RenderPixelArray(byte[] pixels) this.FrameDisplayImage.Source = this.bitmap; } + private static void ExtractFaceRotationInDegrees(Vector4 rotQuaternion, out double pitch, out double yaw, out double roll) + { + double x = rotQuaternion.X; + double y = rotQuaternion.Y; + double z = rotQuaternion.Z; + double w = rotQuaternion.W; + + // convert face rotation quaternion to Euler angles in degrees + pitch = Math.Atan2(2 * ((y * z) + (w * x)), (w * w) - (x * x) - (y * y) + (z * z)) / Math.PI * 180.0; + yaw = Math.Asin(2 * ((w * y) - (x * z))) / Math.PI * 180.0; + roll = Math.Atan2(2 * ((x * y) + (w * z)), (w * w) + (x * x) - (y * y) - (z * z)) / Math.PI * 180.0; + } + private void InfraredButton_Click(object sender, RoutedEventArgs e) { SetupCurrentDisplay(DisplayFrameType.Infrared); @@ -914,5 +995,10 @@ interface IBufferByteAccess { unsafe void Buffer(out byte* pByte); } + + private void FaceGameButton_Click(object sender, RoutedEventArgs e) + { + SetupCurrentDisplay(DisplayFrameType.FaceGame); + } } }