Skip to content

Commit

Permalink
lab07
Browse files Browse the repository at this point in the history
  • Loading branch information
@jesse-nsquared
jesse-nsquared committed Mar 16, 2015
1 parent 7ec37f4 commit a7e180d
Show file tree
Hide file tree
Showing 5 changed files with 196 additions and 20 deletions.
Binary file added Kinect2Sample/Assets/Background.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
33 changes: 33 additions & 0 deletions Kinect2Sample/DisplayTypeToVisibilityConverter.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Data;

namespace Kinect2Sample
{
class DisplayTypeToVisibilityConverter :IValueConverter
{
public object Convert(object value, Type targetType, object parameter, string language)
{
String boundString = Enum.GetName(typeof(DisplayFrameType), value);
String matchString = (String)parameter;

if (String.Equals(boundString, matchString))
{
return Visibility.Visible;
}
else
{
return Visibility.Collapsed;
}
}

public object ConvertBack(object value, Type targetType, object parameter, string language)
{
throw new NotImplementedException();
}
}
}
2 changes: 2 additions & 0 deletions Kinect2Sample/Kinect2Sample.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
</Compile>
<Compile Include="BodiesManager.cs" />
<Compile Include="BodyInfo.cs" />
<Compile Include="DisplayTypeToVisibilityConverter.cs" />
<Compile Include="MainPage.xaml.cs">
<DependentUpon>MainPage.xaml</DependentUpon>
</Compile>
Expand All @@ -79,6 +80,7 @@
</AppxManifest>
</ItemGroup>
<ItemGroup>
<Content Include="Assets\Background.png" />
<Content Include="Assets\Logo.scale-100.png" />
<Content Include="Assets\SmallLogo.scale-100.png" />
<Content Include="Assets\SplashScreen.scale-100.png" />
Expand Down
19 changes: 17 additions & 2 deletions Kinect2Sample/MainPage.xaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<Setter Property="Margin" Value="6"/>
<Setter Property="FontSize" Value="20"/>
</Style>
<local:DisplayTypeToVisibilityConverter x:Key="DisplayTypeToVisibilityConverter"/>
</Page.Resources>

<Grid Background="{ThemeResource ApplicationPageBackgroundThemeBrush}" >
Expand All @@ -44,12 +45,22 @@
<TextBlock Text=", FrameHeight = "/>
<TextBlock Text="{Binding CurrentFrameDescription.Height}"/>
</StackPanel>
<!--...-->
<Image x:Name="FrameBackground" Source="Assets/Background.png" Grid.Row="1" Stretch="Fill"
Visibility="{Binding CurrentDisplayFrameType,
Converter={StaticResource DisplayTypeToVisibilityConverter},
ConverterParameter=BackgroundRemoved }"/>
<Image x:Name="FrameDisplayImage" Grid.Row="1" Stretch="Uniform"/>
<Viewbox Grid.Row="1" HorizontalAlignment="Center">
<Grid x:Name="BodyJointsGrid" Background="Transparent" Width="512" Height="414"/>
</Viewbox>
<!--...-->
<StackPanel Grid.Row="1" Orientation="Vertical" HorizontalAlignment="Left"
Visibility="{Binding CurrentDisplayFrameType,
Converter={StaticResource DisplayTypeToVisibilityConverter},
ConverterParameter=BackgroundRemoved }">
<TextBlock Text="DepthMax"/>
<TextBlock Text="{Binding DepthMax}"/>
<Slider Width="250" Minimum="500" Maximum="8000" Value="{Binding DepthMax, Mode=TwoWay}"/>
</StackPanel>
<ScrollViewer Grid.Row="2" ScrollViewer.HorizontalScrollBarVisibility="Auto"
ScrollViewer.VerticalScrollBarVisibility="Auto">
<StackPanel Orientation="Horizontal">
Expand All @@ -67,6 +78,10 @@
Click="BodyJointsButton_Click">
<TextBlock Text="Body Joints" TextWrapping="Wrap"/>
</Button>
<Button Style="{StaticResource FrameSelectorButtonStyle}"
Click="BackgroundButton_Click">
<TextBlock Text="BG Removed" TextWrapping="Wrap" />
</Button>
</StackPanel>
</ScrollViewer>
</Grid>
Expand Down
162 changes: 144 additions & 18 deletions Kinect2Sample/MainPage.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ public enum DisplayFrameType
Color,
Depth,
BodyMask,
BodyJoints
BodyJoints,
BackgroundRemoved
}

public sealed partial class MainPage : Page, INotifyPropertyChanged
Expand Down Expand Up @@ -92,14 +93,14 @@ public sealed partial class MainPage : Page, INotifyPropertyChanged
//Depth Frame
private ushort[] depthFrameData = null;
private byte[] depthPixels = null;
private ushort depthMax = 8000;

//BodyMask Frames
private DepthSpacePoint[] colorMappedToDepthPoints = null;

//Body Joints are drawn here
private Canvas drawingCanvas;


public event PropertyChangedEventHandler PropertyChanged;
public string StatusText
{
Expand Down Expand Up @@ -133,6 +134,38 @@ public FrameDescription CurrentFrameDescription
}
}

public DisplayFrameType CurrentDisplayFrameType
{
get { return this.currentDisplayFrameType; }
set
{
if (this.currentDisplayFrameType != value)
{
this.currentDisplayFrameType = value;
if (this.PropertyChanged != null)
{
this.PropertyChanged(this, new PropertyChangedEventArgs("CurrentDisplayFrameType"));
}
}
}
}

public ushort DepthMax
{
get { return this.depthMax; }
set
{
if (this.depthMax != value)
{
this.depthMax = value;
if (this.PropertyChanged != null)
{
this.PropertyChanged(this, new PropertyChangedEventArgs("DepthMax"));
}
}
}
}

public MainPage()
{
// one sensor is currently supported
Expand Down Expand Up @@ -160,9 +193,10 @@ public MainPage()

private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
{
currentDisplayFrameType = newDisplayFrameType;
CurrentDisplayFrameType = newDisplayFrameType;
// Frames used by more than one type are declared outside the switch
FrameDescription colorFrameDescription = null;
FrameDescription depthFrameDescription = null;
// reset the display methods
if (this.BodyJointsGrid != null)
{
Expand All @@ -172,7 +206,7 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
{
this.FrameDisplayImage.Source = null;
}
switch (currentDisplayFrameType)
switch (CurrentDisplayFrameType)
{
case DisplayFrameType.Infrared:
FrameDescription infraredFrameDescription = this.kinectSensor.InfraredFrameSource.FrameDescription;
Expand All @@ -191,7 +225,7 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
break;

case DisplayFrameType.Depth:
FrameDescription depthFrameDescription = this.kinectSensor.DepthFrameSource.FrameDescription;
depthFrameDescription = this.kinectSensor.DepthFrameSource.FrameDescription;
this.CurrentFrameDescription = depthFrameDescription;
// allocate space to put the pixels being received and converted
this.depthFrameData = new ushort[depthFrameDescription.Width * depthFrameDescription.Height];
Expand Down Expand Up @@ -222,6 +256,17 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
this.BodyJointsGrid.Children.Add(this.drawingCanvas);
bodiesManager = new BodiesManager(this.coordinateMapper, this.drawingCanvas, this.kinectSensor.BodyFrameSource.BodyCount);
break;

case DisplayFrameType.BackgroundRemoved:
colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription;
depthFrameDescription = this.kinectSensor.DepthFrameSource.FrameDescription;
// Actual current frame is going to be a map of depth and color, choosing the larger to display(color)
this.CurrentFrameDescription = colorFrameDescription;
// allocate space to put the pixels being received and converted
this.depthFrameData = new ushort[depthFrameDescription.Width * depthFrameDescription.Height];
this.colorMappedToDepthPoints = new DepthSpacePoint[colorFrameDescription.Width * colorFrameDescription.Height];
this.bitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height);
break;
default:
break;
}
Expand All @@ -234,7 +279,6 @@ private void Sensor_IsAvailableChanged(KinectSensor sender, IsAvailableChangedEv

private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, MultiSourceFrameArrivedEventArgs e)
{

MultiSourceFrame multiSourceFrame = e.FrameReference.AcquireFrame();

// If the Frame has expired by the time we process this event, return.
Expand All @@ -247,12 +291,12 @@ private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, Multi
InfraredFrame infraredFrame = null;
BodyFrame bodyFrame = null;
BodyIndexFrame bodyIndexFrame = null;
IBuffer depthFrameData = null;
IBuffer depthFrameDataBuffer = null;
IBuffer bodyIndexFrameData = null;
// Com interface for unsafe byte manipulation
IBufferByteAccess bodyIndexByteAccess = null;
IBufferByteAccess bufferByteAccess = null;

switch (currentDisplayFrameType)
switch (CurrentDisplayFrameType)
{
case DisplayFrameType.Infrared:
using (infraredFrame = multiSourceFrame.InfraredFrameReference.AcquireFrame())
Expand Down Expand Up @@ -285,18 +329,16 @@ private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, Multi
}

// Access the depth frame data directly via LockImageBuffer to avoid making a copy
depthFrameData = depthFrame.LockImageBuffer();
this.coordinateMapper.MapColorFrameToDepthSpaceUsingIBuffer(depthFrameData, this.colorMappedToDepthPoints);
depthFrameDataBuffer = depthFrame.LockImageBuffer();
this.coordinateMapper.MapColorFrameToDepthSpaceUsingIBuffer(depthFrameDataBuffer, this.colorMappedToDepthPoints);
// Process Color
colorFrame.CopyConvertedFrameDataToBuffer(this.bitmap.PixelBuffer, ColorImageFormat.Bgra);
// Access the body index frame data directly via LockImageBuffer to avoid making a copy
bodyIndexFrameData = bodyIndexFrame.LockImageBuffer();
ShowMappedBodyFrame(depthFrame.FrameDescription.Width, depthFrame.FrameDescription.Height, bodyIndexFrameData, bodyIndexByteAccess);

ShowMappedBodyFrame(depthFrame.FrameDescription.Width, depthFrame.FrameDescription.Height, bodyIndexFrameData, bufferByteAccess);
}
finally
{
// ... disposing of depth, color and bodyIndex frames
if (depthFrame != null)
{
depthFrame.Dispose();
Expand All @@ -310,18 +352,18 @@ private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, Multi
bodyIndexFrame.Dispose();
}

if (depthFrameData != null)
if (depthFrameDataBuffer != null)
{
// We must force a release of the IBuffer in order to ensure that we have dropped all references to it.
System.Runtime.InteropServices.Marshal.ReleaseComObject(depthFrameData);
System.Runtime.InteropServices.Marshal.ReleaseComObject(depthFrameDataBuffer);
}
if (bodyIndexFrameData != null)
{
System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexFrameData);
}
if (bodyIndexByteAccess != null)
if (bufferByteAccess != null)
{
System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexByteAccess);
System.Runtime.InteropServices.Marshal.ReleaseComObject(bufferByteAccess);
}

}
Expand All @@ -332,11 +374,89 @@ private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, Multi
ShowBodyJoints(bodyFrame);
}
break;
case DisplayFrameType.BackgroundRemoved:
// Put in a try catch to utilise finally() and clean up frames
try
{
depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame();
colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame();
if ((depthFrame == null) || (colorFrame == null))
{
return;
}
depthFrame.CopyFrameDataToArray(depthFrameData);
this.coordinateMapper.MapColorFrameToDepthSpace(depthFrameData, this.colorMappedToDepthPoints);
// Process Color.
colorFrame.CopyConvertedFrameDataToBuffer(this.bitmap.PixelBuffer, ColorImageFormat.Bgra);

ShowMappedColorBackgroundRemoved(colorMappedToDepthPoints, depthFrameData, depthFrame.FrameDescription);
}
finally
{
if (depthFrame != null)
{
depthFrame.Dispose();
}
if (colorFrame != null)
{
colorFrame.Dispose();
}
}
break;
default:
break;
}
}

unsafe private void ShowMappedColorBackgroundRemoved(DepthSpacePoint[] colorMappedToDepthPoints, ushort[] depthFrameData, FrameDescription frameDescription)
{
fixed (DepthSpacePoint* colorMappedToDepthPointsPointer = colorMappedToDepthPoints)
{
IBufferByteAccess bitmapBackBufferByteAccess = (IBufferByteAccess)this.bitmap.PixelBuffer;

byte* bitmapBackBufferBytes = null;
bitmapBackBufferByteAccess.Buffer(out bitmapBackBufferBytes);

// Treat the color data as 4-byte pixels
uint* bitmapPixelsPointer = (uint*)bitmapBackBufferBytes;

int depthWidth = frameDescription.Width;
int depthHeight = frameDescription.Height;

// Loop over each row and column of the color image
// Zero out any pixels that don't correspond to a body index
for (int colorIndex = 0; colorIndex < this.colorMappedToDepthPoints.Length; ++colorIndex)
{
float colorMappedToDepthX = colorMappedToDepthPoints[colorIndex].X;
float colorMappedToDepthY = colorMappedToDepthPoints[colorIndex].Y;

// The sentinel value is -inf, -inf, meaning that no depth pixel corresponds to this color pixel.
if (!float.IsNegativeInfinity(colorMappedToDepthX) &&
!float.IsNegativeInfinity(colorMappedToDepthY))
{
// Make sure the depth pixel maps to a valid point in color space
int depthX = (int)(colorMappedToDepthX + 0.5f);
int depthY = (int)(colorMappedToDepthY + 0.5f);

// If the point is not valid, there is no body index there.
if ((depthX >= 0) && (depthX < depthWidth) && (depthY >= 0) && (depthY < depthHeight))
{
int depthIndex = (depthY * depthWidth) + depthX;

if (depthFrameData[depthIndex] < DepthMax)
{
continue;
}
}
}
// no matching depth. zero out the pixel.
bitmapPixelsPointer[colorIndex] = 0;
}
}
this.bitmap.Invalidate();
FrameDisplayImage.Source = this.bitmap;
}

private void ShowBodyJoints(BodyFrame bodyFrame)
{
Body[] bodies = new Body[this.kinectSensor.BodyFrameSource.BodyCount];
Expand Down Expand Up @@ -590,6 +710,12 @@ private void BodyJointsButton_Click(object sender, RoutedEventArgs e)
SetupCurrentDisplay(DisplayFrameType.BodyJoints);
}

private void BackgroundButton_Click(object sender, RoutedEventArgs e)
{
SetupCurrentDisplay(DisplayFrameType.BackgroundRemoved);
}


[Guid("905a0fef-bc53-11df-8c49-001e4fc686da"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
interface IBufferByteAccess
{
Expand Down

0 comments on commit a7e180d

Please sign in to comment.