Kinect学习笔记第八篇Coordinate Mapping(坐标映射) 总结
干货:
Writeablebitmap继承自ImageSource
Bitmap继承自Image
Bitmapdata
以上三种在此图像处理中比较常用
关于本例中图像处理时,若采用数组形式的操作,仅是“拷贝像素”一命令就可以让程序失去响应(1920*1080*4次运算),别问我傻傻等待了多长时间....比较合适的操作是内存或者指针法。在前面的第显示映射后图像程序中采用了指针,指针效率最高,由于直接访问内存速度是三种中最快的,而且例中采用对像素级处理,所以比较适合,省去了类之间的转换。
(之前我访问内存报错是怎么回事?微软有时为了保护自己知识产权,部分代码与所占用的内存无法访问,在思考解决方法时应避开。)
Coordinate Mapper(坐标映射器):可实现以下
(2法一般指访问底层缓存与使用数组)
1所有相机点映射到彩色空间*2法
2个别相机点映射到彩色空间(第二实例采用-下篇)
3所有相机点映射到深度空间*2法
4个别相机点映射到深度空间
5彩色帧映射到相机空间*2法
6彩色帧映射到深度空间*2法(第一个实例采用)
7深度帧映射到相机空间*2法
8深度帧映射到彩色空间*2法
9所有深度点映射到相机空间*2法
10所有深度点映射到彩色空间*2法(第三个实例采用)
11个别深度点映射到相机空间
12个别深度点映射到彩色空间
Writeablebitmap与Bitmap中每个ARGB格式像素点的值在内存中由四个字节组成,分别是保存A值,R值,G值,B值,都是(0-255)并且排列成数组,排列顺序是第一行排完排再从第二行第一个开始,如此循环排完所有像素。这是对二者进行指针操作的基础。
关于映射后的点,由于精度或者二者视界的不同,以及分辨率的不同,肯定会产生不符合标准的映射点,在使用时必须放弃,否则很容易产生越界等问题。具体例子详见笔记六及代码输出部分。
Writeablebitmap与Bitmap在内存中4字节一像素,每字节依次代表分别是B-G-R-A值!
代码功能简介:
本代码通过Kinect中提供的CoordinateMapper(映射器)中的
MapDepthFrameToColorSpaceUsingIntPtr方法,得到彩色点映射到深度后的映射点坐标,然后通过将彩色流中的该点的像素值提取,然后组成一张新的图像(512*424),即为深度点着色,但是还是2维图像。
缺点:
由于Kinect所用的色彩帧与深度帧分辨率不同,所以他采用的是范围映射,(在笔记六中有介绍)在为深度点着色的时候,我舍弃了不必要,无法使用的值,并且没做优化处理,他会采用范围内哪个点并不确定,但在坐标上表现是最近的那个点,所以色彩图可能不完整,对应看起来效果也不是很好。至与为什么匹配后的坐标值是float型而不是int型,因为不能完美的在深度与色彩帧之间匹配,所欲我们所得到的准确映射,是在所在映射像素之间的值,然后我们决定怎么用,我是直接取整。
代码示例:(取消注释部分可输出数据查看)
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
using Microsoft.Kinect;
using System.IO;
using System.Drawing;
using System.Drawing.Imaging;//
namespace myCoordinateMappingViewer
{
/// <summary>
/// MainWindow.xaml 的交互逻辑
/// </summary>
///
public partial class MainWindow : Window
{
private readonly int bytesPerPixel = (PixelFormats.Bgr32.BitsPerPixel + 7) / 8;
private KinectSensor kinectSensor = null;
private CoordinateMapper coordinateMapper = null;//坐标映射器
private MultiSourceFrameReader multiFrameSourceReader = null;//多源阅读器
private WriteableBitmap bitmap = null;
private WriteableBitmap colorbitmap = null;
private uint bitmapBackBufferSize = 0;//Bitmap缓冲区大小
private ColorSpacePoint[] depthMappedToColorPoints = null;
private byte[] depthPixels = null;
public bool savejudge1 = true;
public MainWindow()
{
this.kinectSensor = KinectSensor.GetDefault();
this.multiFrameSourceReader = this.kinectSensor.OpenMultiSourceFrameReader(FrameSourceTypes.Depth | FrameSourceTypes.Color );
this.multiFrameSourceReader.MultiSourceFrameArrived += this.Reader_MultiSourceFrameArrived;
this.coordinateMapper = this.kinectSensor.CoordinateMapper;
//DepthFrame
FrameDescription depthFrameDescription = this.kinectSensor.DepthFrameSource.FrameDescription;
int depthWidth = depthFrameDescription.Width;
int depthHeight = depthFrameDescription.Height;
this.depthPixels = new byte[depthWidth * depthHeight];
//ColorFrame
FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription;
int colorWidth = colorFrameDescription.Width;
int colorHeight = colorFrameDescription.Height;
this.depthMappedToColorPoints = new ColorSpacePoint[depthWidth * depthHeight];
this.bitmap = new WriteableBitmap(colorWidth, colorHeight, 96.0, 96.0, PixelFormats.Bgra32, null);
// Calculate the WriteableBitmap back buffer size 1920*1080*4(以下这么写是让你了解底层数据大小)
this.bitmapBackBufferSize = (uint)((this.bitmap.BackBufferStride * (this.bitmap.PixelHeight - 1)) + (this.bitmap.PixelWidth * this.bytesPerPixel));
this.colorbitmap = new WriteableBitmap(depthWidth, depthHeight, 96.0, 96.0, PixelFormats.Bgra32, null);//新图
this.kinectSensor.Open();
this.DataContext = this;
InitializeComponent();
}
private void Reader_MultiSourceFrameArrived(object sender, MultiSourceFrameArrivedEventArgs e)
{
int depthWidth = 0;
int depthHeight = 0;
DepthFrame depthFrame = null;
ColorFrame colorFrame = null;
bool isBitmapLocked = false;
MultiSourceFrame multiSourceFrame = e.FrameReference.AcquireFrame();
if (multiSourceFrame == null)
{
return;
}
try
{
depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame();
colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame();
if ((depthFrame == null) || (colorFrame == null) )
{
return;
}
// Process Depth
FrameDescription depthFrameDescription = depthFrame.FrameDescription;
depthWidth = depthFrameDescription.Width;
depthHeight = depthFrameDescription.Height;
//映射
using (KinectBuffer depthFrameData = depthFrame.LockImageBuffer())
{
//映射到一个与深度图大小相同的depthMappedToColorPoints处
this.coordinateMapper.MapDepthFrameToColorSpaceUsingIntPtr(
depthFrameData.UnderlyingBuffer,
depthFrameData.Size,
this.depthMappedToColorPoints);
}
#region
//输出映射后点的坐标
/*if (true)
{
FileStream fs = new FileStream("D:\\depthMappedToColorPoints.txt", FileMode.OpenOrCreate);
StreamWriter sw = new StreamWriter(fs);
int num = 0;
for (int i=0; i < this.depthMappedToColorPoints.Length;++i )
{
//过滤
if (!float.IsNegativeInfinity(this.depthMappedToColorPoints[i].X) && !float.IsNegativeInfinity(this.depthMappedToColorPoints[i].Y))
{
//if(this.depthMappedToColorPoints[i].X>1920||this.depthMappedToColorPoints[i].Y>1080)
if(true)
{
sw.Write("[{0},{1}]", this.depthMappedToColorPoints[i].X, this.depthMappedToColorPoints[i].Y);
num++;
if (num == 10)
{
sw.Write("\r\n");
num = 0;
}
}
}
}
sw.Write("\r\n共计{0}", this.depthMappedToColorPoints.Length);
sw.Flush();
sw.Close();
fs.Close();
this.savejudge1 = false;
}*/
#endregion
// Process Color
//彩色数据写入this.bitmap.BackBuffer
colorFrame.CopyConvertedFrameDataToIntPtr(this.bitmap.BackBuffer, this.bitmapBackBufferSize, ColorImageFormat.Bgra);
using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
{
this.bitmap.Lock();
this.colorbitmap.Lock();
//处理并取得新图
ProcessDepthFrameData(this.depthMappedToColorPoints,this.bitmap);
isBitmapLocked = true;
}
this.bitmap.Unlock();
this.colorbitmap.Unlock();
isBitmapLocked = false;
depthFrame.Dispose();
depthFrame = null;
colorFrame.Dispose();
colorFrame = null;
}
finally
{
if (isBitmapLocked)
{
this.bitmap.Unlock();
this.colorbitmap.Unlock();
}
if (depthFrame != null) depthFrame.Dispose();
if (colorFrame != null) colorFrame.Dispose();
}
}
private unsafe void ProcessDepthFrameData(ColorSpacePoint[] depthMappedToColorPoints,WriteableBitmap bitmapItem)
{
int BytePerPixel=4;
int stride = 1920 * BytePerPixel;
var pixels = (byte *)bitmapItem.BackBuffer.ToPointer();
var pointer = (byte*)this.colorbitmap.BackBuffer.ToPointer();
#region
取得像素点源
//FileStream fs = new FileStream("D:\\彩色像素点查看.txt", FileMode.OpenOrCreate);
//StreamWriter sw = new StreamWriter(fs);
//int num = 0;
//for (int x = 0; x < 1080*1920*4; x++)
//{
// sw.Write("{0},",pixels[x]);
// num++;
// if (num == 4)
// {
// sw.Write("\r\n");
// num = 0;
// }
//}
//sw.Write("\r\n共计{0}", this.depthMappedToColorPoints.Length);
//sw.Flush();
//sw.Close();
//fs.Close();
#endregion
for (int y = 0; y < 424; y++)
{
for (int x = 0; x < 512; x++)
{
if (!float.IsNegativeInfinity(depthMappedToColorPoints[y * 512 + x].X) &&
!float.IsNegativeInfinity(depthMappedToColorPoints[y * 512 + x].Y) &&
(depthMappedToColorPoints[y * 512 + x].X >= 0) &&
(depthMappedToColorPoints[y * 512 + x].Y >= 0) &&
(depthMappedToColorPoints[y * 512 + x].Y <= 1080) &&
(depthMappedToColorPoints[y * 512 + x].X <= 1920)
)
{
//获得对应色彩点的内存地址(byte信息)
int yOfMemory = (int)(depthMappedToColorPoints[y * 512 + x].Y + 0.5);
if (yOfMemory + 1 > 1080) yOfMemory--;//除去大于等于1080的值
//int adress = (int)(depthMappedToColorPoints[y * 512 + x].X) + (int)(depthMappedToColorPoints[y * 512 + x].Y ) * 1920;
int adress = (int)(depthMappedToColorPoints[y * 512 + x].X + 0.5) + yOfMemory * 1920;
adress *= BytePerPixel;
//获得对应深度点的内存地址
int index = y * 512 + x;
index *= BytePerPixel;
//写入
pointer[index] = pixels[adress];
pointer[index + 1] = pixels[adress + 1];
pointer[index + 2] = pixels[adress + 2];
pointer[index + 3] = pixels[adress + 3];
}
else
{
//获得对应深度点的内存地址
int index = y * 512 + x;
index *= BytePerPixel;
//无效映射值状态显示为红色
/*pointer[index] = 0xff;
pointer[index + 1] = 0xff;
pointer[index + 2] = 0;
pointer[index + 3] = 0;*/
}
}
}
//更新
this.colorbitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorbitmap.PixelWidth, this.colorbitmap.PixelHeight));
#region
//FileStream fs = new FileStream("D:\\合成图像素查看.txt", FileMode.OpenOrCreate);
//StreamWriter sw = new StreamWriter(fs);
//int num = 0;
//for (int y =0 ; y < 424; y++)
//{
// for (int x = 0; x < 512; x++)
// {
// //sw.Write("{0},{1},{2},{3}",xx1,xx2,xx3,xx4);
// int index = y * 512 + x;
// index *= BytePerPixel;
// for (int i = 0; i < 4;i++ )
// {
// sw.Write("{0},", pointer[(index+ i)]);
// }
// sw.Write("\r\n");
// num++;
// if (num == 4)
// {
// sw.Write("\r\n");
// num = 0;
// }
// }
//}
//sw.Write("\r\n共计{0}",512*424);
//sw.Flush();
//sw.Close();
//fs.Close();
#endregion
}
public ImageSource colorImageSource
{
get
{
return (this.colorbitmap);
}
}
private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
{
if (this.multiFrameSourceReader != null)
{
this.multiFrameSourceReader.Dispose();
this.multiFrameSourceReader = null;
}
if (this.kinectSensor != null)
{
this.kinectSensor.Close();
this.kinectSensor = null;
}
}
}
}
程序流程图:
!字还是那个字! (╯‵□′)╯︵┻━┻
关于本程序:如果把处理无效的映射值部分注释,在Kinect前不改变任何东西,在位图内存的部分数据继承原来的基础上(部分刷新,算是一个投机),会将映射后图片质量略有提高,但是一旦Kinect前情景改变,所有点的数据会有刷新与不刷新,造成残影。不加注释,就会每张位图数据完全刷新(实时),不继承原来的,不会有残影,但是质量不是很高。
补充:
Xaml代码
<Window x:Class="myCoordinateMappingViewer.MainWindow"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
Title="CoordinateMapper" Height="600" Width="700" Closing="Window_Closing" Background="White" Foreground="#FFFF0057">
<Grid>
<!-- <Image HorizontalAlignment="Left" VerticalAlignment="Top" Source="{Binding ImageSource}" Stretch="UniformToFill"/>-->
<Image Source="{Binding colorImageSource}" Stretch="UniformToFill" />
</Grid>
</Window>