OpenCV单kinect多帧静止场景的深度图像去噪

老板kinect去噪的任务下达已经有半个多月了，前期除了看了几天文献之外就打酱油了，好像每天都很忙，可是就是不知道在忙什么。这几天为了交差，就胡乱凑了几段代码，得到一个结果，也知道不行，先应付一下，再图打算。

程序思想很简单，先对静止的场景连续采样若干帧，然后对所有点在时间域取中值，对取完中值之后的无效点在空间域取最近邻，勉强将黑窟窿填上了。由于代码较长，现在奉上关键的几个片段：

#include<cv.h>
#include<highgui.h>
#include<iostream>
using namespace std;

#ifndef _DENOISE
#define _DENOISE

const int nFrames = 9;   // number of consecutive frames
const int width = 640;   // frame width
const int height = 480;  // frame height

class kinectDenoising
{
private:                       
	IplImage* denoisedImage;
	IplImage* frameSet[nFrames];
	unsigned int numOfFrames;
	CvRect imageROI;
public:
	kinectDenoising();
	~kinectDenoising();
	void addFrame(IplImage* img); 
	void setImageROI(bool isUpdate = true);
	void medianFiltering();	
	void nearestFiltering();
	void updateFrameSet(IplImage* img);
	void showDenoiedImage(const char* window);
	void showCurrentImage(const char* window);
};

void insertSort(unsigned short* data,int& len,unsigned short newData);

#endif

这是定义的头文件，装模作样的写了一个类，在构造函数里面，除了对denoisedImage分配内存之外其他都置0，析构函数需要释放denoisedImage和frameSet数组的内存。numOfFrames本来设计为frameSet中的图像的帧数，结果由于偷懒就用了一个定长的数组。

void kinectDenoising::setImageROI(bool isUpdate)
{
	if(!isUpdate) 
	{
		imageROI = cvRect(22,44,591,434);
	}
	else
	{
		IplImage* image8u = cvCreateImage(cvSize(width,height),IPL_DEPTH_8U,1);
		IplImage* bitImage = cvCreateImage(cvSize(width,height),IPL_DEPTH_8U,1);

		// cvThreshold can only handle images of 8UC1 or 32FC1
		cvConvertScale(frameSet[0],image8u,255.0/4096.0);
		cvThreshold(image8u,bitImage,0,1,CV_THRESH_BINARY);

        // the two mats rowReduced and colReduced have to be CV_32SC1 type
	    // for function cvReduce() seems not to suitable for 16U type and 
	    // 8U type doesn't have enough room for the result.
	    CvMat* rowReduced = cvCreateMat(1,bitImage->width,CV_32FC1);
	    // bitImage->width represents number of cols, while bitImage->height stands for rows
	    CvMat* colReduced = cvCreateMat(bitImage->height,1,CV_32FC1);

	    cvReduce(bitImage,rowReduced,0,CV_REDUCE_SUM);
	    cvReduce(bitImage,colReduced,1,CV_REDUCE_SUM);

		// compute imageROI.x
	    for(int i=0;i<rowReduced->cols;i++)
	    {
			float temp = CV_MAT_ELEM(*rowReduced,float,0,i);
		    if(temp>bitImage->height/3)
		    {
				imageROI.x = i;
			    break;
		    }
	    }

		// computer imageROI.width
	    for(int i=rowReduced->cols;i>0;i--)
	    {
			float temp = CV_MAT_ELEM(*rowReduced,float,0,i-1);
			if(temp>bitImage->height/3)
		    {
				imageROI.width = i-imageROI.x;
		     	break;
			}
        }

		// compute imageROI.y
	    for(int i=0;i<colReduced->rows;i++)
    	{
    		float temp = CV_MAT_ELEM(*colReduced,float,i,0);
    		if(temp>bitImage->height/3)
	    	{
	    		imageROI.y = i;
    			break;
	    	}
    	}

		// compute imageROI.height
    	for(int i=colReduced->rows;i>0;i--)
    	{
    		float temp = CV_MAT_ELEM(*colReduced,float,i-1,0);
    		if(temp>bitImage->height/3)
    		{
	    		imageROI.height = i-imageROI.y;
	    		break;
	    	}
    	}

		// set memory free
		cvReleaseImage(&bitImage);
		cvReleaseImage(&image8u);
		cvReleaseMat(&rowReduced);
		cvReleaseMat(&colReduced);
	}
}

这是计算深度图像的滤波范围。由于深度图像和彩色图像的视点不一致，导致了将深度图像映射到彩色图像上时有效像素会缩小，典型的现象就是在深度图像的四周会出现黑色的区域。这个函数就是用来将四周的黑色框框去掉。用OpenCV的投影的方法。由于cvReduce()函数要进行累积和的计算，为了不使数据溢出，目标数组应该用32位的浮点型（此函数只支持8位unsigned char型和32位float型）。

void kinectDenoising::medianFiltering()
{
	// set result image zero
	cvSetZero(denoisedImage);

	unsigned short data[nFrames];
	int total;
	for(int i=imageROI.y;i<imageROI.y+imageROI.height;i++)
	{
		unsigned short* denoisedImageData = (unsigned short*)(denoisedImage->imageData+denoisedImage->widthStep*i);
		for(int j=imageROI.x;j<imageROI.x+imageROI.width;j++)
		{
			total = 0;
			for(int k=0;k<nFrames;k++)
			{
				insertSort(data,total,CV_IMAGE_ELEM(frameSet[k],unsigned short,i,j));
			}
			if(total != 0)
			{
				denoisedImageData[j] = data[total/2];
			}
		}
	}
}

中值滤波，统计有效点并排序，然后取中值。insertSort()函数用来将值按从小到大的顺序进行插入，鉴于篇幅的关系，就不贴出来了。

void kinectDenoising::nearestFiltering()
{
	CvPoint topLeft,downRight;
	IplImage* tempImage = cvCloneImage(denoisedImage);
	for(int i=imageROI.y;i<imageROI.y+imageROI.height;i++)
	{
		unsigned short* data = (unsigned short*)(denoisedImage->imageData+denoisedImage->widthStep*i);
		for(int j=imageROI.x;j<imageROI.x+imageROI.width;j++)
		{
			for(int k=1;data[j]==0;k++)
			{
				topLeft = cvPoint(j-k,i-k);    // j为行数 i为列数
				downRight = cvPoint(j+k,i+k);
				for(int m=topLeft.x;(m<=downRight.x) && (data[j]==0);m++)
				{
					if(m<0) continue;
					if(m>=width) break;
					if(topLeft.y>=0)
					{
						unsigned short temp = CV_IMAGE_ELEM(tempImage,unsigned short,topLeft.y,m);
			    		if(temp > 0)
			    		{
		    				data[j] = temp;
							break;
		    			}
					}
					if(downRight.y < height)
					{
						unsigned short temp = CV_IMAGE_ELEM(tempImage,unsigned short,downRight.y,m);
			    		if(temp > 0)
						{
		    				data[j] = temp;
							break;
		    			}
					}					
				}

				for(int m=topLeft.y;(m<downRight.y) && (data[j]==0);m++)
				{
					if(m<0) continue;
					if(m>=height) break;
					if(topLeft.x>0)
					{
						unsigned short temp = CV_IMAGE_ELEM(tempImage,unsigned short,m,topLeft.x);
						if(temp > 0)
			    		{
		    				data[j] = temp;
							break;
		    			}
					}

					if(downRight.x<width)
					{
						unsigned short temp = CV_IMAGE_ELEM(tempImage,unsigned short,m,downRight.x);
						if(temp > 0)
			    		{
		    				data[j] = temp;
							break;
		    			}
					}
				}
			}
		}
	}
	cvReleaseImage(&tempImage);
}

最后是中值滤波，从最内层开始，一层层往外扩，直到找到有效值为止。

运行结果：

源图像：

结果图像：

附注：本来这个程序是在8位图像上进行的。先取得16位的unsigned short型深度图像，然后通过cvConvertScale()函数将其转化为8位的unsigned char型，结果在进行去噪的时候怎么都不对，将unsigned char型的数据放到matlab中一看，发现在unsigned short型数据中为0值的像素莫名其妙的在unsigned char型里有了一个很小的值（比如说1, 2, 3, 4, 5什么的，就是不为0）。很奇怪，不知道OpenCV中是怎么搞的。看来还是源数据靠谱，于是将其改为16位的unsigned short型，结果形势一片大好。

http://blog.csdn.net/chenli2010/article/details/7006573