OpenCV之基于特征的跟踪（十）

基于特征的跟踪是指跟踪视频中连续帧的各个特征点，其优点是不必在每一帧中检测特征点，可以只检测一次，之后继续跟踪它们。
采用一种称为光流的技术来跟踪这些特征，光流是计算机视觉中最流行的技术之一。该技术需要选择一组特征点，并通过视频流跟踪它们。当检测到特征点时，则计算位移向量并显示连续帧之间的关键点的运动情况，这些向量称为运动向量。与前一帧相比，特定点的运动向量基本上只是指示该点移动位置的方向线。
当前最流行的检测运动向量的算法Lucas-Kanade方法和Farneback算法
1，Lucas-Kanade方法用于稀疏光流跟踪，即特征点的数量相对较少。参考原始论文：http://cseweb.ucsd.edu/classes/sp02/cse252/lucaskanade81.pdf。提取特征点，对于每个特征点创建3x3个区域，特征点位于中心，假设每个区域中的所有点都具有相似的运动。
当前帧的每个特征点，将周围的3x3区域作为参考点，查看前一帧中的邻域以获得最佳匹配。邻域通常大于3x3，以获得最接近区域的区域。从前一帧匹配区域的中心像素到当前帧被考虑区域的中心像素的路径即运动向量。对所有特征点执行此操作，并提取所有的运动向量。

bool pointTrackingFlag = false;
Point2f currentPoint;

//Function to detect mouse events
void onMouse(int event, int x, int y, int, void*)
{
	//Detect the mouse button down event
	if (event == EVENT_LBUTTONDOWN) {
		//Assign the current (x,y) position to currentPoint
		currentPoint = Point2f((float)x, (float)y);
		//Set the tracking flag
		pointTrackingFlag = true;
	}
}

int main(int argc, char* argv[])
{
	VideoCapture cap(0);
	if (!cap.isOpened())
		return -1;
	//Termination criteria for tracking the points
	TermCriteria terminationCriteria(TermCriteria::COUNT | TermCriteria::EPS, 10, 0.02);
	//Size of the block that is used for matching
	Size windowSize(25, 25);
	//Maximum number of points that you want to track
	const int maxNumPoints = 200;
	string windowName = "Lucas Kanade Tracker";
	namedWindow(windowName, 1);
	setMouseCallback(windowName, onMouse, 0);

	Mat prevGrayImage, curGrayImage, image, frame;
	vector<Point2f> trackingPoints[2];
	float scalingFactor = 0.75;
	while (true) {
		cap >> frame;
		if (frame.empty())
			break;

		resize(frame, frame, Size(), scalingFactor, scalingFactor, INTER_AREA);
		frame.copyTo(image);
		cvtColor(image, curGrayImage, COLOR_BGR2GRAY);
		if (!trackingPoints[0].empty()) {
			//Status vector to indicate whether the flow for the corresponding features has been found
			vector<uchar> statusVector;
			//Error vector to indicate the error for corresponding feature
			vector<float> errorVector;
			if (prevGrayImage.empty()) {
				curGrayImage.copyTo(prevGrayImage);
			}

			//Calculate the optical flow using Lucas-Kanade algorithm
			calcOpticalFlowPyrLK(prevGrayImage, curGrayImage, trackingPoints[0], trackingPoints[1], statusVector, errorVector, windowSize, 3, terminationCriteria, 0, 0.001);

			int count = 0;
			int minDist = 7;
			for (int i = 0; i < trackingPoints[1].size(); i++) {
				if (pointTrackingFlag) {
					//If the new point is within 'minDist' distance from an existing point, it will not be tracked
					if (norm(currentPoint - trackingPoints[1][i]) <= minDist) {
						pointTrackingFlag = false;
						continue;
					}
				}
				if (!statusVector[i])
					continue;

				trackingPoints[1][count++] = trackingPoints[1][i];

				//Draw a filled circle for each of the tracking points
				int radius = 8;
				int thickness = 2;
				int lineType = 8;
				circle(image, trackingPoints[1][i], radius, Scalar(0, 255, 0), thickness, lineType);
			}
			trackingPoints[1].resize(count);
		}

		//Refining the location of the feature points
		if (pointTrackingFlag&&trackingPoints[1].size() < maxNumPoints) {
			vector<Point2f> tempPoints;
			tempPoints.push_back(currentPoint);

			//Function to refine the location of the corners to subpixel accuracy.
            //Here, 'pixel' refers to the image patch of size 'windowSize' and not the actual image pixel
			cornerSubPix(curGrayImage, tempPoints, windowSize, Size(-1, -1), terminationCriteria);
			trackingPoints[1].push_back(tempPoints[0]);
			pointTrackingFlag = false;
		}
		imshow(windowName, image);
		char ch = waitKey(30);
		if (ch == 27)
			break;

		std::swap(trackingPoints[1], trackingPoints[0]);
		cv::swap(prevGrayImage, curGrayImage);
	}

	cap.release();
	destroyAllWindows();
	return 0;
}

添加一些想要跟踪的点，当移动到不同位置时，在一个小的误差范围内仍然可以正确跟踪这些点，但是，由于突出物和移动速度因素，有些点被丢弃。

2，Farneback算法，Gunnar Farneback提出这种光流算法，用于密集跟踪，广泛用于机器人、增强现实和3D映射。原始论文查看：http://www.divaportal.org/smash/get/diva2:273847/FULLTEXT01.pdf。
Lucas-Kanade方法是稀疏技术，只需要处理整个图像中的某些像素，Farneback算法是密集技术，需要处理给定图像中所有像素。密集技术更准确但速度较慢，稀疏技术不太准确但速度快，需要取舍。实时应用选择稀疏技术，不考虑时间和复杂性的应用选择密集技术。
在Farneback论文中，描述了一种基于两帧多项式展开的密集光流估计方法，目标是估计这两帧之间的运动，第一步，用多项式逼近得到两个帧中的每个邻域，此时只关注二次多项式，第二步通过整体位移构建一个新信号，每个邻域是多项式逼近的，因此需要看看多项式经过理想转换后会发生什么，第三步通过使这些二次多项式的系数相等来计算全局位移。

bool pointTrackingFlag = false;
Point2f currentPoint;

//Function to compute the optical flow map
void drawOpticalFlow(const Mat& flowImage, Mat& flowImageGray)
{
	int stepSize = 16;
	Scalar color = Scalar(0, 255, 0);
	// Draw the uniform grid of points on the input image along with the motion vectors
	for (int y = 0; y < flowImageGray.rows; y += stepSize) {
		for (int x = 0; x < flowImageGray.cols; x += stepSize) {
			int radius = 2;
			int thickness = -1;
			circle(flowImageGray, Point(x, y), radius, color, thickness);

			Point2f pt = flowImage.at<Point2f>(y, x);
			line(flowImageGray, Point(x, y), Point(cvRound(x + pt.x), cvRound(y + pt.y)), color);
		}
	}
}

int main(int argc, char* argv[])
{
	VideoCapture cap(0);
	if (!cap.isOpened())
		return -1;
	Mat curGray, prevGray, flowImage, flowImageGray, frame;
	string windowName = "Optical Flow";
	namedWindow(windowName, 1);
	float scalingFactor = 0.75;
	while (true) {
		cap >> frame;
		if (frame.empty())
			break;
		resize(frame, frame, Size(), scalingFactor, scalingFactor, INTER_AREA);
		cvtColor(frame, curGray, COLOR_BGR2GRAY);

		if (prevGray.data) {
			float pyrScale = 0.5;
			int numLevels = 3;
			int windowSize = 15;
			int numIterations = 3;
			int neighborhoodSize = 5;
			float stdDeviation = 1.2;

			//Calculate optical flow map using Farneback algorithm
			calcOpticalFlowFarneback(prevGray, curGray, flowImage, pyrScale, numLevels, windowSize, numIterations, neighborhoodSize, stdDeviation, 0);
			//Convert to 3-channel RGB
			cvtColor(prevGray, flowImageGray, COLOR_GRAY2BGR);
			//flowImageGray = prevGray.clone();
			//Draw the optical flow map
			drawOpticalFlow(flowImage, flowImageGray);

			imshow(windowName, flowImageGray);
		}
		char ch = waitKey(10);
		if (ch == 27)
			break;

		std::swap(prevGray, curGray);
	}
	cap.release();
	destroyAllWindows();
	return 0;
}