opencv_特征匹配与图像拼接

opencv中，常用的特征点检测算法有：SURF算法，SIFT算法，ORB算法，FAST算法。

其中检测速度上，在提取一帧图像特征点的实验中，在提取相同数量的特征点情况下，提取SURF点耗时时间大约是提取ORB特征点的14倍，而提取SIFT点耗时更大，大概比提取ORB特征点多三百多倍，所以ORB是计算量最小的算法。

本文使用的SUFR的算法进行的特征点检测以及图像拼接，若想在在opencv中使用SURF的特征点检测算法，需要编译opencv_contrib的模块，具体的流程可见上一篇博客： 

opencv_contrib的编译与运行(windows)_9分钟带帽的博客-CSDN博客

 两张demo的图片如下：

 代码如下：

#include <iostream>
#include<opencv2\opencv.hpp>
#include<opencv2\xfeatures2d.hpp>
#include<vector>

using namespace std;
using namespace cv;
using namespace xfeatures2d;

typedef struct
{
	Point2f left_top;
	Point2f left_bottom;
	Point2f right_top;
	Point2f right_bottom;
}four_corners_t;

int clip(int x, int min, int max)
{
	if (x > max)
		return max;
	if (x < min)
		return min;
	return x;
}


void compute_overlap_mask(Mat& image1, Mat& image2, Mat& lap_mask)
{
	int height1 = image1.rows;
	int width1 = image1.cols;
	int height2 = image2.rows;
	int width2 = image2.cols;
	int height0 = lap_mask.rows;
	int width0 = lap_mask.cols;
	if (height0 != height1 || height0 != height2 || width0 != width1 || width0 != width2)
	{
		cout << "image shape not match!" << endl;
		return;
	}
	for (int i = 0; i < height0; i++)
		for (int j = 0; j < width0; j++)
		{
			Vec3b pix1 = image1.at<Vec3b>(i, j);
			Vec3b pix2 = image2.at<Vec3b>(i, j);
			Vec3b pix0 = Vec3b(0, 0, 0);
			if (pix1 == pix0 || pix2 == pix0)
				lap_mask.at<uchar>(i, j) = 0;
		}
}


void CalcCorners(const Mat& H, const Mat& src, four_corners_t& corners)
{
	double v2[] = { 0, 0, 1 };//左上角
	double v1[3];//变换后的坐标值
	Mat V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	Mat V1 = Mat(3, 1, CV_64FC1, v1);  //列向量

	V1 = H * V2;
	//左上角(0,0,1)
	cout << "V2: " << V2 << endl;
	cout << "V1: " << V1 << endl;
	corners.left_top.x = v1[0] / v1[2];
	corners.left_top.y = v1[1] / v1[2];

	//左下角(0,src.rows,1)
	v2[0] = 0;
	v2[1] = src.rows;
	v2[2] = 1;
	V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	corners.left_bottom.x = v1[0] / v1[2];
	corners.left_bottom.y = v1[1] / v1[2];

	//右上角(src.cols,0,1)
	v2[0] = src.cols;
	v2[1] = 0;
	v2[2] = 1;
	V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	corners.right_top.x = v1[0] / v1[2];
	corners.right_top.y = v1[1] / v1[2];

	//右下角(src.cols,src.rows,1)
	v2[0] = src.cols;
	v2[1] = src.rows;
	v2[2] = 1;
	V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	corners.right_bottom.x = v1[0] / v1[2];
	corners.right_bottom.y = v1[1] / v1[2];

}

int merge_image_by_mask1(Mat& lap_mask, Mat& mix_room_01, Mat& mix_room_02, Mat& mix_room)
{
	int height1 = mix_room_01.rows;
	int width1 = mix_room_01.cols;
	int height2 = mix_room_02.rows;
	int width2 = mix_room_02.cols;
	int height0 = lap_mask.rows;
	int width0 = lap_mask.cols;
	if (height0 != height1 || height0 != height2 || width0 != width1 || width0 != width2)
	{
		cout << "image shape not match!" << endl;
		return -1;
	}

	int mix_width = lap_mask.cols;
	int mix_height = lap_mask.rows;

	//创建遮罩层并根据mask完成权重初始化
	Mat mask1 = Mat::ones(Size(mix_width, mix_height), CV_32FC1);
	Mat mask2 = Mat::ones(Size(mix_width, mix_height), CV_32FC1);

	//printf("%s %d\n", __FUNCTION__, __LINE__);
	for (int i = 0; i < height0; i++)
	{
		vector<int> indexs;
		indexs.clear();
		for (int j = 0; j < width0; j++)
		{
			if (lap_mask.at<uchar>(i, j) == 255)
				indexs.push_back(j);
		}
		if (indexs.size() == 0)
			continue;
		int start = indexs.front();
		int end = indexs.back();
		//printf("line %d start:%d end:%d\n", i, start, end);
		int length = end - start + 1;
		float interval = 1.0 / length;
		for (int j = start; j < end + 1; j++)
		{
			mask2.at<float>(i, j) = (j - start + 1) * interval;
			mask1.at<float>(i, j) = 1.0 - mask2.at<float>(i, j);
		}
	}

	Mat m1w;
	vector<Mat> mvec;
	mvec.push_back(mask1);
	mvec.push_back(mask1);
	mvec.push_back(mask1);
	merge(mvec, m1w);
	mix_room_01.convertTo(mix_room_01, CV_32F);
	multiply(mix_room_01, m1w, mix_room_01);

	Mat m2w;
	mvec.clear();
	mvec.push_back(mask2);
	mvec.push_back(mask2);
	mvec.push_back(mask2);
	merge(mvec, m2w);
	mix_room_02.convertTo(mix_room_02, CV_32F);
	multiply(mix_room_02, m2w, mix_room_02);

	add(mix_room_01, mix_room_02, mix_room);
	mix_room.convertTo(mix_room, CV_8U);

	return 0;
}

int merge_image_by_mask2(Mat& lap_mask, Mat& mix_room_01, Mat& mix_room_02, Mat& mix_room)
{
	int height1 = mix_room_01.rows;
	int width1 = mix_room_01.cols;
	int height2 = mix_room_02.rows;
	int width2 = mix_room_02.cols;
	int height0 = lap_mask.rows;
	int width0 = lap_mask.cols;
	if (height0 != height1 || height0 != height2 || width0 != width1 || width0 != width2)
	{
		cout << "image shape not match!" << endl;
		return -1;
	}

	int mix_width = lap_mask.cols;
	int mix_height = lap_mask.rows;

	for (int i = 0; i < height0; i++)
	{
		uchar* p1 = mix_room_01.ptr<uchar>(i);
		uchar* p2 = mix_room_02.ptr<uchar>(i);
		uchar* pm = mix_room.ptr<uchar>(i);

		vector<int> indexs;
		indexs.clear();
		for (int j = 0; j < width0; j++)
		{
			if (lap_mask.at<uchar>(i, j) == 255)
				indexs.push_back(j);
		}
		if (indexs.size() == 0)
			continue;
		int start = indexs.front();
		int end = indexs.back();
		int length = end - start + 1;
		float interval = 1.0 / length;
		for (int j = start; j < end + 1; j++)
		{
			float w2 = (j - start + 1) * interval;
			float w1 = 1.0 - w2;
			pm[j * 3] = clip(int(p1[j * 3] * w1 + p2[j * 3] * w2), 0, 255);
			pm[j * 3 + 1] = clip(int(p1[j * 3 + 1] * w1 + p2[j * 3 + 1] * w2), 0, 255);
			pm[j * 3 + 2] = clip(int(p1[j * 3 + 2] * w1 + p2[j * 3 + 2] * w2), 0, 255);
		}
	}

	return 0;
}

int main()
{
	std::string imgpath1 = "../images/room/room1.jpg";
	std::string imgpath2 = "../images/room/room2.jpg";
	Mat room1 = imread(imgpath1);
	Mat room2 = imread(imgpath2);
	if (room1.empty() == true || room2.empty() == true) {
		cout << "error" << endl;
		return -1;
	}

	Ptr<SURF> surf;
	surf = SURF::create(800);
	vector<KeyPoint> kpts_room1;
	vector<KeyPoint> kpts_room2;
	Mat dec_room1, dec_room2;
	surf->detectAndCompute(room1, Mat(), kpts_room1, dec_room1);
	surf->detectAndCompute(room2, Mat(), kpts_room2, dec_room2);


	/*---------暴力 匹配-------------*/
	//创建暴力匹配子对象
	BFMatcher bf_matcher;
	//存放描述子匹配关系
	vector<DMatch> matches_bf;
	//特征点描述子匹配
	bf_matcher.match(dec_room1, dec_room2, matches_bf);


	//特征点筛选
	float good_rate = 0.15f;
	//printf("matches_bf.size:%d\n", matches_bf.size());
	int num_good_matchs = std::min(15, int(matches_bf.size() * good_rate));
	//printf("num_good_matchs:%d\n", num_good_matchs);
	std::sort(matches_bf.begin(), matches_bf.end());
	matches_bf.erase(matches_bf.begin() + num_good_matchs, matches_bf.end());
	//绘制筛选后匹配结果
	Mat result_bf;
	drawMatches(room1, kpts_room1, room2, kpts_room2, matches_bf, result_bf);
	imshow("result_bf", result_bf);
	//获取两张图的特征点
	vector<Point2f>room1_points;
	vector<Point2f>room2_points;
	for (size_t t = 0; t < matches_bf.size(); t++) {
		room1_points.push_back(kpts_room1[matches_bf[t].queryIdx].pt);
		room2_points.push_back(kpts_room2[matches_bf[t].trainIdx].pt);
	}
	//根据对应的特征点获取从demo->scene的变换矩阵
	Mat homo = findHomography(room2_points, room1_points, RANSAC);
	//waitKey(0);


	four_corners_t corners;
	CalcCorners(homo, room2, corners);
	//cout << "left_top:" << corners.left_top << endl;
	//cout << "left_bottom:" << corners.left_bottom << endl;
	//cout << "right_top:" << corners.right_top << endl;
	//cout << "right_bottom:" << corners.right_bottom << endl;
	//cout << "room1.width:" << room1.cols << " " << "room1.height:" << room1.rows << endl;

	//Mat imageTransform;
	int mix_width = int(std::max(corners.right_top.x, corners.right_bottom.x));
	int mix_height = int(room1.rows);


	Mat mix_room_01 = Mat::zeros(Size(mix_width, mix_height), CV_8UC3);
	Rect roi1 = Rect(0, 0, room1.cols, room1.rows);
	room1.copyTo(mix_room_01(roi1));
	imshow("mix_room_01", mix_room_01);

	Mat mix_room_02;
	warpPerspective(room2, mix_room_02, homo, Size(mix_width, mix_height));
	imshow("mix_room2", mix_room_02);

	//计算重合区域mask
	Mat lap_mask = Mat::ones(Size(mix_width, mix_height), CV_8UC1) * 255;
	//lap_mask.setTo(255);
	compute_overlap_mask(mix_room_01, mix_room_02, lap_mask);
	//generate_mask(mix_room_02, mask);
	imshow("lap_mask", lap_mask);

	//method1 计算两张图片分别的mask权重，根据两个mask融合图片
	//Mat mix_room;
	//merge_image_by_mask1(lap_mask, mix_room_01, mix_room_02, mix_room);

	//method2 根据overlap_mask计算重叠区域的像素值，直接融合
	Mat mix_room = mix_room_02.clone();
	room1.copyTo(mix_room(Rect(0, 0, room1.cols, room1.rows)));
	merge_image_by_mask2(lap_mask, mix_room_01, mix_room_02, mix_room);
	imshow("mix_room", mix_room);
	waitKey(0);

	std::string save_imgpath = "result.jpg";
	cv::imwrite(save_imgpath, mix_room);

	//system("pause");
	return 0;
}

其中得到了重叠的mask，进行图像融合的过程中，有两个函数，第一个是merge_image_by_mask1，该函数先计算两张图片对应的mask的权重，方便进行可视化，第二个是merge_image_by_mask2是根据重叠的lap_mask，取两张图片对应坐标的像素，直接计算融合之后的像素值。

参考博客：

opencv ORB特征提取与匹配-实现图像拼接_java opencv 匹配拼接_开阳654的博客-CSDN博客

OpenCV SURF图像拼接、配准和图像融合技术（一）_使用opencv surf算法实现图像的拼接和融合_rjszcb的博客-CSDN博客