基于opencv的手势识别(HSV)控制鼠标

opencv 同时被 3 个专栏收录
5 篇文章 1 订阅
8 篇文章 1 订阅
6 篇文章 1 订阅

基于opencv的手势识别和鼠标控制

opencv3.4.5和vs2015

新建工程->空项目->新建项

摄像头使用

	VideoCapture capture;
	capture.open(0);
	if (!capture.isOpened()){//打开失败返回-1
		cout << "No camera!\n" << endl;
		return -1;
	while (1) {
			capture >> frame;//刷新
			//Mat frame = imread("1.jpg");  //工程测试
			if (frame.empty())//帧空跳出循环
				break;
			imshow("frame_", frame);
			}
	}

分类器的使用

这里使用了自己做的分类器用作手势识别

定义、读取

	CascadeClassifier open_palm_cascade;
	CascadeClassifier closed_palm_cascade;
	open_palm_cascade.load("palm.xml");
	closed_palm_cascade.load("closed_palm.xml");
	if (open_palm_cascade.empty()) {
		cout << "Could not load open_palm configuration file! "
			"Check directory! " << endl << "Press Q to Quit!" << endl;
		while (char(waitKey(0)) != 'q') {}
		return -2;
	}
	if (closed_palm_cascade.empty()) {
		cout << "Could not load closed_palm configuration file! "
			"Check directory! " << endl << "Press Q to Quit!" << endl;
		while (char(waitKey(0)) != 'q') {}
		return -2;
	}

使用

open_palm_cascade.detectMultiScale(frame, open_palms, 1.3, 4, 0, Size(50, 50));
closed_palm_cascade.detectMultiScale(frame, closed_palms, 1.3, 4, 0, Size(50, 50));
			if (closed_palms.size() != 0) {
				for (int i = 0; i < closed_palms.size(); i++) {
					//cout << "=============Detected a closed_palm!=============" << endl;
					// Top left and bottom right points of rectangle.
					Point closed_palm_rect_p1(closed_palms[i].x, closed_palms[i].y);
					Point closed_palm_rect_p2(closed_palms[i].x + closed_palms[i].width, closed_palms[i].y + closed_palms[i].height);
					// Draw the rectangle in the image.
					rectangle(frame, closed_palm_rect_p1, closed_palm_rect_p2, Scalar(0, 255, 0));
					putText(frame, "Closed Palm", closed_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
						1, Scalar(0, 255, 0), 1, 5, false);
					hand_mode[0] = 1;
				}
			}
			for (int i = 0; i < open_palms.size(); i++) {
				//cout << "=============Detected an open_palm!=============" << endl;
				// Top left and bottom right points of rectangle.
				Point open_palm_rect_p1(open_palms[i].x, open_palms[i].y);
				Point open_palm_rect_p2(open_palms[i].x + open_palms[i].width, open_palms[i].y + open_palms[i].height);
				// Draw the rectangle in the image.
				rectangle(frame, open_palm_rect_p1, open_palm_rect_p2, Scalar(255, 0, 0));
				putText(frame, "Open Palm", open_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
					1, Scalar(255, 0, 0), 1, 5, false);
				hand_mode[0] = 2;
			}
			

图像处理、滤波

			medianBlur(frame, frame, 5);
			cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
			split(frame_hsv, frameSplit);//三通道图像分离
										 //显示HSV 3通道
										 //imshow("WIN_H", frameSplit[0]);
										 //imshow("WIN_S", frameSplit[1]);
										 //imshow("WIN_V", frameSplit[2]);
										 ///
			Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
			Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
			// 对HSV空间进行量化,得到2值图像,亮的部分为手的形状
			inRange(frame_hsv, Scalar(0, 50, 30), Scalar(20, 100, 256), dstTemp1);
			inRange(frame_hsv, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
			bitwise_or(dstTemp1, dstTemp2, mask);//按位或
			//imshow("mask", mask);	
			// 形态学操作,去除噪声,并使手的边界更加清晰,提取边缘
			Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));//返回指定形状和尺寸的结构元素,3*3的矩阵
			erode(mask, mask, element);//腐蚀
			morphologyEx(mask, mask, MORPH_OPEN, element);
			dilate(mask, mask, element);//膨胀
			morphologyEx(mask, mask, MORPH_CLOSE, element);
			//frame.copyTo(dst, mask);
			frame.copyTo(show_img, mask);

轮廓提取、重心计算

			//寻找最外层轮廓  //只保存拐点信息
			findContours(mask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
			// 去除伪轮廓
			for (size_t i = 0; i < contours.size(); i++) {
				if (fabs(contourArea(Mat(contours[i]))) > 30000) {//判断手进入区域的阈值
					filterContours.push_back(contours[i]);
				}
			}
			// 画轮廓
			drawContours(show_img, filterContours, -1, Scalar(0, 0, 255), 3/*, 8, hierarchy*/);
			// 得到轮廓的凸包络
			for (size_t j = 0; j<filterContours.size(); j++) {
				convexHull(Mat(filterContours[j]), hull, true);
				int hullcount = (int)hull.size();

				for (int i = 0; i<hullcount - 1; i++) {
					line(show_img, hull[i + 1], hull[i], Scalar(255, 0, 0), 2, 0);
				}
				line(show_img, hull[hullcount - 1], hull[0], Scalar(255, 0, 0), 2, 0);
			}
			//重心//
			Moments moment = moments(mask, true);
			Point center(moment.m10 / moment.m00, moment.m01 / moment.m00);
			circle(show_img, center, 8, Scalar(0, 0, 255), CV_FILLED);//画红色实心圆

鼠标控制

			TEMP.x = center.x - LAST_CENTER.x;
			TEMP.y = center.y - LAST_CENTER.y;
			LAST_CENTER.x = center.x;
			LAST_CENTER.y = center.y;
			
			///
			
			if (hand_mode[0] == 1 && hand_mode[1] == 1){
				mouse_event(MOUSEEVENTF_LEFTDOWN, TEMP.x, TEMP.y, 0, 0);
				mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
				cout << "leftdown" << endl;
			}
			else
			{
				mouse_event(MOUSEEVENTF_LEFTUP, TEMP.x, TEMP.y, 0, 0);
				mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
				cout << "up" << endl;
			}
				
			hand_mode[1] = hand_mode[0];

鼠标取阈值

if (mode == "test"){//阈值测试
		while (1) {
			capture >> frame;//刷新
			if (frame.empty())//帧空跳出循环
				break;
			//
			// 中值滤波,去除椒盐噪声
			medianBlur(frame, frame, 5);
			cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
			setMouseCallback("frame_hsv", on_mouse, &frame_hsv);//鼠标取阈值
			imshow("frame_hsv", frame);
			if (cvWaitKey(20) == 'q')
				break;
		}
	}
void on_mouse(int EVENT, int x, int y, int flags, void* userdata)
{
	Mat hh;
	hh = *(Mat*)userdata;
	Point p(x, y);
	switch (EVENT)
	{
	case EVENT_LBUTTONDOWN:
	{
		//cout << "h=" << hh.at<Vec3b>(p)[0] << endl;
		//cout << "s=" << hh.at<Vec3b>(p)[1] << endl;
		//cout << "v=" << hh.at<Vec3b>(p)[2] << endl;
		printf("h=%d\t", hh.at<Vec3b>(p)[0]);
		printf("s=%d\t", hh.at<Vec3b>(p)[1]);
		printf("v=%d\n", hh.at<Vec3b>(p)[2]);
		circle(hh, p, 2, Scalar(255), 3);
	}
	break;

	}
}


整体程序

/*
*日期:12月15日
*功能:手势识别
*原理简述:预处理->HSV三通道阈值分割图像->提取轮廓->计算重心点->重心每帧偏差用来控制鼠标移动

*/
#include "opencv2/opencv.hpp"  
#include <windows.h>

using namespace cv;
using namespace std;


POINT TEMP;
POINT LAST_CENTER;

vector<Rect> open_palms;
vector<Rect> closed_palms;

void skinExtract(const Mat &frame, Mat &skinArea);
void on_mouse(int EVENT, int x, int y, int flags, void* userdata);

int main(int argc, char* argv[])
{
	Mat frame, skinArea;
	Mat frame_hsv;//HSV图像
	Mat mask(frame.rows, frame.cols, CV_8UC1);    // 2值掩膜
	Mat frameSplit[4];//分离通道
	Mat show_img;//结果图像

	vector<vector<Point> > contours;// 轮廓
	vector<Vec4i> hierarchy;//四维int向量// 轮廓的结构信息
	vector< vector<Point> > filterContours;    // 筛选后的轮廓
	vector< Point > hull;    // 凸包络的点集

	//摄像头初始化、开启
	VideoCapture capture;
	
	string mode;
	int hand_mode[3];

	capture.open(0);
	if (!capture.isOpened()){//打开失败返回-1
		cout << "No camera!\n" << endl;
		return -1;
	}
	/
	CascadeClassifier open_palm_cascade;
	CascadeClassifier closed_palm_cascade;
	open_palm_cascade.load("palm.xml");
	closed_palm_cascade.load("closed_palm.xml");
	if (open_palm_cascade.empty()) {
		cout << "Could not load open_palm configuration file! "
			"Check directory! " << endl << "Press Q to Quit!" << endl;
		while (char(waitKey(0)) != 'q') {}
		return -2;
	}
	if (closed_palm_cascade.empty()) {
		cout << "Could not load closed_palm configuration file! "
			"Check directory! " << endl << "Press Q to Quit!" << endl;
		while (char(waitKey(0)) != 'q') {}
		return -2;
	}
	// Start the open_palm and eye detection phase

	
	cin >>mode;//输入模式
	if (mode == "hsv") {
		while (1) {
			capture >> frame;//刷新
			//Mat frame = imread("1.jpg");  //工程测试
			if (frame.empty())//帧空跳出循环
				break;
			/
			//skinArea.create(frame.rows, frame.cols, CV_8UC1);//和原图像等行数/等列数的/8位无符号通/道数1
			//skinExtract(frame, skinArea);//找到皮肤
			// 中值滤波,去除椒盐噪声
			medianBlur(frame, frame, 5);
			cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
			split(frame_hsv, frameSplit);//三通道图像分离
			//显示HSV 3通道
			//imshow("WIN_H", frameSplit[0]);
			//imshow("WIN_S", frameSplit[1]);
			//imshow("WIN_V", frameSplit[2]);
			///
			Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
			Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
			// 对HSV空间进行量化,得到2值图像,亮的部分为手的形状
			inRange(frame_hsv, Scalar(0, 30, 30), Scalar(40, 170, 256), dstTemp1);
			inRange(frame_hsv, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
			bitwise_or(dstTemp1, dstTemp2, mask);//按位或
			
			//imshow("mask", mask);
			
			// 形态学操作,去除噪声,并使手的边界更加清晰,提取边缘
			Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));//返回指定形状和尺寸的结构元素,3*3的矩阵
			erode(mask, mask, element);//腐蚀
			morphologyEx(mask, mask, MORPH_OPEN, element);
			dilate(mask, mask, element);//膨胀
			morphologyEx(mask, mask, MORPH_CLOSE, element);
			//frame.copyTo(dst, mask);
			frame.copyTo(show_img, mask);

			//imshow("mask", mask);
			/
			//清空
			contours.clear();
			hierarchy.clear();
			filterContours.clear();
			/
			//寻找最外层轮廓  //只保存拐点信息
			findContours(mask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
			// 去除伪轮廓
			for (size_t i = 0; i < contours.size(); i++){
				if (fabs(contourArea(Mat(contours[i]))) > 30000){//判断手进入区域的阈值
					filterContours.push_back(contours[i]);
				}
			}
			// 画轮廓
			drawContours(show_img, filterContours, -1, Scalar(0, 0, 255), 3/*, 8, hierarchy*/);
			// 得到轮廓的凸包络
			for (size_t j = 0; j<filterContours.size(); j++){
				convexHull(Mat(filterContours[j]), hull, true);
				int hullcount = (int)hull.size();

				for (int i = 0; i<hullcount - 1; i++){
					line(show_img, hull[i + 1], hull[i], Scalar(255, 0, 0), 2, 0);
				}
				line(show_img, hull[hullcount - 1], hull[0], Scalar(255, 0, 0), 2, 0);
			}
			//重心//
			Moments moment = moments(mask, true);
			Point center(moment.m10 / moment.m00, moment.m01 / moment.m00);
			circle(show_img, center, 8, Scalar(0, 0, 255), CV_FILLED);//画红色实心圆
			
			TEMP.x = center.x - LAST_CENTER.x;
			TEMP.y = center.y - LAST_CENTER.y;
			LAST_CENTER.x = center.x;
			LAST_CENTER.y = center.y;
			///

			///
			//cout << "TEMP.x=" << TEMP.x << "\tTEMP.y=" << TEMP.y << endl;
			//mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
			setMouseCallback("frame_hsv", on_mouse, &frame_hsv);//鼠标取阈值
			imshow("frame_hsv", frame);
			imshow("result", show_img);
			show_img.release();
			if (cvWaitKey(20) == 'q')
				break;
		}
	}
	if (mode == "test"){//阈值测试
		while (1) {
			capture >> frame;//刷新
			if (frame.empty())//帧空跳出循环
				break;
			//
			// 中值滤波,去除椒盐噪声
			medianBlur(frame, frame, 5);
			cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
			setMouseCallback("frame_hsv", on_mouse, &frame_hsv);//鼠标取阈值
			imshow("frame_hsv", frame);
			if (cvWaitKey(20) == 'q')
				break;
		}
	}
	if (mode == "1") {
		while (1){
			capture >> frame;//刷新
			if (frame.empty())//帧空跳出循环
				break;
			open_palm_cascade.detectMultiScale(frame, open_palms, 1.3, 4, 0, Size(50, 50));
			closed_palm_cascade.detectMultiScale(frame, closed_palms, 1.3, 4, 0, Size(50, 50));

			if (closed_palms.size() != 0) {

				for (int i = 0; i < closed_palms.size(); i++) {

					//cout << "=============Detected a closed_palm!=============" << endl;

					// Top left and bottom right points of rectangle.
					Point closed_palm_rect_p1(closed_palms[i].x, closed_palms[i].y);
					Point closed_palm_rect_p2(closed_palms[i].x + closed_palms[i].width, closed_palms[i].y + closed_palms[i].height);

					// Draw the rectangle in the image.
					rectangle(frame, closed_palm_rect_p1, closed_palm_rect_p2, Scalar(0, 255, 0));
					putText(frame, "Closed Palm", closed_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
						1, Scalar(0, 255, 0), 1, 5, false);
					hand_mode[0] = 1;
				}
			}
			for (int i = 0; i < open_palms.size(); i++) {

				//cout << "=============Detected an open_palm!=============" << endl;

				// Top left and bottom right points of rectangle.
				Point open_palm_rect_p1(open_palms[i].x, open_palms[i].y);
				Point open_palm_rect_p2(open_palms[i].x + open_palms[i].width, open_palms[i].y + open_palms[i].height);

				// Draw the rectangle in the image.
				rectangle(frame, open_palm_rect_p1, open_palm_rect_p2, Scalar(255, 0, 0));
				putText(frame, "Open Palm", open_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
					1, Scalar(255, 0, 0), 1, 5, false);
				hand_mode[0] = 2;
			}
			
			imshow("fram", frame);
			/
			//skinArea.create(frame.rows, frame.cols, CV_8UC1);//和原图像等行数/等列数的/8位无符号通/道数1
			//skinExtract(frame, skinArea);//找到皮肤
			// 中值滤波,去除椒盐噪声
			medianBlur(frame, frame, 5);
			cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
			split(frame_hsv, frameSplit);//三通道图像分离
										 //显示HSV 3通道
										 //imshow("WIN_H", frameSplit[0]);
										 //imshow("WIN_S", frameSplit[1]);
										 //imshow("WIN_V", frameSplit[2]);
										 ///
			Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
			Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
			// 对HSV空间进行量化,得到2值图像,亮的部分为手的形状
			inRange(frame_hsv, Scalar(0, 50, 30), Scalar(20, 100, 256), dstTemp1);
			inRange(frame_hsv, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
			bitwise_or(dstTemp1, dstTemp2, mask);//按位或

												 //imshow("mask", mask);

												 // 形态学操作,去除噪声,并使手的边界更加清晰,提取边缘
			Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));//返回指定形状和尺寸的结构元素,3*3的矩阵
			erode(mask, mask, element);//腐蚀
			morphologyEx(mask, mask, MORPH_OPEN, element);
			dilate(mask, mask, element);//膨胀
			morphologyEx(mask, mask, MORPH_CLOSE, element);
			//frame.copyTo(dst, mask);
			frame.copyTo(show_img, mask);

			//imshow("mask", mask);
			/
			//清空
			contours.clear();
			hierarchy.clear();
			filterContours.clear();
			/
			//寻找最外层轮廓  //只保存拐点信息
			findContours(mask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
			// 去除伪轮廓
			for (size_t i = 0; i < contours.size(); i++) {
				if (fabs(contourArea(Mat(contours[i]))) > 30000) {//判断手进入区域的阈值
					filterContours.push_back(contours[i]);
				}
			}
			// 画轮廓
			drawContours(show_img, filterContours, -1, Scalar(0, 0, 255), 3/*, 8, hierarchy*/);
			// 得到轮廓的凸包络
			for (size_t j = 0; j<filterContours.size(); j++) {
				convexHull(Mat(filterContours[j]), hull, true);
				int hullcount = (int)hull.size();

				for (int i = 0; i<hullcount - 1; i++) {
					line(show_img, hull[i + 1], hull[i], Scalar(255, 0, 0), 2, 0);
				}
				line(show_img, hull[hullcount - 1], hull[0], Scalar(255, 0, 0), 2, 0);
			}
			//重心//
			Moments moment = moments(mask, true);
			Point center(moment.m10 / moment.m00, moment.m01 / moment.m00);
			circle(show_img, center, 8, Scalar(0, 0, 255), CV_FILLED);//画红色实心圆
		    
			TEMP.x = center.x - LAST_CENTER.x;
			TEMP.y = center.y - LAST_CENTER.y;
			LAST_CENTER.x = center.x;
			LAST_CENTER.y = center.y;
			
			///
			
			if (hand_mode[0] == 1 && hand_mode[1] == 1){
				mouse_event(MOUSEEVENTF_LEFTDOWN, TEMP.x, TEMP.y, 0, 0);
				mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
				cout << "leftdown" << endl;
			}
			else
			{
				mouse_event(MOUSEEVENTF_LEFTUP, TEMP.x, TEMP.y, 0, 0);
				mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
				cout << "up" << endl;
			}
				
			hand_mode[1] = hand_mode[0];
			///
			//cout << "TEMP.x=" << TEMP.x << "\tTEMP.y=" << TEMP.y << endl;

			imshow("result", show_img);
			show_img.release();
			if (cvWaitKey(20) == 'q')
				break;

		}
	}
	
	return 0;
}
//肤色提取,skinArea为二值化肤色图像  
void skinExtract(const Mat &frame, Mat &skinArea)
{
	Mat YCbCr;
	vector<Mat> planes;

	//转换为YCrCb颜色空间  
	cvtColor(frame, YCbCr, CV_RGB2YCrCb);
	//将多通道图像分离为多个单通道图像  
	split(YCbCr, planes);

	//运用迭代器访问矩阵元素  
	MatIterator_<uchar> it_Cb = planes[1].begin<uchar>(), it_Cb_end = planes[1].end<uchar>();
	MatIterator_<uchar> it_Cr = planes[2].begin<uchar>();
	MatIterator_<uchar> it_skin = skinArea.begin<uchar>();

	//人的皮肤颜色在YCbCr色度空间的分布范围:100<=Cb<=127, 138<=Cr<=170  
	for (; it_Cb != it_Cb_end; ++it_Cr, ++it_Cb, ++it_skin)
	{
		if (138 <= *it_Cr &&  *it_Cr <= 170 && 100 <= *it_Cb &&  *it_Cb <= 127)
			*it_skin = 255;
		else
			*it_skin = 0;
	}

	//膨胀和腐蚀,膨胀可以填补凹洞(将裂缝桥接),腐蚀可以消除细的凸起(“斑点”噪声)  
	dilate(skinArea, skinArea, Mat(5, 5, CV_8UC1), Point(-1, -1));
	erode(skinArea, skinArea, Mat(5, 5, CV_8UC1), Point(-1, -1));
}


void on_mouse(int EVENT, int x, int y, int flags, void* userdata)
{
	Mat hh;
	hh = *(Mat*)userdata;
	Point p(x, y);
	switch (EVENT)
	{
	case EVENT_LBUTTONDOWN:
	{
		//cout << "h=" << hh.at<Vec3b>(p)[0] << endl;
		//cout << "s=" << hh.at<Vec3b>(p)[1] << endl;
		//cout << "v=" << hh.at<Vec3b>(p)[2] << endl;
		printf("h=%d\t", hh.at<Vec3b>(p)[0]);
		printf("s=%d\t", hh.at<Vec3b>(p)[1]);
		printf("v=%d\n", hh.at<Vec3b>(p)[2]);
		circle(hh, p, 2, Scalar(255), 3);
	}
	break;

	}
}

在这里插入图片描述

  • 14
    点赞
  • 1
    评论
  • 35
    收藏
  • 一键三连
    一键三连
  • 扫一扫,分享海报

©️2021 CSDN 皮肤主题: 大白 设计师:CSDN官方博客 返回首页
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、C币套餐、付费专栏及课程。

余额充值