C++opencv+Advancedeast文本位置检测

Advancedeast项目地址:https://github.com/huoyijie/AdvancedEAST
环境:VS2017+opencv4.1.2

运行结果如下:

基本步骤:

首先介绍一下与AdvancedEAST的使用相关的一些原理.AdvancedEAST的网络结构如下图:


图片输入网络后依次输出三种数据,简单使用的话可以只用第一个,我只用了第一个.
从网络结构也可以看出来有三个输出网络,第一个是score,可以通过score的值判断图片上的点是否属于文本的区域,一般大于10是文本的像素点.score相当于是否属于文本的置信度.
第二个输出网络是code,数据的第一位表示该像素是否属于边界像素,第二个判断是头部的像素还是尾部的像素.
第三个输出的数据输出的4位geo，是边界像素可以预测的2个顶点坐标,原文是这么说,不过我没有搞清楚是做什么的,希望有大佬可以告知.
使用opencv的readnet函数读取Advancedeast的模型,我用的是现成的模型,如果需要自己训练模型,可以参见原项目.

网络的输出是长和宽为原图像的四分之一的矩阵,假设我们读入的图片为640640,则score为一个160160的矩阵,矩阵中i行j列的score,代表原图中4i行,4j列元素及其临近区域是否是文本区域.
具体代码如下:

#include "pch.h"
#include <iostream>
#include<opencv2/opencv.hpp>
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"

using namespace std;
using namespace cv;
using namespace cv::dnn;

//解码
void decode(const Mat &scores, const Mat &geometry);

Mat text_detect(Mat srcImg, int inpWidth, int inpHeight, float confThreshold, float nmsThreshold, Net net)
{
	//输出
	std::vector<Mat> output;
	std::vector<String> outputLayers(4);
	outputLayers[0] = "side_vertex_coord/convolution";
	outputLayers[1] = "side_vertex_code/convolution";
	outputLayers[2] = "inside_score/convolution";
	outputLayers[3] = "east_detect/concat";


	//检测图像
	Mat frame, blob;
	frame = srcImg.clone();
	//获取深度学习模型的输入
	blobFromImage(frame, blob, 1.0, Size(inpWidth, inpHeight), Scalar(123.68, 116.78, 103.94), true, false);
	net.setInput(blob);
	//输出结果
	net.forward(output, outputLayers);

	//置信度
	Mat scores = output[3];
	//位置参数
	Mat geometry = output[0];

	// Decode predicted bounding boxes， 对检测框进行解码，获取文本框位置方向
	//文本框位置参数
	decode(scores, geometry);



	return frame;
}

//模型地址
auto model = "K:\\UC下载\\east_model_3T256.pb";
//auto model = "./model/frozen_east_text_detection.pb";
//检测图像
auto detect_image = "J:\\MFCApplication1 - 副本\\Testimage\\001019080900028_0113_1.jpg";
//输入框尺寸
auto inpWidth = 640;
auto inpHeight = 640;
//置信度阈值
auto confThreshold = 0.5;
//非极大值抑制算法阈值

auto nmsThreshold = 0.1;
Mat srcImg;
Mat heibai(160, 160, CV_8UC1);

//轮廓按照面积大小升序排序
bool ascendSort(vector<Point> a, vector<Point> b) {
	return a.size() < b.size();

}

//轮廓按照面积大小降序排序
bool descendSort(vector<Point> a, vector<Point> b) {

	return a.size() > b.size();
}

int main()
{
	//读取模型
	Net net = readNet(model);
	//读取检测图像
	//vector<String> layer_names = net.getLayerNames();
	//for (int i = 0; i < layer_names.size(); i++) {
	//	int id = net.getLayerId(layer_names[i]);
	//	auto layer = net.getLayer(id);
	//	printf("layer id:%d, type: %s, name:%s \n", id, layer->type.c_str(), layer->name.c_str());
	//}

	//system("pause");   // 用双引号，不要用单引号
	srcImg = imread(detect_image);
	int or_h = srcImg.rows;
	int or_w = srcImg.cols;
	resize(srcImg, srcImg, Size(640, 640));
	if (!srcImg.empty())
	{
		cout << "read image success!" << endl;
	}
	Mat resultImg = text_detect(srcImg, inpWidth, inpHeight, confThreshold, nmsThreshold, net);
	srcImg = imread(detect_image);
	//Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));
	//dilate(heibai, heibai, element);

	vector< vector< Point> > contours;  //用于保存所有轮廓信息
	
	vector<Point> tempV;				//暂存的轮廓
	findContours(heibai, contours, RETR_EXTERNAL, CHAIN_APPROX_NONE);
	//轮廓按照面积大小进行升序排序
	sort(contours.begin(), contours.end(), descendSort);//升序排序
	vector<vector<Point> >::iterator itc = contours.begin();
	/*Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));
	erode(heibai, heibai, element);*/
	while (itc != contours.end())
	{
		int y = itc->size();
		if (itc->size() < 29)
		{
		itc = contours.erase(itc);
		}

		else
		{
			++itc;
		}

	}

	//draw


	Mat B;
	heibai.copyTo(B);
	drawContours(B, contours, -1, Scalar(0, 0, 0), FILLED);
	heibai = heibai - B;
	findContours(heibai, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE, Point());
	heibai.copyTo(B);
	int i = 0;
	vector<vector<Point> >::iterator itr = contours.begin();
	int last_i = 0;
	for (int i = 0; i < contours.size(); i++)
	{
		Mat tmp(contours.at(i));
		Moments moment = moments(tmp, false);
		if (moment.m00 != 0)//除数不能为0
		{
			int x = cvRound(moment.m10 / moment.m00);//计算重心横坐标
			int y = cvRound(moment.m01 / moment.m00);//计算重心纵坐标
			if (x < 30 || y < 30||x>120||y>120)
			{
				vector< vector< Point> > contours2; //用于保存面积不足100的轮廓
				for (int j = last_i; j < i; j++)
				{
					++itr;
				}
				last_i = i;
				contours2.push_back(*itr);


				drawContours(heibai, contours2, -1, Scalar(0, 0, 0), FILLED);
			}
		}

	}

	//heibai =  B- heibai;
    Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));
	dilate(heibai, heibai, element);
	findContours(heibai, contours, RETR_EXTERNAL, CHAIN_APPROX_NONE);
	Mat tmp1(contours.at(0));
	RotatedRect rect = minAreaRect(tmp1);
	Point2f fourPoint2f[4];
	rect.points(fourPoint2f);
	float ratio_h = (float)or_h / 640;
	float ratio_w = (float)or_w / 640;

	fourPoint2f[0].x = fourPoint2f[0].x * 4 * ratio_w;
	fourPoint2f[0].y = fourPoint2f[0].y * 4 * ratio_h;
	fourPoint2f[1].x = fourPoint2f[1].x * 4 * ratio_w;
	fourPoint2f[1].y = fourPoint2f[1].y * 4 * ratio_h;
	fourPoint2f[2].x = fourPoint2f[2].x * 4 * ratio_w;
	fourPoint2f[2].y = fourPoint2f[2].y * 4 * ratio_h;
	fourPoint2f[3].x = fourPoint2f[3].x * 4 * ratio_w;
	fourPoint2f[3].y = fourPoint2f[3].y * 4 * ratio_h;
	line(srcImg, fourPoint2f[0], fourPoint2f[1], Scalar(0, 255, 0), 2);
	line(srcImg, fourPoint2f[1], fourPoint2f[2], Scalar(0, 255, 0), 2);
	line(srcImg, fourPoint2f[2], fourPoint2f[3], Scalar(0, 255, 0), 2);
	line(srcImg, fourPoint2f[3], fourPoint2f[0], Scalar(0, 255, 0), 2);
	imshow("result", srcImg);
	waitKey();
	return 0;
}


void decode(const Mat &scores, const Mat &geometry)
{
	const int height = geometry.size[2];
	const int width = geometry.size[3];


	for (int y = 0; y < height; y++)
	{
		//识别概率
		const float *isinside= scores.ptr<float>(0, 0, y);
		const float *isbound = scores.ptr<float>(0, 1, y);
		const float *headorwei = scores.ptr<float>(0, 2, y);
		const float *x_1 = geometry.ptr<float>(0, 0, y);
		const float *y_1 = geometry.ptr<float>(0, 1, y);
		const float *x_2 = geometry.ptr<float>(0, 2, y);
		const float *y_2 = geometry.ptr<float>(0, 3, y);
		//遍历所有检测到的检测框
		for (int x = 0; x < width; x++)
		{
			float isinside_1 = isinside[x];
			float isbound_1 = isbound[x];
			float headorwei_1 = headorwei[x];
			
			float x_11 = x_1[x];
			float y_11 = y_1[x];
			float x_22 = x_2[x];
			float y_22 = y_2[x];

			float offsetX = x * 4.0f, offsetY = y * 4.0f;

			//低于阈值忽略该检测框
			if (isinside_1 <15)
			{
				heibai.at<uchar>(y, x) = 0;
				continue;
			}
			circle(srcImg,Point(offsetX, offsetY),2, Scalar(0, 255, 0),-1);
			heibai.at<uchar>(y, x)=255;
		
		}
	}
}

所用模型:
链接：https://pan.baidu.com/s/1EVdtCnY8lwylCgCqvnrwkw
提取码：ojge