车牌识别的属于常见的 模式识别 ,其基本流程为下面三个步骤:
1) 分割: 检测并检测图像中感兴趣区域;
2)特征提取: 对字符图像集中的每个部分进行提取;
3)分类: 判断图像快是不是车牌或者 每个车牌字符的分类。
车牌识别分为两个步骤, 车牌检测, 车牌识别, 都属于模式识别。
基本结构如下:
一、车牌检测
1、车牌局部化(分割车牌区域),根据尺寸等基本信息去除非车牌图像;
2、判断车牌是否存在 (训练支持向量机 -svm, 判断车牌是否存在)。
二、车牌识别
1、字符局部化(分割字符),根据尺寸等信息剔除不合格图像
2、字符识别 ( knn 分类)
1.1 车牌局部化、并剔除不合格区域
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
vector<Plate> DetectRegions::segment(Mat input) { vector<Plate> output; //转为灰度图,并去噪 Mat img_gray; cvtColor(input, img_gray, CV_BGR2GRAY); blur(img_gray, img_gray, Size(5, 5)); //找垂直边 Mat img_sobel; Sobel(img_gray, img_sobel, CV_8U, 1, 0, 3, 1, 0, BORDER_DEFAULT); // 阈值化过滤像素 Mat img_threshold; threshold(img_sobel, img_threshold, 0, 255, CV_THRESH_OTSU + CV_THRESH_BINARY); // 开运算 Mat element = getStructuringElement(MORPH_RECT, Size(17, 3)); morphologyEx(img_threshold, img_threshold, CV_MOP_CLOSE, element); //查找轮廓 vector<vector<Point>> contours; findContours(img_threshold, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE); vector<vector<Point>>::iterator itc = contours.begin(); vector<RotatedRect> rects; // 去除面积以及宽高比不合适区域 while (itc != contours.end()) { // create bounding rect of object RotatedRect mr = minAreaRect(Mat(*itc)); if (!verifySizes(mr)) { itc = contours.erase(itc); } else { ++itc; rects.push_back(mr); } } // 绘出获取区域 cv::Mat result; input.copyTo(result); cv::drawContours(result, contours, -1, cv::Scalar(255, 0, 0), 1); for (int i = 0; i < rects.size(); i++) { //For better rect cropping for each posible box //Make floodfill algorithm because the plate has white background //And then we can retrieve more clearly the contour box circle(result, rects[i].center, 3, Scalar(0, 255, 0), -1); //get the min size between width and height float minSize = (rects[i].size.width < rects[i].size.height) ? rects[i].size.width : rects[i].size.height; minSize = minSize - minSize * 0.5; //initialize rand and get 5 points around center for floodfill algorithm srand(time(NULL)); //Initialize floodfill parameters and variables Mat mask; mask.create(input.rows + 2, input.cols + 2, CV_8UC1); mask = Scalar::all(0); int loDiff = 30; int upDiff = 30; int connectivity = 4; int newMaskVal = 255; int NumSeeds = 10; Rect ccomp; int flags = connectivity + (newMaskVal << 8) + CV_FLOODFILL_FIXED_RANGE + CV_FLOODFILL_MASK_ONLY; for (int j = 0; j < NumSeeds; j++) { Point seed; seed.x = rects[i].center.x + rand() % (int)minSize - (minSize / 2); seed.y = rects[i].center.y + rand() % (int)minSize - (minSize / 2); circle(result, seed, 1, Scalar(0, 255, 255), -1); int area = floodFill(input, mask, seed, Scalar(255, 0, 0), &ccomp, Scalar(loDiff, loDiff, loDiff), Scalar(upDiff, upDiff, upDiff), flags); } if (showSteps) imshow("MASK", mask); //cvWaitKey(0); //Check new floodfill mask match for a correct patch. //Get all points detected for get Minimal rotated Rect vector<Point> pointsInterest; Mat_<uchar>::iterator itMask = mask.begin<uchar>(); Mat_<uchar>::iterator end = mask.end<uchar>(); for (; itMask != end; ++itMask) if (*itMask == 255) pointsInterest.push_back(itMask.pos()); RotatedRect minRect = minAreaRect(pointsInterest); if (verifySizes(minRect)) { // rotated rectangle drawing Point2f rect_points[4]; minRect.points(rect_points); for (int j = 0; j < 4; j++) line(result, rect_points[j], rect_points[(j + 1) % 4], Scalar(0, 0, 255), 1, 8); // 获取旋转矩阵 float r = (float)minRect.size.width / (float)minRect.size.height; float angle = minRect.angle; if (r < 1) angle = 90 + angle; Mat rotmat = getRotationMatrix2D(minRect.center, angle, 1); // 获取映射图像 Mat img_rotated; warpAffine(input, img_rotated, rotmat, input.size(), CV_INTER_CUBIC); // Crop image Size rect_size = minRect.size; if (r < 1) swap(rect_size.width, rect_size.height); Mat img_crop; getRectSubPix(img_rotated, rect_size, minRect.center, img_crop); Mat resultResized; resultResized.create(33, 144, CV_8UC3); resize(img_crop, resultResized, resultResized.size(), 0, 0, INTER_CUBIC); // 直方图 Mat grayResult; cvtColor(resultResized, grayResult, CV_BGR2GRAY); blur(grayResult, grayResult, Size(3, 3)); grayResult = histeq(grayResult); output.push_back(Plate(grayResult, minRect.boundingRect())); } } return output; }
1.2 判断车牌是否存在
1.2.1 训练 svm
svm 会创建一个或多个超平面, 这些超级平面能判断数据属于那个类。
训练数据: 所有训练数据存储再一个 N x M 的矩阵中, 其中 N 为样本数, M 为特征数(每个样本是该训练矩阵中的一行)。这些数据 所有数据存在 xml 文件中,
标签数据: 每个样本的类别信息存储在另一个 N x 1 的矩阵中, 每行为一个样本标签。
训练数据存放在本地 svm.xml 文件中。
// TrainSvm.cpp 文件
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
#include <iostream> #include <opencv2/opencv.hpp> #include "Preprocess.h" using namespace std; using namespace cv; using namespace cv::ml; int main(int argc, char** argv) { FileStorage fs; fs.open("SVM.xml", FileStorage::READ); Mat SVM_TrainingData; Mat SVM_Classes; fs["TrainingData"] >> SVM_TrainingData; fs["classes"] >> SVM_Classes; // Set SVM storage Ptr<ml::SVM> model = ml::SVM::create(); model->setType(SVM::C_SVC); model->setKernel(SVM::LINEAR); // 核函数 // 训练数据 Ptr<TrainData> tData = TrainData::create(SVM_TrainingData, ROW_SAMPLE, SVM_Classes); // 训练分类器 model->train(tData); model->save("model.xml"); // TODO: 测试 return 0;
// Preprocess.cpp
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
#include <string> #include <vector> #include <fstream> #include <algorithm> #include "Preprocess.h" using namespace cv; void Preprocess::getAllFiles(string path, vector<string> &files, string fileType) { long hFile = 0; struct _finddata_t fileInfo; string p; if ((hFile = _findfirst(p.assign(path).append("\\*" + fileType).c_str(), &fileInfo)) != -1) { do { files.push_back(p.assign(path).append("\\").append(fileInfo.name)); } while (_findnext(hFile, &fileInfo) == 0); _findclose(hFile); // 关闭句柄 } } void Preprocess::extract_img_data(string path_plates, string path_noPlates) { cout << "OpenCV Training SVM Automatic Number Plate Recognition\n"; int imgWidth = 144; int imgHeight = 33; int numPlates = 100; int numNoPlates = 100; Mat classes; Mat trainingData; Mat trainingImages; vector<int> trainingLabels; for (int i = 0; i < numPlates; i++) { stringstream ss(stringstream::in | stringstream::out); ss << path_plates << i << ".jpg"; Mat img = imread(ss.str(), 0); resize(img, img, Size(imgWidth, imgWidth)); img = img.reshape(1, 1); trainingImages.push_back(img); trainingLabels.push_back(1); } for (int i = 0; i < numNoPlates; i++) { stringstream ss; ss << path_noPlates << i << ".jpg"; Mat img = imread(ss.str(), 0); img = img.reshape(1, 1); trainingImages.push_back(img); trainingLabels.push_back(0); } Mat(trainingImages).copyTo(trainingData); trainingData.convertTo(trainingData, CV_32FC1); Mat(trainingLabels).copyTo(classes); FileStorage fs("SVM.xml", FileStorage::WRITE); fs << "TrainingData" << trainingData; fs << "classess" << classes; fs.release(); }
1.2.2 利用 svm 判断车牌是否存在
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
// load model Ptr<ml::SVM> model = SVM::load("model.xml"); // For each possible plate, classify with svm if it's plate vector<Plate> plates; for (int i = 0; i < posible_regions.size(); i++) { Mat img = posible_regions[i].plateImg; Mat p = img.reshape(1, 1); p.convertTo(p, CV_32FC1); int reponse = (int)model->predict(p); if (reponse) { plates.push_back(posible_regions[i]); //bool res = imwrite("test.jpg", img); } }
以上,已经找了存在车牌的区域,并保存到一个 vector 中。
下面使用 k 邻近算法, 来识别车牌图像中的车牌字符。
2.1 字符分割
分割字符,并剔除不合格图像
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
vector<CharSegment> OCR::segment(Plate plate) { Mat input = plate.plateImg; vector<CharSegment> output; //使字符为白色,背景为黑色 Mat img_threshold; threshold(input, img_threshold, 60, 255, CV_THRESH_BINARY_INV); Mat img_contours; img_threshold.copyTo(img_contours); // 找到所有物体 vector< vector< Point> > contours; findContours(img_contours, contours, // a vector of contours CV_RETR_EXTERNAL, // retrieve the external contours CV_CHAIN_APPROX_NONE); // all pixels of each contours // Draw blue contours on a white image cv::Mat result; img_threshold.copyTo(result); cvtColor(result, result, CV_GRAY2RGB); cv::drawContours(result, contours, -1, // draw all contours cv::Scalar(255, 0, 0), // in blue 1); // with a thickness of 1 //Remove patch that are no inside limits of aspect ratio and area. vector<vector<Point> >::iterator itc = contours.begin(); while (itc != contours.end()) { //Create bounding rect of object Rect mr = boundingRect(Mat(*itc)); rectangle(result, mr, Scalar(0, 255, 0)); //提取合格图像区域 Mat auxRoi(img_threshold, mr); if (verifySizes(auxRoi)) { auxRoi = preprocessChar(auxRoi); output.push_back(CharSegment(auxRoi, mr)); rectangle(result, mr, Scalar(0, 125, 255)); } ++itc; } return output; } Mat OCR::preprocessChar(Mat in) { //Remap image int h = in.rows; int w = in.cols; Mat transformMat = Mat::eye(2, 3, CV_32F); int m = max(w, h); transformMat.at<float>(0, 2) = m / 2 - w / 2; transformMat.at<float>(1, 2) = m / 2 - h / 2; // 仿射变换,将图像投射到尺寸更大的图像上(使用偏移) Mat warpImage(m, m, in.type()); warpAffine(in, warpImage, transformMat, warpImage.size(), INTER_LINEAR, BORDER_CONSTANT, Scalar(0)); Mat out; resize(warpImage, out, Size(charSize, charSize)); return out; }
2.2 字符识别
2.2.1 训练 knn
使用 opencv 自带的 digits.png 文件, 可以训练训练识别识别数字的 knn 。
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
#include <iostream> #include <opencv2/opencv.hpp> using namespace cv; using namespace std; using namespace cv::ml; const int numFilesChars[] = { 35, 40, 42, 41, 42, 33, 30, 31, 49, 44, 30, 24, 21, 20, 34, 9, 10, 3, 11, 3, 15, 4, 9, 12, 10, 21, 18, 8, 15, 7 }; int main() { std::cout << "OpenCV Training OCR Automatic Number Plate Recognition\n"; string path = "D:/Program Files (x86)/opencv_3.4.3/opencv/sources/samples/data/digits.png"; Mat img = imread(path); Mat gray; cvtColor(img, gray, CV_BGR2GRAY); int b = 20; int m = gray.rows / b; // 将原图裁剪为 20 * 20 的小图块 int n = gray.cols / b; // 将原图裁剪为 20 * 20 的小图块 Mat data, labels; // 特征矩阵 // 按照列来读取数据, 每 5 个数据为一个类 for (int i = 0; i < n; i++) { int offsetCol = i * b; // 列上的偏移量 for (int j = 0; j < m; j++) { int offsetRow = j * b; // 行上的偏移量 Mat tmp; gray(Range(offsetRow, offsetRow + b), Range(offsetCol, offsetCol + b)).copyTo(tmp); data.push_back(tmp.reshape(0, 1)); // 序列化后放入特征矩阵 labels.push_back((int)j / 5); // 对应的标注 } } data.convertTo(data, CV_32F); int samplesNum = data.rows; int trainNum = 3000; Mat trainData, trainLabels; trainData = data(Range(0, trainNum), Range::all()); // 前 3000 个为训练数据 trainLabels = labels(Range(0, trainNum), Range::all()); // 使用k 邻近算法那(knn, k-nearest_neighbor) 算法 int K = 5; Ptr<cv::ml::TrainData> tData = cv::ml::TrainData::create(trainData, ROW_SAMPLE, trainLabels); Ptr<KNearest> model = KNearest::create(); model->setDefaultK(K); // 设定查找时返回数量为 5 // 设置分类器为分类 或回归 // 分类问题:输出离散型变量(如 -1,1, 100), 为定性输出(如预测明天是下雨、天晴还是多云) // 回归问题: 回归问题的输出为连续型变量,为定量输出(如明天温度为多少度) model->setIsClassifier(true); model->train(tData); // 预测分类 double train_hr = 0, test_hr = 0; Mat response; // compute prediction error on train and test data for (int i = 0; i < samplesNum; i++) { Mat smaple = data.row(i); float r = model->predict(smaple); // 对所有进行预测 // 预测结果与原结果对比,相等为 1, 不等为 0 r = std::abs(r - labels.at<int>(i)) <= FLT_EPSILON ? 1.f : 0.f; if (i < trainNum) { train_hr += r; // 累计正确数 } else { test_hr += r; } } test_hr /= samplesNum - trainNum; train_hr = trainNum > 0 ? train_hr / trainNum : 1.; cout << "train accuracy : " << train_hr * 100. << "\n"; cout << "test accuracy : " << test_hr * 100. << "\n"; // 保存 ocr 模型 string model_path = "ocr.xml"; model->save(model_path); // 载入模型 // Ptr<KNearest> knn = KNearest::load<KNearest>(model_path); waitKey(1); return 0; }
2.2.2 使用 knn 识别字符
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
// Mat target_img 为目标图像矩阵 model->save(model_path); // 载入模型 Ptr<KNearest> knn = KNearest::load<KNearest>(model_path); float it_type = knn->predict(target_img)
以上就是车牌识别的核心代码了。
全部流程的代码我放到下面这个群里面了,欢迎来交流下载。
广州 OpenCV 学校交流群: 892083812
参考:
深入理解 OpenCV
https://www.cnblogs.com/denny402/p/5032839.html