车牌识别的属于常见的 模式识别 ,其基本流程为下面三个步骤:
(1)分割: 检测并检测图像中感兴趣区域;
(2)特征提取: 对字符图像集中的每个部分进行提取;
(3)分类: 判断图像快是不是车牌或者 每个车牌字符的分类。
车牌识别分为两个步骤, 车牌检测, 车牌识别, 都属于模式识别。
基本结构
一、车牌检测
1、车牌局部化(分割车牌区域),根据尺寸等基本信息去除非车牌图像;
2、判断车牌是否存在 (训练支持向量机 -svm, 判断车牌是否存在)。
二、车牌识别
1、字符局部化(分割字符),根据尺寸等信息剔除不合格图像
2、字符识别 ( knn 分类)
一、车牌检测
1.1 车牌局部化、并剔除不合格区域
vector<Plate> DetectRegions::segment(Mat input) {
vector<Plate> output;
//转为灰度图,并去噪
Mat img_gray;
cvtColor(input, img_gray, CV_BGR2GRAY);
blur(img_gray, img_gray, Size(5, 5));
//找垂直边
Mat img_sobel;
Sobel(img_gray, img_sobel, CV_8U, 1, 0, 3, 1, 0, BORDER_DEFAULT);
// 阈值化过滤像素
Mat img_threshold;
threshold(img_sobel, img_threshold, 0, 255, CV_THRESH_OTSU + CV_THRESH_BINARY);
// 开运算
Mat element = getStructuringElement(MORPH_RECT, Size(17, 3));
morphologyEx(img_threshold, img_threshold, CV_MOP_CLOSE, element);
//查找轮廓
vector<vector<Point>> contours;
findContours(img_threshold, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
vector<vector<Point>>::iterator itc = contours.begin();
vector<RotatedRect> rects;
// 去除面积以及宽高比不合适区域
while (itc != contours.end())
{
// create bounding rect of object
RotatedRect mr = minAreaRect(Mat(*itc));
if (!verifySizes(mr))
{
itc = contours.erase(itc);
}
else
{
++itc;
rects.push_back(mr);
}
}
// 绘出获取区域
cv::Mat result;
input.copyTo(result);
cv::drawContours(result, contours, -1, cv::Scalar(255, 0, 0), 1);
for (int i = 0; i < rects.size(); i++) {
//For better rect cropping for each posible box
//Make floodfill algorithm because the plate has white background
//And then we can retrieve more clearly the contour box
circle(result, rects[i].center, 3, Scalar(0, 255, 0), -1);
//get the min size between width and height
float minSize = (rects[i].size.width < rects[i].size.height) ? rects[i].size.width : rects[i].size.height;
minSize = minSize - minSize * 0.5;
//initialize rand and get 5 points around center for floodfill algorithm
srand(time(NULL));
//Initialize floodfill parameters and variables
Mat mask;
mask.create(input.rows + 2, input.cols + 2, CV_8UC1);
mask = Scalar::all(0);
int loDiff = 30;
int upDiff = 30;
int connectivity = 4;
int newMaskVal = 255;
int NumSeeds = 10;
Rect ccomp;
int flags = connectivity + (newMaskVal << 8) + CV_FLOODFILL_FIXED_RANGE + CV_FLOODFILL_MASK_ONLY;
for (int j = 0; j < NumSeeds; j++) {
Point seed;
seed.x = rects[i].center.x + rand() % (int)minSize - (minSize / 2);
seed.y = rects[i].center.y + rand() % (int)minSize - (minSize / 2);
circle(result, seed, 1, Scalar(0, 255, 255), -1);
int area = floodFill(input, mask, seed, Scalar(255, 0, 0), &ccomp, Scalar(loDiff, loDiff, loDiff), Scalar(upDiff, upDiff, upDiff), flags);
}
if (showSteps)
imshow("MASK", mask);
//cvWaitKey(0);
//Check new floodfill mask match for a correct patch.
//Get all points detected for get Minimal rotated Rect
vector<Point> pointsInterest;
Mat_<uchar>::iterator itMask = mask.begin<uchar>();
Mat_<uchar>::iterator end = mask.end<uchar>();
for (; itMask != end; ++itMask)
if (*itMask == 255)
pointsInterest.push_back(itMask.pos());
RotatedRect minRect = minAreaRect(pointsInterest);
if (verifySizes(minRect)) {
// rotated rectangle drawing
Point2f rect_points[4];
minRect.points(rect_points);
for (int j = 0; j < 4; j++)
line(result, rect_points[j], rect_points[(j + 1) % 4], Scalar(0, 0, 255), 1, 8);
// 获取旋转矩阵
float r = (float)minRect.size.width / (float)minRect.size.height;
float angle = minRect.angle;
if (r < 1)
angle = 90 + angle;
Mat rotmat = getRotationMatrix2D(minRect.center, angle, 1);
// 获取映射图像
Mat img_rotated;
warpAffine(input, img_rotated, rotmat, input.size(), CV_INTER_CUBIC);
// Crop image
Size rect_size = minRect.size;
if (r < 1)
swap(rect_size.width, rect_size.height);
Mat img_crop;
getRectSubPix(img_rotated, rect_size, minRect.center, img_crop);
Mat resultResized;
resultResized.create(33, 144, CV_8UC3);
resize(img_crop, resultResized, resultResized.size(), 0, 0, INTER_CUBIC);
// 直方图
Mat grayResult;
cvtColor(resultResized, grayResult, CV_BGR2GRAY);
blur(grayResult, grayResult, Size(3, 3));
grayResult = histeq(grayResult);
output.push_back(Plate(grayResult, minRect.boundingRect()));
}
}
return output;
}
1.2 判断车牌是否存在
- 1.2.1 训练 svm
svm 会创建一个或多个超平面, 这些超级平面能判断数据属于那个类。
训练数据: 所有训练数据存储再一个 N x M 的矩阵中, 其中 N 为样本数, M 为特征数 (每个样本是该训练矩阵中的一行)。这些数据 所有数据存在 xml 文件中,
标签数据: 每个样本的类别信息存储在另一个 N x 1 的矩阵中, 每行为一个样本标签。
训练数据存放在本地 svm.xml 文件中。
#include <iostream>
#include <opencv2/opencv.hpp>
#include "Preprocess.h"
using namespace std;
using namespace cv;
using namespace cv::ml;
int main(int argc, char** argv)
{
FileStorage fs;
fs.open("SVM.xml", FileStorage::READ);
Mat SVM_TrainingData;
Mat SVM_Classes;
fs["TrainingData"] >> SVM_TrainingData;
fs["classes"] >> SVM_Classes;
// Set SVM storage
Ptr<ml::SVM> model = ml::SVM::create();
model->setType(SVM::C_SVC);
model->setKernel(SVM::LINEAR); // 核函数
// 训练数据
Ptr<TrainData> tData = TrainData::create(SVM_TrainingData, ROW_SAMPLE, SVM_Classes);
// 训练分类器
model->train(tData);
model->save("model.xml");
// TODO: 测试
return 0;
#include <string>
#include <vector>
#include <fstream>
#include <algorithm>
#include "Preprocess.h"
using namespace cv;
void Preprocess::getAllFiles(string path, vector<string> &files, string fileType)
{
long hFile = 0;
struct _finddata_t fileInfo;
string p;
if ((hFile = _findfirst(p.assign(path).append("\\*" + fileType).c_str(), &fileInfo)) != -1)
{
do
{
files.push_back(p.assign(path).append("\\").append(fileInfo.name));
} while (_findnext(hFile, &fileInfo) == 0);
_findclose(hFile); // 关闭句柄
}
}
void Preprocess::extract_img_data(string path_plates, string path_noPlates)
{
cout << "OpenCV Training SVM Automatic Number Plate Recognition\n";
int imgWidth = 144;
int imgHeight = 33;
int numPlates = 100;
int numNoPlates = 100;
Mat classes;
Mat trainingData;
Mat trainingImages;
vector<int> trainingLabels;
for (int i = 0; i < numPlates; i++)
{
stringstream ss(stringstream::in | stringstream::out);
ss << path_plates << i << ".jpg";
Mat img = imread(ss.str(), 0);
resize(img, img, Size(imgWidth, imgWidth));
img = img.reshape(1, 1);
trainingImages.push_back(img);
trainingLabels.push_back(1);
}
for (int i = 0; i < numNoPlates; i++)
{
stringstream ss;
ss << path_noPlates << i << ".jpg";
Mat img = imread(ss.str(), 0);
img = img.reshape(1, 1);
trainingImages.push_back(img);
trainingLabels.push_back(0);
}
Mat(trainingImages).copyTo(trainingData);
trainingData.convertTo(trainingData, CV_32FC1);
Mat(trainingLabels).copyTo(classes);
FileStorage fs("SVM.xml", FileStorage::WRITE);
fs << "TrainingData" << trainingData;
fs << "classess" << classes;
fs.release();
}
- 1.2.2 利用 svm 判断车牌是否存在
// load model
Ptr<ml::SVM> model = SVM::load("model.xml");
// For each possible plate, classify with svm if it's plate
vector<Plate> plates;
for (int i = 0; i < posible_regions.size(); i++)
{
Mat img = posible_regions[i].plateImg;
Mat p = img.reshape(1, 1);
p.convertTo(p, CV_32FC1);
int reponse = (int)model->predict(p);
if (reponse)
{
plates.push_back(posible_regions[i]);
//bool res = imwrite("test.jpg", img);
}
}
以上,已经找了存在车牌的区域,并保存到一个 vector 中。 下面使用 k 邻近算法, 来识别车牌图像中的车牌字符。
二、车牌识别
2.1 字符分割
分割字符,并剔除不合格图像
vector<CharSegment> OCR::segment(Plate plate) {
Mat input = plate.plateImg;
vector<CharSegment> output;
//使字符为白色,背景为黑色
Mat img_threshold;
threshold(input, img_threshold, 60, 255, CV_THRESH_BINARY_INV);
Mat img_contours;
img_threshold.copyTo(img_contours);
// 找到所有物体
vector< vector< Point> > contours;
findContours(img_contours,
contours, // a vector of contours
CV_RETR_EXTERNAL, // retrieve the external contours
CV_CHAIN_APPROX_NONE); // all pixels of each contours
// Draw blue contours on a white image
cv::Mat result;
img_threshold.copyTo(result);
cvtColor(result, result, CV_GRAY2RGB);
cv::drawContours(result, contours,
-1, // draw all contours
cv::Scalar(255, 0, 0), // in blue
1); // with a thickness of 1
//Remove patch that are no inside limits of aspect ratio and area.
vector<vector<Point> >::iterator itc = contours.begin();
while (itc != contours.end()) {
//Create bounding rect of object
Rect mr = boundingRect(Mat(*itc));
rectangle(result, mr, Scalar(0, 255, 0));
//提取合格图像区域
Mat auxRoi(img_threshold, mr);
if (verifySizes(auxRoi)) {
auxRoi = preprocessChar(auxRoi);
output.push_back(CharSegment(auxRoi, mr));
rectangle(result, mr, Scalar(0, 125, 255));
}
++itc;
}
return output;
}
Mat OCR::preprocessChar(Mat in) {
//Remap image
int h = in.rows;
int w = in.cols;
Mat transformMat = Mat::eye(2, 3, CV_32F);
int m = max(w, h);
transformMat.at<float>(0, 2) = m / 2 - w / 2;
transformMat.at<float>(1, 2) = m / 2 - h / 2;
// 仿射变换,将图像投射到尺寸更大的图像上(使用偏移)
Mat warpImage(m, m, in.type());
warpAffine(in, warpImage, transformMat, warpImage.size(), INTER_LINEAR, BORDER_CONSTANT, Scalar(0));
Mat out;
resize(warpImage, out, Size(charSize, charSize));
return out;
}
2.2 字符识别
- 2.2.1 训练 knn
使用 opencv 自带的 digits.png 文件, 可以训练训练识别识别数字的 knn 。
#include <iostream>
#include <opencv2/opencv.hpp>
using namespace cv;
using namespace std;
using namespace cv::ml;
const int numFilesChars[] = { 35, 40, 42, 41, 42, 33, 30, 31, 49, 44, 30, 24, 21, 20, 34, 9, 10, 3, 11, 3, 15, 4, 9, 12, 10, 21, 18, 8, 15, 7 };
int main()
{
std::cout << "OpenCV Training OCR Automatic Number Plate Recognition\n";
string path = "D:/Program Files (x86)/opencv_3.4.3/opencv/sources/samples/data/digits.png";
Mat img = imread(path);
Mat gray;
cvtColor(img, gray, CV_BGR2GRAY);
int b = 20;
int m = gray.rows / b; // 将原图裁剪为 20 * 20 的小图块
int n = gray.cols / b; // 将原图裁剪为 20 * 20 的小图块
Mat data, labels; // 特征矩阵
// 按照列来读取数据, 每 5 个数据为一个类
for (int i = 0; i < n; i++)
{
int offsetCol = i * b; // 列上的偏移量
for (int j = 0; j < m; j++)
{
int offsetRow = j * b; // 行上的偏移量
Mat tmp;
gray(Range(offsetRow, offsetRow + b), Range(offsetCol, offsetCol + b)).copyTo(tmp);
data.push_back(tmp.reshape(0, 1)); // 序列化后放入特征矩阵
labels.push_back((int)j / 5); // 对应的标注
}
}
data.convertTo(data, CV_32F);
int samplesNum = data.rows;
int trainNum = 3000;
Mat trainData, trainLabels;
trainData = data(Range(0, trainNum), Range::all()); // 前 3000 个为训练数据
trainLabels = labels(Range(0, trainNum), Range::all());
// 使用k 邻近算法那(knn, k-nearest_neighbor) 算法
int K = 5;
Ptr<cv::ml::TrainData> tData = cv::ml::TrainData::create(trainData, ROW_SAMPLE, trainLabels);
Ptr<KNearest> model = KNearest::create();
model->setDefaultK(K); // 设定查找时返回数量为 5
// 设置分类器为分类 或回归
// 分类问题:输出离散型变量(如 -1,1, 100), 为定性输出(如预测明天是下雨、天晴还是多云)
// 回归问题: 回归问题的输出为连续型变量,为定量输出(如明天温度为多少度)
model->setIsClassifier(true);
model->train(tData);
// 预测分类
double train_hr = 0, test_hr = 0;
Mat response;
// compute prediction error on train and test data
for (int i = 0; i < samplesNum; i++)
{
Mat smaple = data.row(i);
float r = model->predict(smaple); // 对所有进行预测
// 预测结果与原结果对比,相等为 1, 不等为 0
r = std::abs(r - labels.at<int>(i)) <= FLT_EPSILON ? 1.f : 0.f;
if (i < trainNum)
{
train_hr += r; // 累计正确数
}
else
{
test_hr += r;
}
}
test_hr /= samplesNum - trainNum;
train_hr = trainNum > 0 ? train_hr / trainNum : 1.;
cout << "train accuracy : " << train_hr * 100. << "\n";
cout << "test accuracy : " << test_hr * 100. << "\n";
// 保存 ocr 模型
string model_path = "ocr.xml";
model->save(model_path);
// 载入模型
// Ptr<KNearest> knn = KNearest::load<KNearest>(model_path);
waitKey(1);
return 0;
}
- 2.2.2 使用 knn 识别字符
// Mat target_img 为目标图像矩阵
model->save(model_path);
// 载入模型
Ptr<KNearest> knn = KNearest::load<KNearest>(model_path);
float it_type = knn->predict(target_img)
参考
深入理解 OpenCV
https://www.cnblogs.com/denny402/p/5032839.html