10、【图像识别】垃圾瓶自动分选项目

新建 OCR 项目

输入命令python create_directories.py -n plastic_bottle_classification

数据预处理

收集数据

直接把从百度云盘的下载的数据拿过来

数据清洗

不合格的图片已经清洗过了，剩下的都是合格的。

标注数据

图片已经标注好了，直接拿来用。

训练模型

输入命令 python one_command_train.py --steps=5000 --batch_size=12，启动训练

导出 TensorFlow 冻结图模型

输入命令 python export_inference_graph.py --inpu_type=image_tensor --pipeline_config_path=training\ssd_inception_v2_coco.config --trained_checkponit_prefix=training\model.ckpt-5000 --output_directory=trained_frozen_models\ocr_model

测试模型

优化并部署模型

需要先安装 OpenVINO 工具套件才能进行接下来的操作。
安装 OpenVINO：https://blog.csdn.net/lemon4869/article/details/107145684

基于 OpenVINO 工具套件优化并加速模型

输入命令cd C:\Program Files (x86)\IntelSWTools\openvino_2019.3.334\deployment_tools\model_optimizer，进入 model_optimizer 目录
输入命令 cd mo_tf.py --input_mode=E:\1-tf_train\workspaces\ocr_project\trained_frozen_models\ocr_model\frozen_inference_graph.pb --tensorflow_use_custom_operations_config="C:\Program Files (x86)\IntelSWTools\openvino_2019.3.334\deployment_tools\model_optimizer\extensions\front\tf\ssd_v2_support.json" --tensorflow_object_detection_api_pipeline_configE:\1-tf_train\workspaces\ocr_project\trained_frozen_models\ocr_model\pipeline.config --reverse_input_channels --data_type=FP16 --output_dir=E:\1-tf_train\workspaces\ocr_project\IR_model --model_name=ocr_detector，对模型进行优化

基于 OpenVINO 工具套件部署模型

所谓部署模型，就是在我们的应用程序当中调用优化好的模型进行推理。

• ocr_infer.cpp

//包含必要的头文件
#include<string> //C++ string标准库
#include<inference_engine.hpp> //Inference Engine库
#include<samples/ocv_common.hpp>//OpenCV库及matU8ToBlob函数定义
#include<ext_list.hpp>//IE CPU扩展库
#include<time.h>//C time标准库

using namespace InferenceEngine;

//将OpenCV Mat对象中的图像数据传给为InferenceEngine Blob对象
void frameToBlob(const cv::Mat& frame, InferRequest::Ptr& inferRequest, const std::string& inputName) {
	/* 从OpenCV Mat对象中拷贝图像数据到InferenceEngine 输入Blob对象 */
	Blob::Ptr frameBlob = inferRequest->GetBlob(inputName);
	matU8ToBlob<uint8_t>(frame, frameBlob);
}
//配置推理计算设备，IR文件路径，图片路径，阈值和标签
std::string DEVICE = "CPU";
std::string IR_FileNameNoExt = "D:/tf_train/workspaces/ocr_project/IR_model/ocr_detector";
std::string imageFile = "D:/tf_train/workspaces/ocr_project/images/test/353.jpeg";
float confidence_threshold = 0.7; //取值0~1
std::vector<std::string> labels = { "Fake","8","2","P","O","9","R","1" }; //标签输入

int main(void) {
	// ------------------------------------------------------------------------------------------

	// --------------------------- 1. 载入硬件插件(Plugin) --------------------------------------
	Core ie;
	std::cout << "1.Load Plugin..." << std::endl;     //输出当前执行步骤信息
	std::cout << ie.GetVersions(DEVICE) << std::endl; //输出插件版本信息

	 /** 载入默认扩展库 **/
	if (DEVICE.find("CPU") != std::string::npos) {
		/**
		 * CPU扩展库包含自定义的MKLDNNPlugin层实现
		 * 这些没有被MKLDNN实现的自定义层对的网络层推理计算很有用
		**/
		ie.AddExtension(std::make_shared<Extensions::Cpu::CpuExtensions>(), "CPU");

	}
	// ----------------------------------------------------------------------------------

	// ------------------- 2. 读取IR文件 (.xml and .bin files) --------------------------
	std::cout << "2.Read IR File..." << std::endl;

	CNNNetReader networkReader;
	/** 读取神经网络模型文件*.xml **/
	networkReader.ReadNetwork(IR_FileNameNoExt + ".xml");
	/** 读取神经网络权重文件*.bin **/
	networkReader.ReadWeights(IR_FileNameNoExt + ".bin");
	/** 获取神经网络对象 **/
	CNNNetwork network = networkReader.getNetwork();
	// ----------------------------------------------------------------------------------

	// -------------------- 3. 配置网络输入输出 -----------------------------------------
	std::cout << "3.Prepare Input and Output..." << std::endl;
	/** 获得神经网络的输入信息 **/
	InputsDataMap inputsInfo(network.getInputsInfo());
	std::string imageInputName, imInfoInputName;
	InputInfo::Ptr inputInfo = nullptr;
	SizeVector inputImageDims;

	/** 遍历模型所有的输入blobs **/
	for (auto & item : inputsInfo) {
		/** 处理保存图像数据的第一个张量 **/
		if (item.second->getInputData()->getTensorDesc().getDims().size() == 4) {
			imageInputName = item.first;
			inputInfo = item.second;
			/** 创建第一个输入blob **/
			Precision inputPrecision = Precision::U8;
			item.second->setPrecision(inputPrecision);
		}
		else if (item.second->getInputData()->getTensorDesc().getDims().size() == 2) {
			imInfoInputName = item.first;
			Precision inputPrecision = Precision::FP32;
			item.second->setPrecision(inputPrecision);
			if ((item.second->getTensorDesc().getDims()[1] != 3 && item.second->getTensorDesc().getDims()[1] != 6)) {
				throw std::logic_error("Invalid input info. Should be 3 or 6 values length");
			}
		}
	}
	/** 获得神经网络的输出信息 **/
	OutputsDataMap outputsInfo(network.getOutputsInfo());
	std::string outputName;
	DataPtr outputInfo;
	for (const auto& out : outputsInfo) {
		if (out.second->getCreatorLayer().lock()->type == "DetectionOutput") {
			outputName = out.first;
			outputInfo = out.second;
		}
	}
	const SizeVector outputDims = outputInfo->getTensorDesc().getDims();
	const int maxProposalCount = outputDims[2];
	const int objectSize = outputDims[3];
	outputInfo->setPrecision(Precision::FP32);
	// --------------------------------------------------------------------------------------------

	// --------------------------- 4. 载入模型到AI推理计算设备---------------------------------------
	std::cout << "4.Load model into device..." << std::endl;
	ExecutableNetwork executable_network = ie.LoadNetwork(network, DEVICE, {});

	// --------------------------------------------------------------------------------------------

	// --------------------------- 5. 创建Infer Request--------------------------------------------
	std::cout << "5.Create Infer Request..." << std::endl;
	InferRequest::Ptr infer_request = executable_network.CreateInferRequestPtr();

	// --------------------------------------------------------------------------------------------

	// --------------------------- 6. 准备输入数据 ------------------------------------------------
	std::cout << "6.Prepare Input..." << std::endl;
	cv::Mat img = cv::imread(imageFile);
	float scale = 0.4;//由于图片太大，先进行放缩
	cv::Size dsize = cv::Size(int(img.cols*scale), int(img.rows*scale));
	cv::Mat resized_img;
	cv::resize(img, resized_img, dsize);
	const size_t width = (size_t)resized_img.cols;
	const size_t height = (size_t)resized_img.rows;
	frameToBlob(resized_img, infer_request, imageInputName);

	// --------------------------------------------------------------------------------------------

	// --------------------------- 7. 执行推理计算 ------------------------------------------------
	std::cout << "7.Start inference..." << std::endl;
	std::clock_t begin, end;
	begin = std::clock();
	infer_request->Infer();
	end = std::clock();
	std::ostringstream infer_time;//计算推理计算所花费的时间
	infer_time << "Infer Time:" << (double)(end - begin) << "ms";
	cv::putText(resized_img, infer_time.str(), cv::Point2f(0, 12), cv::FONT_HERSHEY_TRIPLEX, 0.6, cv::Scalar(0, 255, 0));
	// --------------------------------------------------------------------------------------------

	// --------------------------- 8. 处理输出 ----------------------------------------------------
	std::cout << "8.Process output blobs..." << std::endl;
	const float *detections = infer_request->GetBlob(outputName)->buffer().as<PrecisionTrait<Precision::FP32>::value_type*>();
	for (int i = 0; i < maxProposalCount; i++) {
		float image_id = detections[i * objectSize + 0];
		float confidence = detections[i * objectSize + 2];
		auto label = static_cast<int>(detections[i * objectSize + 1]);
		float xmin = detections[i * objectSize + 3] * width;
		float ymin = detections[i * objectSize + 4] * height;
		float xmax = detections[i * objectSize + 5] * width;
		float ymax = detections[i * objectSize + 6] * height;

		if (confidence > confidence_threshold) {
			/** 仅当> confidence_threshold值时，显示推理计算结果 **/
			std::ostringstream conf;
			conf << ":" << std::fixed << std::setprecision(3) << confidence;
			cv::putText(resized_img, (labels[label] + conf.str()),
				cv::Point2f(xmin, ymin - 5), cv::FONT_HERSHEY_COMPLEX_SMALL, 1,
				cv::Scalar(0, 0, 255));
			cv::rectangle(resized_img, cv::Point2f(xmin, ymin), cv::Point2f(xmax, ymax), cv::Scalar(0, 0, 255));
		}
	}
	std::cout << "Infer done!" << std::endl;	
	cv::imshow("Detection results", resized_img);
	cv::waitKey(0);
	cv::destroyAllWindows();

	return 0;
}