7、同步、异步推理

同步模式例子

同步推理：程序等待模型推理完成后，再去获取下一张图片，进行相应的处理，然后交给模型进行推理。

cats_dogs_infer_SYNC.cpp

// --------------------------- 6. 同步推理计算 ------------------------------------------------
	std::cout << "6.Start SYNC inference..." << std::endl;
	std::clock_t begin, end;
	cv::Mat img;
	const float *detections;

	begin = std::clock();
	for (size_t i = 0; i < 200; i++)
	{   //图像采集
		img = cv::imread(imageFile);
		//图像预处理
		frameToBlob(img, infer_request, imageInputName);
		//AI同步推理计算
		infer_request->Infer();
		//获取输出结果
		detections = infer_request->GetBlob(outputName)->buffer().as<PrecisionTrait<Precision::FP32>::value_type*>();
	}
	end = std::clock();

	const size_t width = (size_t)img.cols;
	const size_t height = (size_t)img.rows;	
	std::ostringstream infer_time;//计算推理计算所花费的时间
	infer_time << "Infer Time:" << (double)(end - begin) << "ms";
	cv::putText(img, infer_time.str(), cv::Point2f(0, 12), cv::FONT_HERSHEY_TRIPLEX, 0.6, cv::Scalar(0, 255, 0));
	// --------------------------------------------------------------------------------------------

	// --------------------------- 7. 处理输出 ----------------------------------------------------
	std::cout << "7.Process output blobs..." << std::endl;
	 
	for (int i = 0; i < maxProposalCount; i++) {
		float image_id = detections[i * objectSize + 0];

		float confidence = detections[i * objectSize + 2];
		auto label = static_cast<int>(detections[i * objectSize + 1]);
		float xmin = detections[i * objectSize + 3] * width;
		float ymin = detections[i * objectSize + 4] * height;
		float xmax = detections[i * objectSize + 5] * width;
		float ymax = detections[i * objectSize + 6] * height;

		if (confidence > confidence_threshold) {
			/** 仅当> confidence_threshold值时，显示推理计算结果 **/
			std::ostringstream conf;
			conf << ":" << std::fixed << std::setprecision(3) << confidence;
			cv::putText(img, (labels[label] + conf.str()),
				cv::Point2f(xmin, ymin - 5), cv::FONT_HERSHEY_COMPLEX_SMALL, 1,
				cv::Scalar(0, 0, 255));
			cv::rectangle(img, cv::Point2f(xmin, ymin), cv::Point2f(xmax, ymax), cv::Scalar(0, 0, 255));
		}
	}

异步模式例子

异步推理：把一张图片交给模型推理的同时，程序立即获取下一张图片，进行相应的预处理，然后等待上一次推理结束，再把该图片交给模型推理。

cats_dogs_infer_ASYNC.cpp

// --------------------------- 6. 异步推理计算 ------------------------------------------------
	std::cout << "6.Start ASYNC inference..." << std::endl;
	std::clock_t begin, end;
	cv::Mat curr_frame,next_frame;
	const float *detections;

	begin = std::clock();
	curr_frame = cv::imread(imageFile);//采集当前帧图像
	frameToBlob(curr_frame, async_infer_request_curr, imageInputName);//预处理当前帧图像	
	for (size_t i = 0; i < 200; i++)
	{		
		//启动当前帧图像的AI异步推理计算
		async_infer_request_curr->StartAsync();
		//采集下一帧图像
		next_frame = cv::imread(imageFile);
		//预处理下一帧图像
		frameToBlob(next_frame, async_infer_request_next, imageInputName);
		//等待当前帧AI推理计算结束		
		async_infer_request_curr->Wait(IInferRequest::WaitMode::RESULT_READY);
		//获取当前帧输出结果
		detections = async_infer_request_curr->GetBlob(outputName)->buffer().as<PrecisionTrait<Precision::FP32>::value_type*>();
		//下一帧图像转为当前帧图像
		curr_frame = next_frame;
		//下一帧图像清空，准备采集新的图像
		next_frame = cv::Mat();
		//下一帧推理请求转为当前帧推理请求
		async_infer_request_curr.swap(async_infer_request_next);
	}
	end = std::clock();

	const size_t width = (size_t)curr_frame.cols;
	const size_t height = (size_t)curr_frame.rows;
	std::ostringstream infer_time;//计算推理计算所花费的时间
	infer_time << "Infer Time:" << (double)(end - begin) << "ms";
	cv::putText(curr_frame, infer_time.str(), cv::Point2f(0, 12), cv::FONT_HERSHEY_TRIPLEX, 0.6, cv::Scalar(0, 255, 0));
	// --------------------------------------------------------------------------------------------

	// --------------------------- 7. 处理输出 ----------------------------------------------------
	std::cout << "7.Process output blobs..." << std::endl;
	 
	for (int i = 0; i < maxProposalCount; i++) {
		float image_id = detections[i * objectSize + 0];

		float confidence = detections[i * objectSize + 2];
		auto label = static_cast<int>(detections[i * objectSize + 1]);
		float xmin = detections[i * objectSize + 3] * width;
		float ymin = detections[i * objectSize + 4] * height;
		float xmax = detections[i * objectSize + 5] * width;
		float ymax = detections[i * objectSize + 6] * height;

		if (confidence > confidence_threshold) {
			/** 仅当> confidence_threshold值时，显示推理计算结果 **/
			std::ostringstream conf;
			conf << ":" << std::fixed << std::setprecision(3) << confidence;
			cv::putText(curr_frame, (labels[label] + conf.str()),
				cv::Point2f(xmin, ymin - 5), cv::FONT_HERSHEY_COMPLEX_SMALL, 1,
				cv::Scalar(0, 0, 255));
			cv::rectangle(curr_frame, cv::Point2f(xmin, ymin), cv::Point2f(xmax, ymax), cv::Scalar(0, 0, 255));
		}
	}