-
训练模式保存为onnx格式文件;
-
根据平台环境,下载第三方库onnxruntime;
首先根据部署需要,配置环境:
获取如下安装指令:比如使用visual studio 2019,可以直接在”程序包管理器控制台“输入如下指令进行安装:
Install-Package Microsoft.ML.OnnxRuntime -Version 1.9.0
- 使用onnxruntime,C++实现onnx模型调用:
#include <assert.h>
#include <vector>
#include <onnxruntime_cxx_api.h>
#include <opencv2/opencv.hpp>
#include <opencv2\imgproc\types_c.h>
int main(int argc, char* argv[])
{
Mat img = imread("..\\data\\face3.bmp");
const int row = 480;
const int col = 480;
Mat dst;
cvtColor(img, dst, CV_BGR2GRAY);
Mat input_tmp;
Mat img_padding(640, 640, CV_8UC1);
memset(img_padding.data, 0, 640 * 640);
memcpy(img_padding.data, dst.data, 640 * 480);
resize(img_padding, input_tmp, Size(col, row));
//imshow("src", input_tmp);
//waitKey(0);
FDRect box[20];
int16_t fd_count = 0;
// FD detect
{
//Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test");
Ort::Env env(ORT_LOGGING_LEVEL_ERROR, "test");
Ort::SessionOptions session_options;
session_options.SetIntraOpNumThreads(1);
session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
#ifdef _WIN32
const wchar_t* model_path = L"./../model.onnx";
#else
const char* model_path = "./../model.onnx";
#endif
Ort::Session session(env, model_path, session_options);
// print model input layer (node names, types, shape etc.)
Ort::AllocatorWithDefaultOptions allocator;
// print number of model input nodes
size_t num_input_nodes = session.GetInputCount();
std::vector<const char*> input_node_names = { "input" }; // Need to known model input node name
//std::vector<const char*> output_node_names = { "output","output_mask" };
std::array<float, 1 * 1 * 480 * 480 > input_image_{};
const char* const output_names[] = { "feature1", "feature2", "feature3", "feature4", "feature5", "feature6", "feature7", "feature8" }; // Need to known model output nodes name
Ort::Value input_tensor{ nullptr };
std::array<int64_t, 4> input_shape_{ 1,1, 480, 480 };
auto allocator_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
input_tensor = Ort::Value::CreateTensor<float>(allocator_info, input_image_.data(), input_image_.size(), input_shape_.data(), input_shape_.size());
// If model with multi input, use:
//{
// std::vector<Ort::Value> ort_inputs;
// ort_inputs.push_back(std::move(label_input_tensor));
// ort_inputs.push_back(std::move(f2_input_tensor));
// ort_inputs.push_back(std::move(f11_input_tensor));
// std::vector<const char*> input_names = { "Label", "F2", "F1" };
// const char* const output_names[] = { "Label0", "F20", "F11" };
// std::vector<Ort::Value> ort_outputs = session.Run(Ort::RunOptions{ nullptr }, input_names.data(),
// ort_inputs.data(), ort_inputs.size(), output_names, countof(output_names));
//}
float* i_data = input_image_.data();
fill(input_image_.begin(), input_image_.end(), 0.f);
for (int i = 0; i < row; i++)
{
for (int j = 0; j < col; j++)
{
i_data[i * col + j] = (float)((input_tmp.ptr<uchar>(i)[j]) - 128.0) / 128.0;
}
}
std::vector<Ort::Value> ort_inputs;
ort_inputs.push_back(std::move(input_tensor));
// score model & input tensor, get back output tensor
auto output_tensors = session.Run(Ort::RunOptions{ nullptr }, input_node_names.data(), ort_inputs.data(), ort_inputs.size(), output_names, 8);
// Get pointer to output tensor float values
float* feature1 = output_tensors[0].GetTensorMutableData<float>();
float* feature2 = output_tensors[1].GetTensorMutableData<float>();
float* feature3 = output_tensors[2].GetTensorMutableData<float>();
float* feature4 = output_tensors[3].GetTensorMutableData<float>();
float* feature5 = output_tensors[4].GetTensorMutableData<float>();
float* feature6 = output_tensors[5].GetTensorMutableData<float>();
float* feature7 = output_tensors[6].GetTensorMutableData<float>();
float* feature8 = output_tensors[7].GetTensorMutableData<float>();
}
}
4. 多模型调度报错问题:
如果同时调度多个模型推理(比如进行人脸检测模型推理后同时进行人脸关键点检测模型推理),可能出现编译/运行失败:
解决办法:使用更大的堆栈保留空间
5. 总结:
使用轻巧的神经网络推理框架(比如onnxruntime),可以大大提升模型快速部署验证的效率。比如需要将模型集成到大型解决方案中,如果使用opencv dnn会导致依赖库文件过大(若兼容window和Linux平台,库依赖内存~1G),且跨平台运行稳定性差。使用onnxruntime,依赖库内存<30M,且稳定性好。