模型编译:
#encoding:utf-8
import onnx
import numpy as np
import tvm
import tvm.relay as relay
import os
from tvm.contrib import ndk
onnx_model = onnx.load('mobilenet_v3_small.onnx')
x = np.ones([1,3,224,224])
arch = "arm64"
target = "llvm -mtriple=arm64-linux-android"
input_name = 'input1'
shape_dict = {input_name: x.shape}
sym, params = relay.frontend.from_onnx(onnx_model, shape_dict)
with tvm.transform.PassContext(opt_level=3):
graph, lib, params = relay.build(sym, target=target, params=params)
lib.export_library("deploy.so", cc=/path_to_ndk/26.0.10792818/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android34-clang++")
graph_json_path = "deploy.json"
with open(graph_json_path, 'w') as fo:
fo.write(graph)
param_path = "deploy.params"
with open(param_path, 'wb') as fo:
fo.write(relay.save_param_dict(params))
---------------------------------------------------------------------------------------------------
c++ 推理:
#include <dlpack/dlpack.h>
#include <tvm/runtime/module.h>
#include <tvm/runtime/registry.h>
#include <tvm/runtime/packed_func.h>
#include <fstream>
#include <iterator>
#include <algorithm>
int main()
{
// tvm module for compiled functions
tvm::runtime::Module mod_syslib = tvm::runtime::Module::LoadFromFile("deploy.so");
// json graph
std::ifstream json_in("deploy.json", std::ios::in);
std::string json_data((std::istreambuf_iterator<char>(json_in)), std::istreambuf_iterator<char>());
json_in.close();
// parameters in binary
std::ifstream params_in("deploy.params", std::ios::binary);
std::string params_data((std::istreambuf_iterator<char>(params_in)), std::istreambuf_iterator<char>());
params_in.close();
// parameters need to be TVMByteArray type to indicate the binary data
TVMByteArray params_arr;
params_arr.data = params_data.c_str();
params_arr.size = params_data.length();
int dtype_code = kDLFloat;
int dtype_bits = 32;
int dtype_lanes = 1;
int device_type = kDLCPU;
int device_id = 0;
// get global function module for graph runtime
tvm::runtime::Module mod = (*tvm::runtime::Registry::Get("tvm.graph_executor.create"))(json_data, mod_syslib, device_type, device_id);
DLTensor* x;
int in_ndim = 4;
int64_t in_shape[4] = {1, 3, 224, 224};
TVMArrayAlloc(in_shape, in_ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &x);
// load image data saved in binary
std::ifstream data_fin("cat.bin", std::ios::binary);
data_fin.read(static_cast<char*>(x->data), 3 * 224 * 224 * 4);
// get the function from the module(set input data)
tvm::runtime::PackedFunc set_input = mod.GetFunction("set_input");
set_input("input1", x);
// get the function from the module(load patameters)
tvm::runtime::PackedFunc load_params = mod.GetFunction("load_params");
load_params(params_arr);
// get the function from the module(run it)
tvm::runtime::PackedFunc run = mod.GetFunction("run");
run();
DLTensor* y;
int out_ndim = 2;
int64_t out_shape[2] = {1, 1000};
TVMArrayAlloc(out_shape, out_ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &y);
// get the function from the module(get output data)
tvm::runtime::PackedFunc get_output = mod.GetFunction("get_output");
get_output(0, y);
// get the maximum position in output vector
auto y_iter = static_cast<float*>(y->data);
auto max_iter = std::max_element(y_iter, y_iter + 1000);
auto max_index = std::distance(y_iter, max_iter);
std::cout << "The maximum position in output vector is: " << max_index << std::endl;
TVMArrayFree(x);
TVMArrayFree(y);
return 0;
}
--------------------------------------------------------------------------------------------------------
CMakelist:
cmake_minimum_required(VERSION 3.0)
project(testdemo)
set(CMAKE_CXX_STANDARD 17)
include_directories(
/path_to_tvm/include
/path_to_tvm/3rdparty/dmlc-core/include
/path_to_tvm/3rdparty/dlpack/include
/path_to_tvm/src/runtime
)
link_directories(
/path_to_tvm/build_android_runtime
)
add_executable(testdemo cpp_inference.cpp)
target_link_libraries(testdemo libtvm_runtime.so)
注意:这里链接的是android的libtvm_runtime.so
---------------------------------------------------------------------
数据预处理:
img = np.array(Image.open("cat.jpg").resize((224,224))).astype(np.float32)
img_np = (img/255. - (0.485, 0.456,0.406)) / (0.229, 0.224, 0.225)
img_np = img_np[np.newaxis, :]
img_np = img_np.transpose(0,3,1,2)
img_np = img_np.astype(np.float32)
img_np.tofile('cat.bin')