RK3568的rknn环境配置

本文实现yolov8的rk3568板端部署的基本npu环境配置，以比较简洁的方式，亲测可行。本专栏部署为官方yolov8n模型，自己训练的 1 类物体的识别的部署（与官方的80类不一样）

1.先安装rknn-toolkit2

pip install rknn-toolkit2

2.第一步安装完成以后，可以实现python文件的推理，想要使用C++还需要配置相关的SDK环境。

git clone https://github.com/airockchip/rknn-toolkit2.git

下载rknn-toolkit2的安装包，有点大，如果git clone不行可以考虑直接下载包即可。

这里注意要保证安装包的版本和第1步安装的python版本一致（而且必须保证生成rknn文件时，python版本的toolkit版本和C++的版本一致，与及生成rknn时平台选择不要搞混），例如我的都是v2.3.0，平台为rk3568（千万别搞混成rk3588，会导致模型load不成功）

3.去到 rknn-toolkit2/rknpu2 目录  ，

# 安装 rknn_server
# 注：在64位Linux系统中，BOARD_ARCH对应aarch64目录，在32位系统，对应armhf目录。
sudo cp runtime/Linux/rknn_server/${BOARD_ARCH}/usr/bin/* /usr/bin
# 安装 librknnrt.so
sudo cp runtime/Linux/librknn_api/${BOARD_ARCH}/librknnrt.so /usr/lib

# 赋予可执行权限
chmod +x /usr/bin/rknn_server
chmod +x /usr/bin/start_rknn.sh
chmod +x /usr/bin/restart_rknn.sh
# 重启 rknn_server 服务
restart_rknn.sh

3.*如果要使用rga需要安装rga的库。这里附带一下安装方法

(1)git clone https://github.com/airockchip/librga.git

然后把librga.so进行安装。（这里注意选择你rk平台对应的rga版本，否则可能会在运行的时候报错）

sudo cp   librga/libs/Linux/gcc-aarch64/librga.so   /usr/lib

4.如果没有报错，就安装成功了

同时附带我测试的demo

(1)生成rknn的python脚本

import os

import urllib

import traceback

import time

import sys

import numpy as np

import cv2

from rknn.api import RKNN

from math import exp

ONNX_MODEL = './last0303.onnx'

RKNN_MODEL = './last0303_3.rknn'

DATASET = '/home/yjh/thj/data/plate_base/images/images_list.txt'

QUANTIZE_ON = False

CLASSES = ['plate_base']

class_num = len(CLASSES)

head_num = 3

nms_thresh = 0.45

object_thresh = 0.3

strides = [8, 16, 32]

input_height = 640

input_width = 640

anchors = int(input_height / strides[0] * input_width / strides[0] + input_height / strides[1] * input_width / strides[1] + input_height / strides[2] * input_width / strides[2])

 

class DetectBox:

def __init__(self, classId, score, xmin, ymin, xmax, ymax):

self.classId = classId

self.score = score

self.xmin = xmin

self.ymin = ymin

self.xmax = xmax

self.ymax = ymax

def IOU(xmin1, ymin1, xmax1, ymax1, xmin2, ymin2, xmax2, ymax2):

xmin = max(xmin1, xmin2)

ymin = max(ymin1, ymin2)

xmax = min(xmax1, xmax2)

ymax = min(ymax1, ymax2)

innerWidth = xmax - xmin

innerHeight = ymax - ymin

innerWidth = innerWidth if innerWidth > 0 else 0

innerHeight = innerHeight if innerHeight > 0 else 0

innerArea = innerWidth * innerHeight

area1 = (xmax1 - xmin1) * (ymax1 - ymin1)

area2 = (xmax2 - xmin2) * (ymax2 - ymin2)

total = area1 + area2 - innerArea

return innerArea / total

 

def NMS(detectResult):

predBoxs = []

sort_detectboxs = sorted(detectResult, key=lambda x: x.score, reverse=True)

for i in range(len(sort_detectboxs)):

xmin1 = sort_detectboxs[i].xmin

ymin1 = sort_detectboxs[i].ymin

xmax1 = sort_detectboxs[i].xmax

ymax1 = sort_detectboxs[i].ymax

classId = sort_detectboxs[i].classId

if sort_detectboxs[i].classId != -1:

predBoxs.append(sort_detectboxs[i])

for j in range(i + 1, len(sort_detectboxs), 1):

if classId == sort_detectboxs[j].classId:

xmin2 = sort_detectboxs[j].xmin

ymin2 = sort_detectboxs[j].ymin

xmax2 = sort_detectboxs[j].xmax

ymax2 = sort_detectboxs[j].ymax

iou = IOU(xmin1, ymin1, xmax1, ymax1, xmin2, ymin2, xmax2, ymax2)

if iou > nms_thresh:

sort_detectboxs[j].classId = -1

return predBoxs

 

def postprocess(outputs, image_h, image_w):

print('postprocess ... ')

scale_h = image_h / input_height

scale_w = image_w / input_width

detectResult = []

output = outputs[0]

for i in range(0, anchors):

cls_index = 0

cls_vlaue = -1

for cl in range(class_num):

val = output[i + (4 + cl) * anchors]

if val > cls_vlaue:

cls_vlaue = val

cls_index = cl

if cls_vlaue > object_thresh:

cx = output[i + 0 * anchors]

cy = output[i + 1 * anchors]

cw = output[i + 2 * anchors]

ch = output[i + 3 * anchors]

xmin = (cx - 0.5 * cw) * scale_w

ymin = (cy - 0.5 * ch) * scale_h

xmax = (cx + 0.5 * cw) * scale_w

ymax = (cy + 0.5 * ch) * scale_h

box = DetectBox(cls_index, cls_vlaue, xmin, ymin, xmax, ymax)

detectResult.append(box)

# NMS

print('detectResult:', len(detectResult))

predBox = NMS(detectResult)

return predBox

 

def export_rknn_inference(img):

# Create RKNN object

rknn = RKNN(verbose=False)

# pre-process config

print('--> Config model')

rknn.config(mean_values=[[0, 0, 0]], std_values=[[255, 255, 255]], quantized_algorithm='normal', quantized_method='channel', target_platform='rk3568') # mmse

print('done')

# Load ONNX model

print('--> Loading model')

ret = rknn.load_onnx(model=ONNX_MODEL)

if ret != 0:

print('Load model failed!')

exit(ret)

print('done')

# Build model

print('--> Building model')

ret = rknn.build(do_quantization=QUANTIZE_ON, dataset=DATASET, rknn_batch_size=1)

if ret != 0:

print('Build model failed!')

exit(ret)

print('done')

# Export RKNN model

print('--> Export rknn model')

ret = rknn.export_rknn(RKNN_MODEL)

if ret != 0:

print('Export rknn model failed!')

exit(ret)

print('done')

# Init runtime environment

print('--> Init runtime environment')

ret = rknn.init_runtime()

# ret = rknn.init_runtime(target='rk3566')

if ret != 0:

print('Init runtime environment failed!')

exit(ret)

print('done')

# Inference

print('--> Running model')

outputs = rknn.inference(inputs=[img])

rknn.release()

print('outputs: ', outputs)

print('done')

return outputs

 

if __name__ == '__main__':

print('This is main ...')

img_path = './test.jpg'

orig_img = cv2.imread(img_path)

img_h, img_w = orig_img.shape[:2]

origimg = cv2.resize(orig_img, (input_width, input_height), interpolation=cv2.INTER_LINEAR)

origimg = cv2.cvtColor(origimg, cv2.COLOR_BGR2RGB)

img = np.expand_dims(origimg, 0)

outputs = export_rknn_inference(img)

out = []

for i in range(len(outputs)):

print(outputs[i].shape)

out.append(outputs[i].reshape(-1))

predbox = postprocess(out, img_h, img_w)

print(len(predbox))

for i in range(len(predbox)):

xmin = int(predbox[i].xmin)

ymin = int(predbox[i].ymin)

xmax = int(predbox[i].xmax)

ymax = int(predbox[i].ymax)

classId = predbox[i].classId

score = predbox[i].score

cv2.rectangle(orig_img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)

ptext = (xmin, ymin)

title = CLASSES[classId] + "%.2f" % score

cv2.putText(orig_img, title, ptext, cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2, cv2.LINE_AA)

cv2.imwrite('./test_rknn_result.jpg', orig_img)

 

(2)CMakeList.txt和cpp

cmake_minimum_required(VERSION 3.0.2)
project(yolov8_rknn_ros)

find_package(catkin REQUIRED COMPONENTS
  roscpp
  sensor_msgs
  cv_bridge
)

find_package(OpenCV REQUIRED)

catkin_package(CATKIN_DEPENDS ${PACKAGE_DEPENDENCIES} roscpp sensor_msgs cv_bridge)

include_directories(
  ${catkin_INCLUDE_DIRS}
  ${OpenCV_INCLUDE_DIRS}
  ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/librknn_api/include
)

link_directories(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/librknn_api/aarch64)

#add_executable(yolov8_rknn_ros yolov8_rknn_ros.cpp)
#target_link_libraries(yolov8_rknn_ros
#                      ${catkin_LIBRARIES}
#                      ${OpenCV_LIBRARIES}
#                      ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/librknn_api/aarch64/librknn_api.so
#                      ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/librknn_api/aarch64/librknnrt.so)

add_executable(onnx2rknn_demo onnx2rknn_demo.cpp)
target_link_libraries(onnx2rknn_demo
                      ${catkin_LIBRARIES}
                      ${OpenCV_LIBRARIES}
                      librknnrt.so)

#include <iostream>

#include <vector>

#include <cmath>

#include <opencv2/opencv.hpp>

#include "rknn_api.h"

#define ONNX_MODEL "/home/robot/rknn_ws/last0303.onnx"

#define RKNN_MODEL "/home/robot/rknn_ws/last0303_3.rknn"

#define DATASET "/home/yjh/thj/data/plate_base/images/images_list.txt"

#define QUANTIZE_ON false

#define NMS_THRESH 0.45

#define OBJECT_THRESH 0.3

#define INPUT_WIDTH 640

#define INPUT_HEIGHT 640

using namespace std;

using namespace cv;

// 定义类别

const vector<string> CLASSES = {"plate_base"};

const int CLASS_NUM = CLASSES.size();

const int STRIDES[3] = {8, 16, 32};

// 计算 anchors 数量

const int ANCHORS = (INPUT_WIDTH / STRIDES[0]) * (INPUT_HEIGHT / STRIDES[0]) +

(INPUT_WIDTH / STRIDES[1]) * (INPUT_HEIGHT / STRIDES[1]) +

(INPUT_WIDTH / STRIDES[2]) * (INPUT_HEIGHT / STRIDES[2]);

struct DetectBox {

int classId;

float score;

float xmin, ymin, xmax, ymax;

DetectBox(int classId, float score, float xmin, float ymin, float xmax, float ymax)

: classId(classId), score(score), xmin(xmin), ymin(ymin), xmax(xmax), ymax(ymax) {}

};

// 计算IOU

float IOU(DetectBox &box1, DetectBox &box2) {

float xmin = max(box1.xmin, box2.xmin);

float ymin = max(box1.ymin, box2.ymin);

float xmax = min(box1.xmax, box2.xmax);

float ymax = min(box1.ymax, box2.ymax);

float interWidth = max(0.0f, xmax - xmin);

float interHeight = max(0.0f, ymax - ymin);

float interArea = interWidth * interHeight;

float area1 = (box1.xmax - box1.xmin) * (box1.ymax - box1.ymin);

float area2 = (box2.xmax - box2.xmin) * (box2.ymax - box2.ymin);

return interArea / (area1 + area2 - interArea);

}

// 非极大值抑制（NMS）

vector<DetectBox> NMS(vector<DetectBox> &detectResult) {

vector<DetectBox> predBoxes;

// 置信度排序

sort(detectResult.begin(), detectResult.end(),

[](const DetectBox &a, const DetectBox &b) { return a.score > b.score; });

for (size_t i = 0; i < detectResult.size(); i++) {

if (detectResult[i].classId == -1) continue;

predBoxes.push_back(detectResult[i]);

for (size_t j = i + 1; j < detectResult.size(); j++) {

if (detectResult[j].classId == detectResult[i].classId) {

if (IOU(detectResult[i], detectResult[j]) > NMS_THRESH) {

detectResult[j].classId = -1;

}

}

}

}

return predBoxes;

}

// 后处理

vector<DetectBox> postprocess(float *output, int img_h, int img_w) {

vector<DetectBox> detectResult;

float scale_h = float(img_h) / INPUT_HEIGHT;

float scale_w = float(img_w) / INPUT_WIDTH;

for (int i = 0; i < ANCHORS; i++) {

int cls_index = 0;

float cls_value = -1;

for (int cl = 0; cl < CLASS_NUM; cl++) {

float val = output[i + (4 + cl) * ANCHORS];

if (val > cls_value) {

cls_value = val;

cls_index = cl;

}

}

if (cls_value > OBJECT_THRESH) {

float cx = output[i];

float cy = output[i + ANCHORS];

float cw = output[i + 2 * ANCHORS];

float ch = output[i + 3 * ANCHORS];

float xmin = (cx - 0.5 * cw) * scale_w;

float ymin = (cy - 0.5 * ch) * scale_h;

float xmax = (cx + 0.5 * cw) * scale_w;

float ymax = (cy + 0.5 * ch) * scale_h;

detectResult.emplace_back(cls_index, cls_value, xmin, ymin, xmax, ymax);

}

}

return NMS(detectResult);

}

// 加载并运行 RKNN 推理

vector<DetectBox> run_rknn_inference(Mat &image) {

rknn_context ctx;

std::string model_path = "/home/robot/rknn_ws/last0303_3.rknn";

void* model_ptr = (void*)model_path.c_str(); // 推荐

int ret = rknn_init(&ctx, model_ptr, 0, 0, NULL);

if (ret != 0) {

cerr << "rknn_init failed: " << ret << endl;

exit(-1);

}

std::cout<<"test 1"<<std::endl;

rknn_input inputs[1];

rknn_output outputs[1];

// 设置输入

Mat img_resized;

resize(image, img_resized, Size(INPUT_WIDTH, INPUT_HEIGHT));

cvtColor(img_resized, img_resized, COLOR_BGR2RGB);

inputs[0].index = 0;

inputs[0].buf = img_resized.data;

inputs[0].size = INPUT_WIDTH * INPUT_HEIGHT * 3;

inputs[0].pass_through = false;

inputs[0].type = RKNN_TENSOR_UINT8;

inputs[0].fmt = RKNN_TENSOR_NHWC;

ret = rknn_inputs_set(ctx, 1, inputs);

if (ret != 0) {

cerr << "rknn_inputs_set failed: " << ret << endl;

exit(-1);

}

// 推理

ret = rknn_run(ctx, nullptr);

if (ret != 0) {

cerr << "rknn_run failed: " << ret << endl;

exit(-1);

}

// 获取输出

outputs[0].want_float = true;

ret = rknn_outputs_get(ctx, 1, outputs, nullptr);

if (ret != 0) {

cerr << "rknn_outputs_get failed: " << ret << endl;

exit(-1);

}

// 解析输出

vector<DetectBox> results = postprocess((float *)outputs[0].buf, image.rows, image.cols);

// 释放 RKNN 资源

rknn_outputs_release(ctx, 1, outputs);

rknn_destroy(ctx);

return results;

}

int main() {

cout << "Running RKNN Model..." << endl;

// 读取图像

Mat image = imread("/home/robot/rknn_ws/test.jpg");

if (image.empty()) {

cerr << "Failed to load image!" << endl;

return -1;

}

vector<DetectBox> predictions = run_rknn_inference(image);

// 画出检测框

for (const auto &box : predictions) {

rectangle(image, Point(box.xmin, box.ymin), Point(box.xmax, box.ymax), Scalar(255, 0, 0), 2);

putText(image, CLASSES[box.classId] + to_string(box.score), Point(box.xmin, box.ymin - 5),

FONT_HERSHEY_SIMPLEX, 0.7, Scalar(0, 0, 255), 2);

}

imwrite("/home/robot/rknn_ws/test_rknn_result.jpg", image);

cout << "Detection complete!" << endl;

return 0;

}