首先要知道的概念：
①rknn-toolkit2
rockchip官方为了将不同格式的神经网络模型文件转换为rk芯片支持的rknn格式而打包的python库项目，可以通过pip安装。
②rknn_model_zoo
rockchip官方提供的目前比较流行的神经网络模型在rk芯片上运行的demo，包括模型转换和推理。
③rknpu
需要放在板端驱动NPU的动态库文件。
截止到目前为止，rockchip将部署神经网络用的rknn-toolkit2更新到了v2.0.0-beta0GitHub - airockchip/rknn-toolkit2.之前根据文章RK3588部署YOLOv8_rk3588 yolov8-CSDN博客在rknn-toolkit2_1.5.0上部署成功，但是量化得到的模型效果并不是很令人满意。因此，我看到官方已经将rknn-toolkit2更新到了2.0.0，就通过最新版本的rknn-toolkit2部署试试效果。
在GitHub - airockchip/rknn_model_zoo项目中，rockchip更新了目前为止RKNPU支持的比较流行的神经网络模型。以YOLOv8为例，路径rknn_model_zoo/examples/yolov8 at main · airockchip/rknn_model_zoo · GitHub下写明了rockchip官方根据模型转换需要适配的项目代码GitHub - airockchip/ultralytics_yolov8: NEW - YOLOv8 🚀 in PyTorch > ONNX > CoreML > TFLite，下载这一项目代码，通过自己数据集训练得到的.pt权重文件可以根据以下代码导出onnx格式的模型文件。
from ultralytics import YOLO
model = YOLO('./weights/yolov8.pt') # 将这里换成你模型所在的路径
path = model.export(format="rknn", opset=12)同时，需要在电脑虚拟机安装Linux系统，在Linux环境下安装rknn-toolkit2的python环境用于转换rknn格式文件。
在路径https://github.com/airockchip/rknn_model_zoo/tree/main/examples/yolov8/python下，运行convert.py脚本，利用之前得到的onnx文件转换得到rknn文件。
python convert.py <onnx_model> <TARGET_PLATFORM> <dtype(optional，默认量化为int8)> <output_rknn_path(optional)>
# such as:
python convert.py ../model/yolov8.onnx rk3588 在运行推理代码之前，我们还需要更换板端的rknpu驱动库文件，我的rk3588是firefly出的，他们家的的SDK中rknpu目前只更新到了1.5.0，因此需要手动更新。具体步骤为：
①先去官方下载最新的rknpu，GitHub或者联想fileZ链接
https://github.com/airockchip/rknpu
联想Filez 密码：rknn
②将文件发送给3588板端，替换相关文件，替换前最好备份原文件。
cp rknpu2/runtime/Linux/rknn_server/aarch64/usr/bin/rknn_server /usr/bin/rknn_server
cp rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so /usr/lib/librknnrt.so
③这样的替换可以跑推理程序，完善的推理需要参照rknpu中的文档说明。
推理代码我选择adb连板调试，我在之前的基础上做了一点修改，主要体现在输入的resize和输出坐标的转换上。
import os
import cv2
from rknn.api import RKNN
import numpy as np
RKNN_MODEL = "../model/yolov8.rknn"
IMG_FOLDER = "../model/imgs"
RESULT_PATH = '../model/results/'
CLASSES = ['hole' ]
OBJ_THRESH = 0.8
NMS_THRESH = 0.45
MODEL_SIZE = (640, 640)
color_palette = np.random.uniform(0, 255, size=(len(CLASSES), 3))
def sigmoid(x):
return 1 / (1 + np.exp(-x))
def letter_box(im, new_shape, pad_color=(0,0,0), info_need=False):
# Resize and pad image while meeting stride-multiple constraints
shape = im.shape[:2] # current shape [height, width]
if isinstance(new_shape, int):
new_shape = (new_shape, new_shape)
# Scale ratio
r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
# Compute padding
ratio = r # width, height ratios
new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding
dw /= 2 # divide padding into 2 sides
dh /= 2
if shape[::-1] != new_unpad: # resize
im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=pad_color) # add border
if info_need is True:
return im, ratio, (dw, dh)
else:
return im
def filter_boxes(boxes, box_confidences, box_class_probs):
"""Filter boxes with object threshold.
"""
box_confidences = box_confidences.reshape(-1)
candidate, class_num = box_class_probs.shape
class_max_score = np.max(box_class_probs, axis=-1)
classes = np.argmax(box_class_probs, axis=-1)
_class_pos = np.where(class_max_score* box_confidences >= OBJ_THRESH)
scores = (class_max_score * box_confidences)[_class_pos]
boxes = boxes[_class_pos]
classes = classes[_class_pos]
return boxes, classes, scores
def nms_boxes(boxes, scores):
"""Suppress non-maximal boxes.
# Returns
keep: ndarray, index of effective boxes.
"""
x = boxes[:, 0]
y = boxes[:, 1]
w = boxes[:, 2] - boxes[:, 0]
h = boxes[:, 3] - boxes[:, 1]
areas = w * h
order = scores.argsort()[::-1]
keep = []
while order.size > 0:
i = order[0]
keep.append(i)
xx1 = np.maximum(x[i], x[order[1:]])
yy1 = np.maximum(y[i], y[order[1:]])
xx2 = np.minimum(x[i] + w[i], x[order[1:]] + w[order[1:]])
yy2 = np.minimum(y[i] + h[i], y[order[1:]] + h[order[1:]])
w1 = np.maximum(0.0, xx2 - xx1 + 0.00001)
h1 = np.maximum(0.0, yy2 - yy1 + 0.00001)
inter = w1 * h1
ovr = inter / (areas[i] + areas[order[1:]] - inter)
inds = np.where(ovr <= NMS_THRESH)[0]
order = order[inds + 1]
keep = np.array(keep)
return keep
def softmax(x, axis=None):
x = x - x.max(axis=axis, keepdims=True)
y = np.exp(x)
return y / y.sum(axis=axis, keepdims=True)
def dfl(position):
# Distribution Focal Loss (DFL)
n,c,h,w = position.shape
p_num = 4
mc = c//p_num
y = position.reshape(n,p_num,mc,h,w)
y = softmax(y, 2)
acc_metrix = np.array(range(mc),dtype=float).reshape(1,1,mc,1,1)
y = (y*acc_metrix).sum(2)
return y
def box_process(position):
grid_h, grid_w = position.shape[2:4]
col, row = np.meshgrid(np.arange(0, grid_w), np.arange(0, grid_h))
col = col.reshape(1, 1, grid_h, grid_w)
row = row.reshape(1, 1, grid_h, grid_w)
grid = np.concatenate((col, row), axis=1)
stride = np.array([MODEL_SIZE[1]//grid_h, MODEL_SIZE[0]//grid_w]).reshape(1,2,1,1)
position = dfl(position)
box_xy = grid +0.5 -position[:,0:2,:,:]
box_xy2 = grid +0.5 +position[:,2:4,:,:]
xyxy = np.concatenate((box_xy*stride, box_xy2*stride), axis=1)
return xyxy
def post_process(input_data):
boxes, scores, classes_conf = [], [], []
defualt_branch=3
pair_per_branch = len(input_data)//defualt_branch
# Python 忽略 score_sum 输出
for i in range(defualt_branch):
boxes.append(box_process(input_data[pair_per_branch*i]))
classes_conf.append(input_data[pair_per_branch*i+1])
scores.append(np.ones_like(input_data[pair_per_branch*i+1][:,:1,:,:], dtype=np.float32))
def sp_flatten(_in):
ch = _in.shape[1]
_in = _in.transpose(0,2,3,1)
return _in.reshape(-1, ch)
boxes = [sp_flatten(_v) for _v in boxes]
classes_conf = [sp_flatten(_v) for _v in classes_conf]
scores = [sp_flatten(_v) for _v in scores]
boxes = np.concatenate(boxes)
classes_conf = np.concatenate(classes_conf)
scores = np.concatenate(scores)
# filter according to threshold
boxes, classes, scores = filter_boxes(boxes, scores, classes_conf)
# nms
nboxes, nclasses, nscores = [], [], []
for c in set(classes):
inds = np.where(classes == c)
b = boxes[inds]
c = classes[inds]
s = scores[inds]
keep = nms_boxes(b, s)
if len(keep) != 0:
nboxes.append(b[keep])
nclasses.append(c[keep])
nscores.append(s[keep])
if not nclasses and not nscores:
return None, None, None
boxes = np.concatenate(nboxes)
classes = np.concatenate(nclasses)
scores = np.concatenate(nscores)
return boxes, classes, scores
def draw_detections(img, left, top, right, bottom, score, class_id):
"""
Draws bounding boxes and labels on the input image based on the detected objects.
Args:
img: The input image to draw detections on.
box: Detected bounding box.
score: Corresponding detection score.
class_id: Class ID for the detected object.
Returns:
None
"""
# Retrieve the color for the class ID
color = color_palette[class_id]
# Draw the bounding box on the image
cv2.rectangle(img, (int(left), int(top)), (int(right), int(bottom)), color, 2)
# 计算矩形框的中心点坐标
center_point = (int((left + right) / 2), int((top + bottom) / 2))
# 在中心点画一个标记点
cv2.circle(img, center_point, 5, (0, 0, 255), -1)
# Create the label text with class name and score
label = f"{CLASSES[class_id]}: {score:.2f}"
# Calculate the dimensions of the label text
(label_width, label_height), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
# Calculate the position of the label text
label_x = left
label_y = top - 10 if top - 10 > label_height else top + 10
# Draw a filled rectangle as the background for the label text
cv2.rectangle(img, (label_x, label_y - label_height), (label_x + label_width, label_y + label_height), color,
cv2.FILLED)
# Draw the label text on the image
cv2.putText(img, label, (label_x, label_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1, cv2.LINE_AA)
def draw(image, boxes, scores, classes):
img_h, img_w = image.shape[:2]
# Calculate scaling factors for bounding box coordinates
if(img_w >= img_h):
x_factor = img_w / MODEL_SIZE[0]
y_rsz = img_h * MODEL_SIZE[1] / img_w
y_factor = img_h / y_rsz
offset = (MODEL_SIZE[1] - y_rsz) / 2
else:
y_factor = img_h / MODEL_SIZE[1]
x_rsz = img_w * MODEL_SIZE[0] / img_h
x_factor = img_w / x_rsz
offset = (MODEL_SIZE[0] - x_rsz) / 2
for box, score, cl in zip(boxes, scores, classes):
x1, y1, x2, y2 = [int(_b) for _b in box]
if(img_w >= img_h):
left = int(x1 * x_factor)
top = int((y1 - offset) * y_factor)
right = int(x2 * x_factor)
bottom = int((y2 - offset) * y_factor)
else:
left = int((x1 - offset) * x_factor)
top = int(y1 * y_factor)
right = int((x2 - offset) * x_factor)
bottom = int(y2 * y_factor)
print('class: {}, score: {}'.format(CLASSES[cl], score))
print('box coordinate left,top,right,down: [{}, {}, {}, {}]'.format(left, top, right, bottom))
# Retrieve the color for the class ID
draw_detections(image, left, top, right, bottom, score, cl)
# cv2.rectangle(image, (left, top), (right, bottom), color, 2)
# cv2.putText(image, '{0} {1:.2f}'.format(CLASSES[cl], score),
# (left, top - 6),
# cv2.FONT_HERSHEY_SIMPLEX,
# 0.6, (0, 0, 255), 2)
def resize_image(image, size, letterbox_image):
"""
对输入图像进行resize
Args:
size:目标尺寸
letterbox_image: bool 是否进行letterbox变换
Returns:指定尺寸的图像
"""
from PIL import Image
ih, iw, _ = image.shape
h, w = size
if letterbox_image:
scale = min(w/iw, h/ih) # 缩放比例
nw = int(iw*scale)
nh = int(ih*scale)
image = cv2.resize(image, (nw, nh), interpolation=cv2.INTER_LINEAR)
# 生成画布
image_back = np.ones((h, w, 3), dtype=np.uint8) * 128
# 将image放在画布中心区域-letterbox
image_back[(h-nh)//2: (h-nh)//2 + nh, (w-nw)//2:(w-nw)//2+nw, :] = image
else:
image_back = image
return image_back
if __name__ == '__main__':
# 创建RKNN对象
rknn = RKNN()
#加载RKNN模型
print('--> Load RKNN model')
ret = rknn.load_rknn(RKNN_MODEL)
if ret != 0:
print('Load RKNN model failed')
exit(ret)
print('done')
# 初始化 runtime 环境
print('--> Init runtime environment')
# run on RK356x/RK3588 with Debian OS, do not need specify target.
ret = rknn.init_runtime(target='rk3588', device_id='03d205b241a91ccc')
# 如果使用电脑进行模拟测试
# ret = rknn.init_runtime()
if ret != 0:
print('Init runtime environment failed!')
exit(ret)
print('done')
# 数据处理
img_list = os.listdir(IMG_FOLDER)
for i in range(len(img_list)):
img_name = img_list[i]
img_path = os.path.join(IMG_FOLDER, img_name)
if not os.path.exists(img_path):
print("{} is not found", img_name)
continue
img_src = cv2.imread(img_path)
if img_src is None:
print("文件不存在\n")
# Due to rga init with (0,0,0), we using pad_color (0,0,0) instead of (114, 114, 114)
pad_color = (0,0,0)
# img = letter_box(im= img_src.copy(), new_shape=(MODEL_SIZE[1], MODEL_SIZE[0]), pad_color=(0,0,0))
img = resize_image(img_src.copy(), (640, 640), True) # direct resize
input = np.expand_dims(img, axis=0)
outputs = rknn.inference([input])
boxes, classes, scores = post_process(outputs)
img_p = img_src.copy()
if boxes is not None:
draw(img_p, boxes, scores, classes)
# 保存结果
if not os.path.exists(RESULT_PATH):
os.mkdir(RESULT_PATH)
result_path = os.path.join(RESULT_PATH, img_name)
cv2.imwrite(result_path, img_p)
print('Detection result save to {}'.format(result_path))
pass
rknn.release()resize的思路和YOLOv8一样，首先将图片等比缩放至长边为640，然后在短边的两边平均填充纯色色素。
坐标的缩放逻辑也是根据resize的逻辑逆推回去的。经过验证，rknn-toolkit2_2.0.0量化的int8模型精度确实优于rknn-toolkit2_1.5.0。
扫一扫
1402
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
