利用AidLux实现热成像电力巡检项目作业展示及流程介绍
前言
对于刚入门的计算机视觉算法工程师而言,目标检测算法的数据集准备+训练+部署依然存在着不小的难度,而Aidlux的出现很好的解决了这一个问题。Aidlux可以将我们的安卓设备进一步改造成边缘计算设备。本文为参加aidlux训练营的学习和作业展示,用以记录该项目的整体流程,如有不当之处,请在评论区联系作者。项目完整代码和数据集均由aidlux提供,不知道放上是否合适,故只放上核心代码,完整资源可去aidlux开发者社区官网寻找。
本项目为基于Aidlux+r-retinanet+tflite,在安卓手机小米6上实现热成像电力训练项目。具体内容为:通过r-retinanet对绝缘子等电力设施进行旋转目标检测。
一、模型训练及onnx模型导出
博主没有训练r-retinanet模型的条件,因此使用的是训练营老师提供的onnx模型(感谢老师),通过下方代码将其导出为能在安卓设备上运行的tflite模型,核心代码如下。
def onnx_converter(onnx_model_path:str, output_path:str=None,
input_node_names:list=None, output_node_names:list=None,
need_simplify:bool=True, target_formats:list = ['keras', 'tflite'],
native_groupconv:bool=False,
weight_quant:bool=False, int8_model:bool=False, image_root:str=None,
int8_mean:list or float = [123.675, 116.28, 103.53], int8_std:list or float = [58.395, 57.12, 57.375])->float:
if not isinstance(target_formats, list) and 'keras' not in target_formats and 'tflite' not in target_formats:
raise KeyError("'keras' or 'tflite' should in list")
model_proto = load_onnx_modelproto(onnx_model_path, input_node_names, output_node_names, need_simplify)
keras_model = keras_builder(model_proto, native_groupconv)
if 'tflite' in target_formats:
tflite_model = tflite_builder(keras_model, weight_quant, int8_model, image_root, int8_mean, int8_std)
onnx_path, model_name = os.path.split(onnx_model_path)
if output_path is None:
output_path = onnx_path
output_path = os.path.join(output_path, model_name.split('.')[0])
keras_model_path = None
if 'keras' in target_formats:
keras_model_path = output_path + ".h5"
keras_model.save(keras_model_path)
LOG.info(f"keras model saved in {keras_model_path}")
tflite_model_path = None
if 'tflite' in target_formats:
tflite_model_path = output_path + ".tflite"
with open(tflite_model_path, "wb") as fp:
fp.write(tflite_model)
convert_result = {"keras":keras_model_path, "tflite":tflite_model_path, "keras_error":0, "tflite_error":0}
# ignore quantization model
if int8_model:
return convert_result
error_dict = {}
try:
error_dict = get_elements_error(model_proto, keras_model_path, tflite_model_path)
keras_error, tflite_error = error_dict.get("keras", None), error_dict.get("tflite", None)
if keras_error:
if keras_error > 1e-2:
LOG.error("h5 model elements' max error has reached {:^.4E}, but convert is done, please check {} carefully!".format(keras_error, keras_model_path))
elif keras_error > 1e-4:
LOG.warning("h5 model elements' max error is {:^.4E}, pass, h5 saved in {}".format(keras_error, keras_model_path))
else:
LOG.info("h5 model elements' max error is {:^.4E}, pass, h5 saved in {}".format(keras_error, keras_model_path))
if tflite_error:
if tflite_error > 1e-2:
LOG.error("tflite model elements' max error has reached {:^.4E}, but convert is done, please check {} carefully!".format(tflite_error, tflite_model_path))
elif tflite_error > 1e-4:
LOG.warning("tflite model elements' max error is {:^.4E}, pass, tflite saved in {}".format(tflite_error, tflite_model_path))
else:
LOG.info("tflite model elements' max error is {:^.4E}, pass, tflite saved in {}".format(tflite_error, tflite_model_path))
except:
LOG.warning("convert is successed, but model running is failed, please check carefully!")
convert_result["keras_error"] = error_dict.get("keras", None)
convert_result["tflite_error"] = error_dict.get("tflite", None)
return convert_result
import os
import sys
sys.path.append("/2T/001_AI/1003_YOLOv8/005_Misc/onnx2tflite-main")
from converter import onnx_converter
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
def onnx2tflite(onnx_path):
onnx_converter(
onnx_model_path = onnx_path,
need_simplify = False,
output_path = os.path.dirname(onnx_path),
target_formats = ['tflite'], # or ['keras'], ['keras', 'tflite']
weight_quant = False,
int8_model = False,
int8_mean = None,
int8_std = None,
image_root = None
)
if __name__ == "__main__":
onnx2tflite("./r-retinanet.onnx")
导出成功后,得到tflite模型。
二、vscode远程连接安卓设备进行aidlux开发
手机上需要先下载AidLux App,进入后点击Cloud_ip图标,会出现自己手机的aidlux ip,注意我们通过vscode远程连接时,端口号不是8000,而是9022。
1.remote-shh安装
在左侧扩展中搜索remote-shh,安装成功后左侧会出现小电脑图标.
按照顺序进行config配置
配置内容如下,HostName填上上面的IP地址,随后输入密码即可连接成功,将电脑上的代码传到手机里即可进行开发。
2.调用手机后置摄像头进行检测
aidlux是通过cvs包捕获摄像头图像的。
if __name__=="__main__":
''' 定义输入输出shape '''
in_shape = [1 * 640 * 800 * 3 * 4] # HWC, float32
out_shape = [1 * 53325 * 8 * 4] # 8400: total cells, 52 = 48(num_classes) + 4(xywh), float32
# out_shape = [1 * 55425 * 8 * 4] # 8400: total cells, 52 = 48(num_classes) + 4(xywh), float32
''' AidLite初始化 '''
aidlite = aidlite_gpu.aidlite()
''' 加载R-RetinaNet模型 '''
tflite_model = '/home/AidLux_Deploy/models/r-retinanet.tflite'
res = aidlite.ANNModel(tflite_model, in_shape, out_shape, 4, -1) # Infer on -1: cpu, 0: gpu, 1: mixed, 2: dsp
print(res)
'''
读取手机后置摄像头
'''
cap = cvs.VideoCapture(0)
frame_id = 0
while True:
frame = cap.read()
if frame is None:
continue
frame_id += 1
if frame_id % 30 != 0:
continue
time0 = time.time()
im, im_scales = process_img(frame, NCHW=False, ToTensor=False) # im: NHWC
''' 设定输入输出 '''
aidlite.setInput_Float32(im, 800, 640)
''' 启动推理 '''
aidlite.invoke()
''' 捕获输出 '''
preds = aidlite.getOutput_Float32(0)
post_process_img(preds,frame,im)
# print('ending')
3.过程中遇到的旋转框偏移问题
模型的输入大小是固定的800x640,博主一开始通过老师提供的代码直接运行时报错,debug时发现我的小米6经过预处理图像的尺寸变为852x640.然后想着直接resize到固定大小,应该就没问题了,结果检测发现结果明显存在偏移,如左图。
博主发现是检测结果的post_process_img模块,将预测结果对应回原图时用到了之前预处理的缩放因子,因此直接resize会出现这种状况,正好想到之前再跑yolov8算法时,通过letterbox方法是能正确显示目标框的,于是对代码中的尺寸处理和解码模块进行了替换,最终显示结果正常,如下图。
process模块代码修改如下
def letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):
# Resize and pad image while meeting stride-multiple constraints
shape = img.shape[:2] # current shape [height, width]
if isinstance(new_shape, int):
new_shape = (new_shape, new_shape)
# Scale ratio (new / old)
r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
if not scaleup: # only scale down, do not scale up (for better test mAP)
r = min(r, 1.0)
# Compute padding
ratio = r, r # width, height ratios
new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding
if auto: # minimum rectangle
dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding
elif scaleFill: # stretch
dw, dh = 0.0, 0.0
new_unpad = (new_shape[1], new_shape[0])
ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios
dw /= 2 # divide padding into 2 sides
dh /= 2
if shape[::-1] != new_unpad: # resize
img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)
top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border
return img, ratio, (dw, dh)
resized_img, ratio ,_= letterbox(img, (640, 800), stride=None, auto=False)
post_process_img模块代码修改如下
scale_boxes方法如下
def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):
if ratio_pad is None: # calculate from img0_shape
gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new
pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding
else:
gain = ratio_pad[0][0]
pad = ratio_pad[1]
boxes[..., [0, 2]] -= pad[0] # x padding
boxes[..., [1, 3]] -= pad[1] # y padding
boxes[..., :4] /= gain
# clip_boxes(boxes, img0_shape)
return boxes
总结
以上是参加Aidlux电力巡检训练营的记录,对整体流程初步了解,核心代码和模型都是老师提供,还需要进行深入研究,才能体会到项目中真正存在的坑。