RV1103调用摄像头运行yolov5进行实时检测

前言

买了luckfox picorv1103开发板,摄像头是SC3336 3MP Camera (A)摄像头,参考RV1103 Luckfox Pico使用SPI NAND Flash烧录镜像焊了nand flash,烧录的buildroot系统。
在这里插入图片描述在这里插入图片描述

官网中有一些调用摄像头进行检测的实例:RKMPI-example,其中retinaface的两个例子luckfox_pico_rtsp_retinafaceluckfox_pico_rtsp_retinaface_osd均在开发板上能跑起来会有人脸的识别结果打印出来,但是rtsp推流时会显示下方的问题,在局域网下的 PC 使用 VLC 软件打开网络串流rtsp://172.32.0.93/live/0并不能获取图像,后续再看啥问题。
在这里插入图片描述

运行Yolov5模型

以官网实例中的luckfox_pico_rtsp_yolov5为基础,实现在rv1103上调用摄像头进行实时检测。

受内存限制,luckfox pico开发板是无法运行该yolov5例子的。
官网上实现的是RKNN推理图像rtsp推流实例,并且为了最大化提高rtsp推流的帧率,实例使用VI组件来实现摄像头图像捕获。在RKNN推理结果的标注上,采用两种方式来实现:

1)将标注好的图像上传到 VENC 组件进行编码传输
2)在图像上传到 VENC 组件后再使用 RGN 模块以打 OSD 的形式标注结果

其中yolov5例子采用的是第1种方法,工作流程为:
在这里插入图片描述
要在rv1103上跑起来yolov5需要将rtsp推流部分去掉,直接以输出打印的方式查看yolov5检测结果,从该流程中可以知道需要将VENC组件和rtsp部分去掉,像下面图中所示,直接VI输入VPSS然后输入rknn,其余都删除。
在这里插入图片描述
具体操作如下:

第一步:修改CMA_SIZE

rknn和摄像头部分都有用到CMA内存,系统烧录时默认的是24M,不太够,需要增加到30M。参考RV1103 Luckfox Pico使用SPI NAND Flash烧录镜像,在第2)步中的BoardConfig-SPI_NAND-Buildroot-RV1103_Luckfox_Pico_Pro-IPC.mk中需要调整这句代码:export RK_BOOTARGS_CMA_SIZE="30M",其他都一样的流程进行烧录。
adb连接上开发板后,可使用下面命令查看CMA的大小是否正确

[root@luckfox ]# grep -i cma /proc/meminfo
CmaTotal:          30720 kB
CmaAllocated:         12 kB
CmaReleased:       30708 kB
CmaFree:               0 kB

第二步:修改yolov5项目代码

源码:https://github.com/luckfox-eng29/luckfox_pico_rtsp_yolov5
很简单,将main.cc代码中与VENCrstp两部分的代码注释掉就行,注释完后代码为:

#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
#include <pthread.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/poll.h>
#include <time.h>
#include <unistd.h>
#include <vector>

#include "rtsp_demo.h"
#include "luckfox_mpi.h"
#include "yolov5.h"

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>

// disp size
int width    = 720;
int height   = 480;

// model size
int model_width = 640;
int model_height = 640;	
float scale ;
int leftPadding ;
int topPadding  ;

bool quit = false;
static void sigterm_handler(int sig) {
	fprintf(stderr, "signal %d\n", sig);
	quit = true;
}

cv::Mat letterbox(cv::Mat input)
{
	float scaleX = (float)model_width  / (float)width; //0.888
	float scaleY = (float)model_height / (float)height; //1.125	
	scale = scaleX < scaleY ? scaleX : scaleY;
	
	int inputWidth   = (int)((float)width * scale);
	int inputHeight  = (int)((float)height * scale);

	leftPadding = (model_width  - inputWidth) / 2;
	topPadding  = (model_height - inputHeight) / 2;	
	

	cv::Mat inputScale;
    cv::resize(input, inputScale, cv::Size(inputWidth,inputHeight), 0, 0, cv::INTER_LINEAR);	
	cv::Mat letterboxImage(640, 640, CV_8UC3,cv::Scalar(0, 0, 0));
    cv::Rect roi(leftPadding, topPadding, inputWidth, inputHeight);
    inputScale.copyTo(letterboxImage(roi));

	return letterboxImage; 	
}

void mapCoordinates(int *x, int *y) {	
	int mx = *x - leftPadding;
	int my = *y - topPadding;

    *x = (int)((float)mx / scale);
    *y = (int)((float)my / scale);
}


int main(int argc, char *argv[]) {
	RK_S32 s32Ret = 0; 
	int sX,sY,eX,eY; 
	
	// Ctrl-c quit
	signal(SIGINT, sigterm_handler);
		
	// Rknn model
	char text[16];
	rknn_app_context_t rknn_app_ctx;	
	object_detect_result_list od_results;
    int ret;
	const char *model_path = "./model/yolov5.rknn";
    memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t));	
	init_yolov5_model(model_path, &rknn_app_ctx);
	printf("init rknn model success!\n");
	init_post_process();

	//h264_frame	
	// VENC_STREAM_S stFrame;	
	// stFrame.pstPack = (VENC_PACK_S *)malloc(sizeof(VENC_PACK_S));
 	// VIDEO_FRAME_INFO_S h264_frame;
 	VIDEO_FRAME_INFO_S stVpssFrame;

	// rkaiq init
	RK_BOOL multi_sensor = RK_FALSE;	
	const char *iq_dir = "/etc/iqfiles";
	rk_aiq_working_mode_t hdr_mode = RK_AIQ_WORKING_MODE_NORMAL;
	//hdr_mode = RK_AIQ_WORKING_MODE_ISP_HDR2;
	SAMPLE_COMM_ISP_Init(0, hdr_mode, multi_sensor, iq_dir);
	SAMPLE_COMM_ISP_Run(0);

	// rkmpi init
	if (RK_MPI_SYS_Init() != RK_SUCCESS) {
		RK_LOGE("rk mpi sys init fail!");
		return -1;
	}

	// rtsp init	
	// rtsp_demo_handle g_rtsplive = NULL;
	// rtsp_session_handle g_rtsp_session;
	// g_rtsplive = create_rtsp_demo(554);
	// g_rtsp_session = rtsp_new_session(g_rtsplive, "/live/0");
	// rtsp_set_video(g_rtsp_session, RTSP_CODEC_ID_VIDEO_H264, NULL, 0);
	// rtsp_sync_video_ts(g_rtsp_session, rtsp_get_reltime(), rtsp_get_ntptime());
	
	// vi init
	vi_dev_init();
	vi_chn_init(0, width, height);

	// vpss init
	vpss_init(0, width, height);

	// bind vi to vpss
	MPP_CHN_S stSrcChn, stvpssChn;
	stSrcChn.enModId = RK_ID_VI;
	stSrcChn.s32DevId = 0;
	stSrcChn.s32ChnId = 0;

	stvpssChn.enModId = RK_ID_VPSS;
	stvpssChn.s32DevId = 0;
	stvpssChn.s32ChnId = 0;
	printf("====RK_MPI_SYS_Bind vi0 to vpss0====\n");
	s32Ret = RK_MPI_SYS_Bind(&stSrcChn, &stvpssChn);
	if (s32Ret != RK_SUCCESS) {
		RK_LOGE("bind 0 ch venc failed");
		return -1;
	}

	// venc init
	// RK_CODEC_ID_E enCodecType = RK_VIDEO_ID_AVC;
	// venc_init(0, width, height, enCodecType);

	// printf("venc init success\n");	
	
  while(!quit)
	{	
		// get vpss frame
		s32Ret = RK_MPI_VPSS_GetChnFrame(0,0, &stVpssFrame,-1);
		if(s32Ret == RK_SUCCESS)
		{
			void *data = RK_MPI_MB_Handle2VirAddr(stVpssFrame.stVFrame.pMbBlk);	
			//opencv	
			cv::Mat frame(height,width,CV_8UC3,data);			
			//cv::Mat frame640;
        	//cv::resize(frame, frame640, cv::Size(640,640), 0, 0, cv::INTER_LINEAR);	
			//letterbox
			cv::Mat letterboxImage = letterbox(frame);	
			memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width*model_height*3);		
			inference_yolov5_model(&rknn_app_ctx, &od_results);

			for(int i = 0; i < od_results.count; i++)
			{					
				//获取框的四个坐标 
				if(od_results.count >= 1)
				{
					object_detect_result *det_result = &(od_results.results[i]);
					printf("%s @ (%d %d %d %d) %.3f\n", coco_cls_to_name(det_result->cls_id),
							 det_result->box.left, det_result->box.top,
							 det_result->box.right, det_result->box.bottom,
							 det_result->prop);
	
					sX = (int)(det_result->box.left   );	
					sY = (int)(det_result->box.top 	  );	
					eX = (int)(det_result->box.right  );	
					eY = (int)(det_result->box.bottom );
					mapCoordinates(&sX,&sY);
					mapCoordinates(&eX,&eY);

					cv::rectangle(frame,cv::Point(sX ,sY),
								        cv::Point(eX ,eY),
										cv::Scalar(0,255,0),3);
					sprintf(text, "%s %.1f%%", coco_cls_to_name(det_result->cls_id), det_result->prop * 100);
					cv::putText(frame,text,cv::Point(sX, sY - 8),
												 cv::FONT_HERSHEY_SIMPLEX,1,
												 cv::Scalar(0,255,0),2);
				}
			}

			memcpy(data, frame.data, width * height * 3);					
		}
		
		// send stream
		// encode H264
		// RK_MPI_VENC_SendFrame(0, &stVpssFrame,-1);
		// rtsp
		// s32Ret = RK_MPI_VENC_GetStream(0, &stFrame, -1);
		// if(s32Ret == RK_SUCCESS)
		// {
		// 	if(g_rtsplive && g_rtsp_session)
		// 	{
		// 		//printf("len = %d PTS = %d \n",stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS);
				
		// 		void *pData = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
		// 		rtsp_tx_video(g_rtsp_session, (uint8_t *)pData, stFrame.pstPack->u32Len,
		// 					  stFrame.pstPack->u64PTS);
		// 		rtsp_do_event(g_rtsplive);
		// 	}
		// }

		// release frame 
		s32Ret = RK_MPI_VPSS_ReleaseChnFrame(0, 0, &stVpssFrame);
		if (s32Ret != RK_SUCCESS) {
			RK_LOGE("RK_MPI_VI_ReleaseChnFrame fail %x", s32Ret);
		}
		// s32Ret = RK_MPI_VENC_ReleaseStream(0, &stFrame);
		// if (s32Ret != RK_SUCCESS) {
		// 	RK_LOGE("RK_MPI_VENC_ReleaseStream fail %x", s32Ret);
		// }
		// memset(text,0,8);
	}



	printf("Release\n");
	RK_MPI_SYS_UnBind(&stSrcChn, &stvpssChn);
	
	RK_MPI_VI_DisableChn(0, 0);
	RK_MPI_VI_DisableDev(0);
	
	RK_MPI_VPSS_StopGrp(0);
	RK_MPI_VPSS_DestroyGrp(0);
	
	// RK_MPI_VENC_StopRecvFrame(0);
	// RK_MPI_VENC_DestroyChn(0);

	// free(stFrame.pstPack);

	// if (g_rtsplive)
	// 	rtsp_del_demo(g_rtsplive);
	SAMPLE_COMM_ISP_Stop(0);

	RK_MPI_SYS_Exit();

	// Release rknn model
    release_yolov5_model(&rknn_app_ctx);		
	deinit_post_process();
	
	return 0;
}

前提需要下载luckfox pico的sdk:https://github.com/LuckfoxTECH/luckfox-pico或者https://gitee.com/LuckfoxTECH/luckfox-pico,编译脚本:

export LUCKFOX_SDK_PATH=< 你的 Luckfox-pico SDK 路径>
cd /home/cw/zhouying/RV1103/luckfox_pico_rtsp_yolov5/
mkdir build
cd build
cmake ..
make && make install
# 在确定能adb连接开发板后将程序push进开发板
adb push ../luckfox_rtsp_yolov5_demo/ /

第三步:运行程序

adb shell连接开发板
top或者ps输出自启动的程序有哪些,需要kill掉一些程序释放出内存才能跑起来yolov5

[root@luckfox ]# top
Mem: 26820K used, 920K free, 228K shrd, 0K buff, 1604K cached
CPU:  17% usr  79% sys   0% nic   2% idle   0% io   0% irq   0% sirq
Load average: 11.21 3.00 1.02 2/106 426
  PID  PPID USER     STAT   VSZ %VSZ %CPU COMMAND
  251     1 root     S    92616 334%  51% rkipc -a /oem/usr/share/iqfiles
  401   399 root     D    31148 112%   8% smbd -D
  425   420 root     R     1796   6%   8% top
   36     2 root     SW       0   0%   8% [kswapd0]
  424   423 root     S     1800   6%   5% {default.script} /bin/sh /usr/share/ud
  190     2 root     SW       0   0%   5% [vcodec_thread_0]
  193     1 root     S     5368  19%   3% /usr/sbin/ntpd -g -p /var/run/ntpd.pid
  355     1 root     S    14392  52%   0% /usr/bin/adbd
  314     1 root     S     4852  17%   0% sshd: /usr/sbin/sshd [listener] 0 of 1
  420   355 root     S     1804   6%   0% /bin/sh -l
   65     1 root     S     1800   6%   0% {rcS} /bin/sh /etc/init.d/rcS
    1     0 root     S     1796   6%   0% init
  423   251 root     S     1792   6%   0% udhcpc -i usb0 -T 1 -A 0 -b -q
  399    65 root     S     1792   6%   0% {S91smb} /bin/sh /etc/init.d/S91smb st
   71     1 root     S     1788   6%   0% /sbin/syslogd -n
  320     1 root     S     1788   6%   0% /usr/sbin/telnetd -F
   75     1 root     S     1784   6%   0% /sbin/klogd -n
   86     1 root     S     1628   6%   0% /sbin/udevd -d
  426   424 root     D     1012   4%   0% /sbin/ifconfig usb0 up
   41     2 root     IW       0   0%   0% [kworker/u2:1-fl]
[root@luckfox ]# killall rkipc
[root@luckfox ]# killall smbd
[root@luckfox ]# killall sshd
[root@luckfox ]# killall ntpd
[root@luckfox ]# killall udhcpc
[root@luckfox ]# killall nmbd

然后就能运行程序了

[root@luckfox ]# cd luckfox_rtsp_yolov5_demo/
[root@luckfox luckfox_rtsp_yolov5_demo]# ./luckfox_rtsp_yolov5 
  index=0, name=images, n_dims=4, dims=[1, 640, 640, 3], n_elems=1228800, size=1228800, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
  index=0, name=output0, n_dims=4, dims=[1, 80, 80, 255], n_elems=1632000, size=1632000, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
  index=1, name=286, n_dims=4, dims=[1, 40, 40, 255], n_elems=408000, size=408000, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
  index=2, name=288, n_dims=4, dims=[1, 20, 20, 255], n_elems=102000, size=102000, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
model is NHWC input fmt
model input height=640, width=640, channel=3
init rknn model success!
load lable ./model/coco_80_labels_list.txt
rkaiq log level ff0
ID: 0, sensor_name is m00_b_sc3336 4-0030, iqfiles is /etc/iqfiles
rk_aiq_uapi2_sysctl_init/prepare succeed
rk_aiq_uapi2_sysctl_start succeed
rockit log path (null), log_size = 0, can use export rt_log_path=, export rt_log_size= change
log_file = (nil) 
RTVersion        00:02:18-532 {dump              :064} ---------------------------------------------------------
RTVersion        00:02:18-534 {dump              :065} rockit version: git-8cb4d25b8 Tue Feb 28 11:12:39 2023 +0800
RTVersion        00:02:18-534 {dump              :066} rockit building: built- 2023-02-28 15:23:19
RTVersion        00:02:18-535 {dump              :067} ---------------------------------------------------------
(null)           00:02:18-535 {log_level_init    :203} 

 please use echo name=level > /tmp/rt_log_level set log level 
        name: all cmpi mb sys vdec venc rgn vpss vgs tde avs wbc vo vi ai ao aenc adec
        log_level: 0 1 2 3 4 5 6 

rockit default level 4, can use export rt_log_level=x, x=0,1,2,3,4,5,6 change
(null)           00:02:18-536 {read_log_level    :093} text is all=4
(null)           00:02:18-536 {read_log_level    :095} module is all, log_level is 4
(null)           00:02:18-541 {monitor_log_level :144} #Start monitor_log_level thread, arg:(nil)
RTIsp3x          00:02:18-567 {ispInitDevice     :208} sensor name = m00_b_sc3336 4-0030
RTIsp3x          00:02:18-567 {ispInitDevice     :211} sensor_index = 0
RTIsp3x          00:02:18-573 {ispInitDevice     :208} sensor name = m00_b_sc3336 4-0030
RTIsp3x          00:02:18-573 {ispInitDevice     :211} sensor_index = 0
vi_dev_init
RKViDev          00:02:18-587 {vi_set_dev_attr   :440} VI_DEV_ATTR_S all parameter reserved
  === VI ATTRS INFO: ===  
        devId  :      0
        chnId  :      0
        buffcnt:      0
        buffSize:     0
        width:        0
        height:       0
        Maxwidth:     0
        Maxwidth:     0
        streaming:    0
RKViChn          00:02:18-593 {prepareRuntime    :334} ---------------------------------------------------------
RKViChn          00:02:18-593 {prepareRuntime    :336} vi version: 1.86, name:vvi
RKViChn          00:02:18-594 {prepareRuntime    :337} rockit-ko version: vmpi:fb2eed2be49e
RKViChn          00:02:18-594 {prepareRuntime    :338} rockit-ko building: -2023-02-09-11:04:49
RKViChn          00:02:18-594 {prepareRuntime    :339} ---------------------------------------------------------
RKViChn          00:02:18-600 {prepareRuntime    :358} mb pool create success, MBCnt= 2
cmpi             00:02:18-601 {createRuntime     :546} [non-WRAP MODE]: buff size = 518400
RTIsp3x          00:02:18-607 {ispInitDevice     :208} sensor name = m00_b_sc3336 4-0030
RTIsp3x          00:02:18-607 {ispInitDevice     :211} sensor_index = 0
RTDeviceV4L2     00:02:18-611 {open              :138} open video name(/dev/video11)
RKViChn          00:02:18-612 {setFrameRate      :1153} [vi] dev(0) ch(0) illegal param s32SrcFrameRate(0) s32DstFrameRate(0)
RTDeviceV4L2     00:02:18-614 {ispCameraInfo     :549} current device:/dev/video11 isn't compatible(cap:0x84201000) device,memoryType:4, retry:0
RTDeviceV4L2     00:02:18-615 {ispInitFormat     :726} ioctl VIDIOC_S_FMT OK
RKViChn          00:02:18-616 {start             :813} =========== vi Start startRuntime ===========
RTDeviceV4L2     00:02:18-616 {ispStreamOn       :440} do ispStreamOn start
RTDeviceV4L2     00:02:18-638 {ispStreamOn       :493} do ispStreamOn done
vpss_init
=== 0 ===
=== 1 ===
rga_api version 1.10.0_[2]
====RK_MPI_SYS_Bind vi0 to vpss0====
cmpi             00:02:18-721 {mb_get_buffer_by_i:422} allocated buffer(this=0x2e78b0, data=(nil), size=0, id=-1)
traffic light @ (27 239 81 297) 0.266
person @ (185 285 488 532) 0.465
laptop @ (533 380 619 500) 0.465
person @ (120 234 356 501) 0.294
laptop @ (464 344 535 430) 0.286
person @ (206 272 506 528) 0.424
person @ (577 297 640 513) 0.251
person @ (215 285 535 527) 0.592
person @ (190 291 427 523) 0.441
laptop @ (529 385 592 455) 0.364
person @ (190 294 423 522) 0.599
bottle @ (524 380 585 456) 0.263
person @ (229 351 445 529) 0.421
^Csignal 2
Release
RTDeviceV4L2     00:02:25-737 {ispStreamOff      :503} do ispStreamOff start
RTDeviceV4L2     00:02:25-802 {ispStreamOff      :514} do ispStreamOff done
RTDeviceV4L2     00:02:25-803 {close             :365} do RTDeviceV4L2 close
rk_aiq_uapi2_sysctl_stop enter
rk_aiq_uapi2_sysctl_deinit enter
MessageParser process loop exit!
rk_aiq_uapi2_sysctl_deinit exit
RKSockServer     00:02:26-568 {start             :162} accept failed
Release success
  • 20
    点赞
  • 14
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值