RKMedia 是瑞芯微提供的媒体处理方案,可支持应用软件快速开发。rkmedia
为 C 接口,其实现均在easymedia
中。后者提供 C++ 接口。
rkmedia
中的行人检测示例,执行的操作为:
- 配置两个 RTSP 会话,初始化媒体处理平台;
- GetMediaBuffer 线程获取缓存并调用 rockx 进行检测跟踪;
- MainStream 线程绘制结果。
rkmedia_vi_person_detect_venc_rtsp_test.c
通过命令行传入配置文件。
load_cfg 加载 RTSP 会话配置。
if (argc != 2) {
printf("You must input the file of cfg.\n");
printf("example file: oem/usr/share/rtsp-nn.cfg.\n");
return -1;
}
load_cfg(argv[1]);
create_rtsp_demo 初始化 rtsp。
RK_MPI_SYS_Init 初始化各种类型的通道数组。
// init rtsp
printf("init rtsp\n");
g_rtsplive = create_rtsp_demo(554);
// init mpi
printf("init mpi\n");
RK_MPI_SYS_Init();
逐一创建会话,绑定输入通道。
其中第二个通道设置为VI_WORK_MODE_GOD_MODE
用于 RKNN 处理。
SAMPLE_COMMON_VI_Start 设置通道属性并启动。
SAMPLE_COMMON_VENC_Start 启动视频流编码,海思中对应函数为SAMPLE_COMM_VENC_Start
。
调用 RK_MPI_VI_StartStream 开启视频流,或者调用 RK_MPI_SYS_Bind 将数据源绑定到数据接收者。
rtsp_set_video 设置编码信息。
rtsp_get_reltime
rtsp_get_ntptime
rtsp_sync_video_ts
// create session
for (int i = 0; i < cfg.session_count; i++) {
cfg.session_cfg[i].session =
rtsp_new_session(g_rtsplive, cfg.session_cfg[i].path);
// VI create
printf("VI create\n");
cfg.session_cfg[i].stViChn.enModId = RK_ID_VI;
cfg.session_cfg[i].stViChn.s32ChnId = i;
if (i == RK_NN_INDEX)
SAMPLE_COMMON_VI_Start(&cfg.session_cfg[i], VI_WORK_MODE_GOD_MODE);
else
SAMPLE_COMMON_VI_Start(&cfg.session_cfg[i], VI_WORK_MODE_NORMAL);
// VENC create
printf("VENC create\n");
cfg.session_cfg[i].stVenChn.enModId = RK_ID_VENC;
cfg.session_cfg[i].stVenChn.s32ChnId = i;
SAMPLE_COMMON_VENC_Start(&cfg.session_cfg[i]);
if (i == DRAW_INDEX)
RK_MPI_VI_StartStream(0, cfg.session_cfg[i].stViChn.s32ChnId);
else
RK_MPI_SYS_Bind(&cfg.session_cfg[i].stViChn,
&cfg.session_cfg[i].stVenChn);
// rtsp video
printf("rtsp video\n");
switch (cfg.session_cfg[i].video_type) {
case RK_CODEC_TYPE_H264:
rtsp_set_video(cfg.session_cfg[i].session, RTSP_CODEC_ID_VIDEO_H264, NULL,
0);
break;
case RK_CODEC_TYPE_H265:
rtsp_set_video(cfg.session_cfg[i].session, RTSP_CODEC_ID_VIDEO_H265, NULL,
0);
break;
default:
printf("video codec not support.\n");
break;
}
rtsp_sync_video_ts(cfg.session_cfg[i].session, rtsp_get_reltime(),
rtsp_get_ntptime());
}
create_rknn_list 创建 rknn_list 列表。
在线程中调用 GetMediaBuffer 获取缓存并调用 rockx 进行检测跟踪。
另一线程中调用 MainStream 绘制结果。
create_rknn_list(&rknn_list_);
// Get the sub-stream buffer for humanoid recognition
pthread_t read_thread;
pthread_create(&read_thread, NULL, GetMediaBuffer, NULL);
// The mainstream draws a box asynchronously based on the recognition result
pthread_t main_stream_thread;
pthread_create(&main_stream_thread, NULL, MainStream, NULL);
设置SIGINT
信号的处理函数为 sig_proc。
调用 RK_MPI_SYS_GetMediaBuffer 获得 MEDIA_BUFFER。
rtsp_tx_video
RK_MPI_MB_ReleaseBuffer 释放缓存。
rtsp_do_event
signal(SIGINT, sig_proc);
while (g_flag_run) {
for (int i = 0; i < cfg.session_count; i++) {
MEDIA_BUFFER buffer;
// send video buffer
buffer = RK_MPI_SYS_GetMediaBuffer(
RK_ID_VENC, cfg.session_cfg[i].stVenChn.s32ChnId, 0);
if (buffer) {
rtsp_tx_video(cfg.session_cfg[i].session, RK_MPI_MB_GetPtr(buffer),
RK_MPI_MB_GetSize(buffer),
RK_MPI_MB_GetTimestamp(buffer));
RK_MPI_MB_ReleaseBuffer(buffer);
}
}
rtsp_do_event(g_rtsplive);
}
RK_MPI_SYS_UnBind 解除通道间的绑定。
RK_MPI_VI_DisableChn 关闭输入通道。
RK_MPI_VENC_DestroyChn 销毁编码通道。
rtsp_del_demo
destory_rknn_list
for (int i = 0; i < cfg.session_count; i++) {
if (i != DRAW_INDEX)
RK_MPI_SYS_UnBind(&cfg.session_cfg[i].stViChn,
&cfg.session_cfg[i].stVenChn);
RK_MPI_VI_DisableChn(0, cfg.session_cfg[i].stViChn.s32ChnId);
RK_MPI_VENC_DestroyChn(cfg.session_cfg[i].stVenChn.s32ChnId);
}
rtsp_del_demo(g_rtsplive);
destory_rknn_list(&rknn_list_);
return 0;
load_cfg
cfg
为全局变量,demo_cfg 类型,保存多个 Session 信息。MPP_CHN_S
// cfgline:
// path=%s video_type=%d width=%u height=%u
FILE *fp = fopen(cfg_file, "r");
char line[1024];
int count = 0;
if (!fp) {
fprintf(stderr, "open %s failed\n", cfg_file);
return -1;
}
memset(&cfg, 0, sizeof(cfg));
while (fgets(line, sizeof(line) - 1, fp)) {
const char *p;
// char codec_type[20];
memset(&cfg.session_cfg[count], 0, sizeof(cfg.session_cfg[count]));
if (line[0] == '#')
continue;
p = strstr(line, "path=");
if (!p)
continue;
if (sscanf(p, "path=%s", cfg.session_cfg[count].path) != 1)
continue;
if ((p = strstr(line, "video_type="))) {
if (sscanf(p,
"video_type=%d width=%u height=%u image_type=%u video_path=%s",
&cfg.session_cfg[count].video_type,
&cfg.session_cfg[count].u32Width,
&cfg.session_cfg[count].u32Height,
&cfg.session_cfg[count].enImageType,
cfg.session_cfg[count].videopath) == 0) {
printf("parse video file failed %s.\n", p);
}
}
if (cfg.session_cfg[count].video_type != RK_CODEC_TYPE_NONE) {
count++;
} else {
printf("parse line %s failed\n", line);
}
}
cfg.session_count = count;
fclose(fp);
dump_cfg();
return count;
RK_MPI_SYS_Init
LOG_INIT 初始化日志级别。
Reset_Channel_Table 初始化通道属性。
LOG_INIT();
Reset_Channel_Table(g_vi_chns, VI_MAX_CHN_NUM, RK_ID_VI);
Reset_Channel_Table(g_venc_chns, VENC_MAX_CHN_NUM, RK_ID_VENC);
Reset_Channel_Table(g_ai_chns, AI_MAX_CHN_NUM, RK_ID_AI);
Reset_Channel_Table(g_aenc_chns, AENC_MAX_CHN_NUM, RK_ID_AENC);
Reset_Channel_Table(g_ao_chns, AO_MAX_CHN_NUM, RK_ID_AO);
Reset_Channel_Table(g_algo_md_chns, ALGO_MD_MAX_CHN_NUM, RK_ID_ALGO_MD);
Reset_Channel_Table(g_algo_od_chns, ALGO_OD_MAX_CHN_NUM, RK_ID_ALGO_OD);
Reset_Channel_Table(g_rga_chns, RGA_MAX_CHN_NUM, RK_ID_RGA);
Reset_Channel_Table(g_adec_chns, ADEC_MAX_CHN_NUM, RK_ID_ADEC);
Reset_Channel_Table(g_vo_chns, VO_MAX_CHN_NUM, RK_ID_VO);
return RK_ERR_SYS_OK;
LOG_INIT
getenv 获取环境变量。
通过环境变量来设置日志级别。
char *ptr = NULL;
rkmedia_log_method = LOG_METHOD_PRINT;
#ifdef RKMEDIA_SUPPORT_MINILOG
ptr = getenv("RKMEDIA_LOG_METHOD");
if (ptr && strstr(ptr, "MINILOG")) {
fprintf(stderr, "##RKMEDIA Method: MINILOG\n");
__minilog_log_init("RKMEDIA", NULL, false, false, "RKMEDIA", "0.1");
rkmedia_log_method = LOG_METHOD_MINILOG;
} else {
fprintf(stderr, "##RKMEDIA Method: PRINT\n");
rkmedia_log_method = LOG_METHOD_PRINT;
}
#endif //#ifdef RKMEDIA_SUPPORT_MINILOG
ptr = getenv("RKMEDIA_LOG_LEVEL");
if (ptr && strstr(ptr, "DBG")) {
fprintf(stderr, "##RKMEDIA Log level: DBG\n");
rkmedia_log_level = LOG_LEVEL_DBG;
} else {
fprintf(stderr, "##RKMEDIA Log level: INFO\n");
rkmedia_log_level = LOG_LEVEL_INFO;
}
SAMPLE_COMMON_VI_Start
RK_MPI_VI_SetChnAttr 设置全局变量g_vi_chns
中对应的通道属性。
RK_MPI_VI_EnableChn 设置相应符号来启动通道。
VI_CHN_ATTR_S vi_chn_attr;
vi_chn_attr.u32BufCnt = 3;
vi_chn_attr.u32Width = session->u32Width;
vi_chn_attr.u32Height = session->u32Height;
vi_chn_attr.enWorkMode = mode;
vi_chn_attr.pcVideoNode = session->videopath;
vi_chn_attr.enPixFmt = session->enImageType;
RK_MPI_VI_SetChnAttr(0, session->stViChn.s32ChnId, &vi_chn_attr);
RK_MPI_VI_EnableChn(0, session->stViChn.s32ChnId);
RK_MPI_VI_SetChnAttr
设置全局变量g_vi_chns
中对应的通道属性。在通道关闭条件下才能设置。
if ((ViPipe < 0) || (ViChn < 0) || (ViChn > VI_MAX_CHN_NUM))
return -RK_ERR_VI_INVALID_CHNID;
if (!pstChnAttr || !pstChnAttr->pcVideoNode)
return -RK_ERR_VI_ILLEGAL_PARAM;
g_vi_mtx.lock();
if (g_vi_chns[ViChn].status != CHN_STATUS_CLOSED) {
g_vi_mtx.unlock();
return -RK_ERR_VI_BUSY;
}
memcpy(&g_vi_chns[ViChn].vi_attr.attr, pstChnAttr, sizeof(VI_CHN_ATTR_S));
g_vi_chns[ViChn].status = CHN_STATUS_READY;
g_vi_mtx.unlock();
return RK_ERR_SYS_OK;
RK_MPI_VI_EnableChn
在CHN_STATUS_READY
时可以启动。
if ((ViPipe < 0) || (ViChn < 0) || (ViChn > VI_MAX_CHN_NUM))
return -RK_ERR_VI_INVALID_CHNID;
g_vi_mtx.lock();
if (g_vi_chns[ViChn].status != CHN_STATUS_READY) {
g_vi_mtx.unlock();
return (g_vi_chns[ViChn].status > CHN_STATUS_READY) ? -RK_ERR_VI_EXIST
: -RK_ERR_VI_NOT_CONFIG;
}
LOG("\n%s %s: Enable VI[%d:%d]:%s, %dx%d Start...\n", LOG_TAG, __func__,
ViPipe, ViChn, g_vi_chns[ViChn].vi_attr.attr.pcVideoNode,
g_vi_chns[ViChn].vi_attr.attr.u32Width,
g_vi_chns[ViChn].vi_attr.attr.u32Height);
PARAM_STRING_APPEND 追加键和字符值到字符串。
PARAM_STRING_APPEND_TO 追加数值。
JoinFlowParam 拼接参数。
// Reading yuv from camera
std::string flow_name = "source_stream";
std::string flow_param;
PARAM_STRING_APPEND(flow_param, KEY_NAME, "v4l2_capture_stream");
std::string stream_param;
PARAM_STRING_APPEND_TO(stream_param, KEY_USE_LIBV4L2, 1);
PARAM_STRING_APPEND_TO(stream_param, KEY_CAMERA_ID, ViPipe);
PARAM_STRING_APPEND(stream_param, KEY_DEVICE,
g_vi_chns[ViChn].vi_attr.attr.pcVideoNode);
PARAM_STRING_APPEND(stream_param, KEY_V4L2_CAP_TYPE,
KEY_V4L2_C_TYPE(VIDEO_CAPTURE));
if (g_vi_chns[ViChn].vi_attr.attr.enBufType == VI_CHN_BUF_TYPE_MMAP) {
PARAM_STRING_APPEND(stream_param, KEY_V4L2_MEM_TYPE,
KEY_V4L2_M_TYPE(MEMORY_MMAP));
} else {
PARAM_STRING_APPEND(stream_param, KEY_V4L2_MEM_TYPE,
KEY_V4L2_M_TYPE(MEMORY_DMABUF));
}
PARAM_STRING_APPEND_TO(stream_param, KEY_FRAMES,
g_vi_chns[ViChn].vi_attr.attr.u32BufCnt);
PARAM_STRING_APPEND(
stream_param, KEY_OUTPUTDATATYPE,
ImageTypeToString(g_vi_chns[ViChn].vi_attr.attr.enPixFmt));
PARAM_STRING_APPEND_TO(stream_param, KEY_BUFFER_WIDTH,
g_vi_chns[ViChn].vi_attr.attr.u32Width);
PARAM_STRING_APPEND_TO(stream_param, KEY_BUFFER_HEIGHT,
g_vi_chns[ViChn].vi_attr.attr.u32Height);
flow_param = easymedia::JoinFlowParam(flow_param, 1, stream_param);
LOGD("\n#VI: v4l2 source flow param:\n%s\n", flow_param.c_str());
REFLECTOR 宏添加相应前缀。FlowReflector::Create 创建一个 Flow 对象。
尝试3次。
RK_S8 s8RetryCnt = 3;
while (s8RetryCnt > 0) {
g_vi_chns[ViChn].rkmedia_flow = easymedia::REFLECTOR(
Flow)::Create<easymedia::Flow>(flow_name.c_str(), flow_param.c_str());
if (g_vi_chns[ViChn].rkmedia_flow)
break; // Stop while
LOG("WARN: VI[%d]:\"%s\" buffer may be occupied by other modules or apps, "
"try again...\n",
ViChn, g_vi_chns[ViChn].vi_attr.attr.pcVideoNode);
s8RetryCnt--;
msleep(50);
}
if (!g_vi_chns[ViChn].rkmedia_flow) {
g_vi_mtx.unlock();
return -RK_ERR_VI_BUSY;
}
RkmediaChnInitBuffer 设置相应通道缓存的符号。
g_vi_chns[ViChn].luma_buf_mtx.lock();
g_vi_chns[ViChn].luma_buf_quit = false;
g_vi_chns[ViChn].luma_buf_start = false;
g_vi_chns[ViChn].luma_buf_mtx.unlock();
RkmediaChnInitBuffer(&g_vi_chns[ViChn]);
g_vi_chns[ViChn].rkmedia_flow->SetOutputCallBack(&g_vi_chns[ViChn],
FlowOutputCallback);
g_vi_chns[ViChn].status = CHN_STATUS_OPEN;
g_vi_mtx.unlock();
LOG("\n%s %s: Enable VI[%d:%d]:%s, %dx%d End...\n", LOG_TAG, __func__, ViPipe,
ViChn, g_vi_chns[ViChn].vi_attr.attr.pcVideoNode,
g_vi_chns[ViChn].vi_attr.attr.u32Width,
g_vi_chns[ViChn].vi_attr.attr.u32Height);
return RK_ERR_SYS_OK;
SAMPLE_COMMON_VENC_Start
设置属性并调整为就绪状态。
if ((ViPipe < 0) || (ViChn < 0) || (ViChn > VI_MAX_CHN_NUM))
return -RK_ERR_VI_INVALID_CHNID;
if (!pstChnAttr || !pstChnAttr->pcVideoNode)
return -RK_ERR_VI_ILLEGAL_PARAM;
g_vi_mtx.lock();
if (g_vi_chns[ViChn].status != CHN_STATUS_CLOSED) {
g_vi_mtx.unlock();
return -RK_ERR_VI_BUSY;
}
memcpy(&g_vi_chns[ViChn].vi_attr.attr, pstChnAttr, sizeof(VI_CHN_ATTR_S));
g_vi_chns[ViChn].status = CHN_STATUS_READY;
g_vi_mtx.unlock();
return RK_ERR_SYS_OK;
RK_MPI_VI_StartStream
Flow::StartStream 中waite_down_flow
拼错了。
if ((ViPipe < 0) || (ViChn < 0) || (ViChn > VI_MAX_CHN_NUM))
return -RK_ERR_VI_INVALID_CHNID;
g_vi_mtx.lock();
if (g_vi_chns[ViChn].status < CHN_STATUS_OPEN) {
g_vi_mtx.unlock();
return -RK_ERR_VI_BUSY;
}
if (!g_vi_chns[ViChn].rkmedia_flow) {
g_vi_mtx.unlock();
return -RK_ERR_VI_NOTREADY;
}
g_vi_chns[ViChn].rkmedia_flow->StartStream();
g_vi_mtx.unlock();
return RK_ERR_SYS_OK;
RK_MPI_SYS_Bind
根据源通道的模式和 id 从相应数组中取出 Flow 对象。
std::shared_ptr<easymedia::Flow> src;
std::shared_ptr<easymedia::Flow> sink;
RkmediaChannel *src_chn = NULL;
RkmediaChannel *dst_chn = NULL;
if (!pstSrcChn || !pstDestChn)
return -RK_ERR_SYS_ILLEGAL_PARAM;
LOG("\n%s %s: Bind Mode[%d]:Chn[%d] to Mode[%d]:Chn[%d]...\n", LOG_TAG,
__func__, pstSrcChn->enModId, pstSrcChn->s32ChnId, pstDestChn->enModId,
pstDestChn->s32ChnId);
switch (pstSrcChn->enModId) {
case RK_ID_VI:
src = g_vi_chns[pstSrcChn->s32ChnId].rkmedia_flow;
src_chn = &g_vi_chns[pstSrcChn->s32ChnId];
break;
case RK_ID_VENC:
src = g_venc_chns[pstSrcChn->s32ChnId].rkmedia_flow;
src_chn = &g_venc_chns[pstSrcChn->s32ChnId];
break;
case RK_ID_AI:
src = g_ai_chns[pstSrcChn->s32ChnId].rkmedia_flow;
src_chn = &g_ai_chns[pstSrcChn->s32ChnId];
break;
case RK_ID_AENC:
src = g_aenc_chns[pstSrcChn->s32ChnId].rkmedia_flow;
src_chn = &g_aenc_chns[pstSrcChn->s32ChnId];
break;
case RK_ID_RGA:
src = g_rga_chns[pstSrcChn->s32ChnId].rkmedia_flow;
src_chn = &g_rga_chns[pstSrcChn->s32ChnId];
break;
case RK_ID_ADEC:
src = g_adec_chns[pstSrcChn->s32ChnId].rkmedia_flow;
src_chn = &g_adec_chns[pstSrcChn->s32ChnId];
break;
default:
return -RK_ERR_SYS_NOT_SUPPORT;
}
if ((src_chn->status < CHN_STATUS_OPEN) || (!src)) {
LOG("ERROR: %s Src Mode[%d]:Chn[%d] is not ready!\n", __func__,
pstSrcChn->enModId, pstSrcChn->s32ChnId);
return -RK_ERR_SYS_NOTREADY;
}
switch (pstDestChn->enModId) {
case RK_ID_VENC:
sink = g_venc_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_venc_chns[pstDestChn->s32ChnId];
break;
case RK_ID_AO:
sink = g_ao_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_ao_chns[pstDestChn->s32ChnId];
break;
case RK_ID_AENC:
sink = g_aenc_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_aenc_chns[pstDestChn->s32ChnId];
break;
case RK_ID_ALGO_MD:
sink = g_algo_md_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_algo_md_chns[pstDestChn->s32ChnId];
break;
case RK_ID_ALGO_OD:
sink = g_algo_od_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_algo_od_chns[pstDestChn->s32ChnId];
break;
case RK_ID_RGA:
sink = g_rga_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_rga_chns[pstDestChn->s32ChnId];
break;
case RK_ID_ADEC:
sink = g_adec_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_adec_chns[pstDestChn->s32ChnId];
break;
case RK_ID_VO:
sink = g_vo_chns[pstDestChn->s32ChnId].rkmedia_flow;
dst_chn = &g_vo_chns[pstDestChn->s32ChnId];
break;
default:
return -RK_ERR_SYS_NOT_SUPPORT;
}
if ((dst_chn->status < CHN_STATUS_OPEN) || (!sink)) {
LOG("ERROR: %s Dst Mode[%d]:Chn[%d] is not ready!\n", __func__,
pstDestChn->enModId, pstDestChn->s32ChnId);
return -RK_ERR_SYS_NOTREADY;
}
Flow::AddDownFlow 绑定流。
Flow::SetOutputCallBack 设置输出的回调函数。
// Rkmedia flow bind
src->AddDownFlow(sink, 0, 0);
// Generally, after the previous Chn is bound to the next stage,
// FlowOutputCallback will be disabled.Because the VI needs to calculate
// the brightness, the VI still retains the FlowOutputCallback after
// binding the lower-level Chn.
if (src_chn->mode_id != RK_ID_VI)
src->SetOutputCallBack(NULL, NULL);
// change status frome OPEN to BIND.
src_chn->status = CHN_STATUS_BIND;
src_chn->bind_ref++;
dst_chn->status = CHN_STATUS_BIND;
dst_chn->bind_ref++;
return RK_ERR_SYS_OK;
GetMediaBuffer
从会话中获取帧尺寸,默认格式为 NV12。
librockx.so
中会有其运行时路径信息。
printf("#Start %s thread, arg:%p\n", __func__, arg);
// rockx
rockx_ret_t rockx_ret;
rockx_handle_t object_det_handle;
rockx_image_t input_image;
input_image.height = cfg.session_cfg[RK_NN_INDEX].u32Height;
input_image.width = cfg.session_cfg[RK_NN_INDEX].u32Width;
input_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12;
input_image.is_prealloc_buf = 1;
// create a object detection handle
rockx_config_t *config = rockx_create_config();
rockx_add_config(config, ROCKX_CONFIG_RKNN_RUNTIME_PATH,
RKNN_RUNTIME_LIB_PATH);
rockx_ret = rockx_create(&object_det_handle, ROCKX_MODULE_PERSON_DETECTION_V2,
config, sizeof(rockx_config_t));
if (rockx_ret != ROCKX_RET_SUCCESS) {
printf("init rockx module ROCKX_MODULE_PERSON_DETECTION_V2 error %d\n",
rockx_ret);
return NULL;
}
printf("ROCKX_MODULE_PERSON_DETECTION_V2 rockx_create success\n");
// create rockx_object_array_t for store result
rockx_object_array_t person_array;
memset(&person_array, 0, sizeof(person_array));
// create a object track handle
rockx_handle_t object_track_handle;
rockx_ret =
rockx_create(&object_track_handle, ROCKX_MODULE_OBJECT_TRACK, NULL, 0);
if (rockx_ret != ROCKX_RET_SUCCESS) {
printf("init rockx module ROCKX_MODULE_OBJECT_DETECTION error %d\n",
rockx_ret);
}
调用 RK_MPI_SYS_GetMediaBuffer 获得 MEDIA_BUFFER。
RK_MPI_MB_GetSize 获取大小。
RK_MPI_MB_GetPtr 获取虚拟指针。
getCurrentTimeMsec 记录当前时间,实现略繁琐。
跟踪功能不需要图像。
RK_MPI_MB_ReleaseBuffer 释放帧缓存。
MEDIA_BUFFER buffer = NULL;
while (g_flag_run) {
buffer = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, RK_NN_INDEX, -1);
if (!buffer) {
continue;
}
// printf("Get Frame:ptr:%p, fd:%d, size:%zu, mode:%d, channel:%d, "
// "timestamp:%lld\n",
// RK_MPI_MB_GetPtr(buffer), RK_MPI_MB_GetFD(buffer),
// RK_MPI_MB_GetSize(buffer),
// RK_MPI_MB_GetModeID(buffer), RK_MPI_MB_GetChannelID(buffer),
// RK_MPI_MB_GetTimestamp(buffer));
input_image.size = RK_MPI_MB_GetSize(buffer);
input_image.data = RK_MPI_MB_GetPtr(buffer);
// detect object
long nn_before_time = getCurrentTimeMsec();
rockx_ret = rockx_person_detect(object_det_handle, &input_image,
&person_array, NULL);
if (rockx_ret != ROCKX_RET_SUCCESS)
printf("rockx_person_detect error %d\n", rockx_ret);
long nn_after_time = getCurrentTimeMsec();
printf("Algorithm time-consuming is %ld\n",
(nn_after_time - nn_before_time));
// printf("person_array.count is %d\n", person_array.count);
rockx_object_array_t out_track_objects;
rockx_ret = rockx_object_track(object_track_handle, input_image.width,
input_image.height, 10, &person_array,
&out_track_objects);
if (rockx_ret != ROCKX_RET_SUCCESS)
printf("rockx_object_track error %d\n", rockx_ret);
// process result
if (person_array.count > 0) {
rknn_list_push(rknn_list_, getCurrentTimeMsec(), person_array);
int size = rknn_list_size(rknn_list_);
if (size >= MAX_RKNN_LIST_NUM)
rknn_list_drop(rknn_list_);
// printf("size is %d\n", size);
}
RK_MPI_MB_ReleaseBuffer(buffer);
}
// release
rockx_image_release(&input_image);
rockx_destroy(object_track_handle);
rockx_destroy(object_det_handle);
return NULL;
RK_MPI_SYS_GetMediaBuffer
根据模式从相应的通道数组中取出 RkmediaChannel
RkmediaChannel *target_chn = NULL;
switch (enModID) {
case RK_ID_VI:
if (s32ChnID < 0 || s32ChnID >= VI_MAX_CHN_NUM) {
LOG("ERROR: %s invalid VI ChnID[%d]\n", __func__, s32ChnID);
return NULL;
}
target_chn = &g_vi_chns[s32ChnID];
break;
case RK_ID_VENC:
if (s32ChnID < 0 || s32ChnID >= VENC_MAX_CHN_NUM) {
LOG("ERROR: %s invalid AENC ChnID[%d]\n", __func__, s32ChnID);
return NULL;
}
target_chn = &g_venc_chns[s32ChnID];
break;
case RK_ID_AI:
if (s32ChnID < 0 || s32ChnID >= AI_MAX_CHN_NUM) {
LOG("ERROR: %s invalid AI ChnID[%d]\n", __func__, s32ChnID);
return NULL;
}
target_chn = &g_ai_chns[s32ChnID];
break;
case RK_ID_AENC:
if (s32ChnID < 0 || s32ChnID > AENC_MAX_CHN_NUM) {
LOG("ERROR: %s invalid AENC ChnID[%d]\n", __func__, s32ChnID);
return NULL;
}
target_chn = &g_aenc_chns[s32ChnID];
break;
case RK_ID_RGA:
if (s32ChnID < 0 || s32ChnID > RGA_MAX_CHN_NUM) {
LOG("ERROR: %s invalid RGA ChnID[%d]\n", __func__, s32ChnID);
return NULL;
}
target_chn = &g_rga_chns[s32ChnID];
break;
case RK_ID_ADEC:
if (s32ChnID < 0 || s32ChnID > ADEC_MAX_CHN_NUM) {
LOG("ERROR: %s invalid RGA ChnID[%d]\n", __func__, s32ChnID);
return NULL;
}
target_chn = &g_adec_chns[s32ChnID];
break;
default:
LOG("ERROR: %s invalid modeID[%d]\n", __func__, enModID);
return NULL;
}
if (target_chn->status < CHN_STATUS_OPEN) {
LOG("ERROR: %s Mode[%d]:Chn[%d] in status[%d], "
"this operation is not allowed!\n",
__func__, enModID, s32ChnID, target_chn->status);
return NULL;
}
RkmediaChnPopBuffer 从缓存列表中取头元素。
return RkmediaChnPopBuffer(target_chn, s32MilliSec);
RkmediaChnPopBuffer
RkmediaPopPipFd 读取指定的 fd。
if (!ptrChn)
return NULL;
std::unique_lock<std::mutex> lck(ptrChn->buffer_mtx);
if (ptrChn->buffer_list.empty()) {
if (s32MilliSec < 0 && !ptrChn->buffer_cond_quit) {
ptrChn->buffer_cond.wait(lck);
} else if (s32MilliSec > 0) {
if (ptrChn->buffer_cond.wait_for(
lck, std::chrono::milliseconds(s32MilliSec)) ==
std::cv_status::timeout) {
LOG("INFO: %s: Mode[%d]:Chn[%d] get mediabuffer timeout!\n", __func__,
ptrChn->mode_id, ptrChn->chn_id);
return NULL;
}
} else {
return NULL;
}
}
MEDIA_BUFFER mb = NULL;
if (!ptrChn->buffer_list.empty()) {
if (ptrChn->wake_fd[0] > 0)
RkmediaPopPipFd(ptrChn->wake_fd[0]);
mb = ptrChn->buffer_list.front();
ptrChn->buffer_list.pop_front();
}
return mb;
RK_MPI_MB_GetSize
内部类型为 MEDIA_BUFFER_IMPLE
if (!mb)
return 0;
MEDIA_BUFFER_IMPLE *mb_impl = (MEDIA_BUFFER_IMPLE *)mb;
return mb_impl->size;
RK_MPI_MB_GetPtr
if (!mb)
return NULL;
MEDIA_BUFFER_IMPLE *mb_impl = (MEDIA_BUFFER_IMPLE *)mb;
return mb_impl->ptr;
RK_MPI_MB_ReleaseBuffer
MEDIA_BUFFER_IMPLE *mb_impl = (MEDIA_BUFFER_IMPLE *)mb;
if (!mb)
return -RK_ERR_SYS_ILLEGAL_PARAM;
if (mb_impl->rkmedia_mb)
mb_impl->rkmedia_mb.reset();
delete mb_impl;
return RK_ERR_SYS_OK;
MainStream
计算横纵坐标的缩放系数。
MEDIA_BUFFER buffer;
float x_rate = (float)cfg.session_cfg[DRAW_INDEX].u32Width /
(float)cfg.session_cfg[RK_NN_INDEX].u32Width;
float y_rate = (float)cfg.session_cfg[DRAW_INDEX].u32Height /
(float)cfg.session_cfg[RK_NN_INDEX].u32Height;
printf("x_rate is %f, y_rate is %f\n", x_rate, y_rate);
调用 RK_MPI_SYS_GetMediaBuffer 获得 MEDIA_BUFFER。
rknn_list_size 返回列表大小。
rknn_list_pop 弹出队列的首元素。
计算显示帧上的矩形框,nv12_border 绘制。
RK_MPI_SYS_SendMediaBuffer
RK_MPI_MB_ReleaseBuffer
while (g_flag_run) {
buffer = RK_MPI_SYS_GetMediaBuffer(
RK_ID_VI, cfg.session_cfg[DRAW_INDEX].stViChn.s32ChnId, -1);
if (!buffer)
continue;
// draw
if (rknn_list_size(rknn_list_)) {
long time_before;
rockx_object_array_t person_array;
memset(&person_array, 0, sizeof(person_array));
rknn_list_pop(rknn_list_, &time_before, &person_array);
// printf("time interval is %ld\n", getCurrentTimeMsec() - time_before);
for (int j = 0; j < person_array.count; j++) {
// printf("person_array.count is %d\n", person_array.count);
// const char *cls_name =
// OBJECT_DETECTION_LABELS_91[person_array.object[j].cls_idx];
// int left = person_array.object[j].box.left;
// int top = person_array.object[j].box.top;
// int right = person_array.object[j].box.right;
// int bottom = person_array.object[j].box.bottom;
// float score = person_array.object[j].score;
// printf("box=(%d %d %d %d) cls_name=%s, score=%f\n", left, top, right,
// bottom, cls_name, score);
int x = person_array.object[j].box.left * x_rate;
int y = person_array.object[j].box.top * y_rate;
int w = (person_array.object[j].box.right -
person_array.object[j].box.left) *
x_rate;
int h = (person_array.object[j].box.bottom -
person_array.object[j].box.top) *
y_rate;
if (x < 0)
x = 0;
if (y < 0)
y = 0;
while ((uint32_t)(x + w) >= cfg.session_cfg[DRAW_INDEX].u32Width) {
w -= 16;
}
while ((uint32_t)(y + h) >= cfg.session_cfg[DRAW_INDEX].u32Height) {
h -= 16;
}
printf("border=(%d %d %d %d)\n", x, y, w, h);
nv12_border((char *)RK_MPI_MB_GetPtr(buffer),
cfg.session_cfg[DRAW_INDEX].u32Width,
cfg.session_cfg[DRAW_INDEX].u32Height, x, y, w, h, 0, 0,
255);
}
}
// send from VI to VENC
RK_MPI_SYS_SendMediaBuffer(
RK_ID_VENC, cfg.session_cfg[DRAW_INDEX].stVenChn.s32ChnId, buffer);
RK_MPI_MB_ReleaseBuffer(buffer);
}
return NULL;
nv12_border
颜色转换成 YUV 值,填充4个内壁。
/* Set up the rectangle border size */
const int border = 5;
/* RGB convert YUV */
int Y, U, V;
Y = 0.299 * R + 0.587 * G + 0.114 * B;
U = -0.1687 * R + 0.3313 * G + 0.5 * B + 128;
V = 0.5 * R - 0.4187 * G - 0.0813 * B + 128;
/* Locking the scope of rectangle border range */
int j, k;
for (j = rect_y; j < rect_y + rect_h; j++) {
for (k = rect_x; k < rect_x + rect_w; k++) {
if (k < (rect_x + border) || k > (rect_x + rect_w - border) ||
j < (rect_y + border) || j > (rect_y + rect_h - border)) {
/* Components of YUV's storage address index */
int y_index = j * pic_w + k;
int u_index =
(y_index / 2 - pic_w / 2 * ((j + 1) / 2)) * 2 + pic_w * pic_h;
int v_index = u_index + 1;
/* set up YUV's conponents value of rectangle border */
pic[y_index] = Y;
pic[u_index] = U;
pic[v_index] = V;
}
}
}
return 0;
RK_MPI_SYS_SendMediaBuffer
RkmediaChannel
获得目标通道和锁。
RkmediaChannel *target_chn = NULL;
std::mutex *target_mutex = NULL;
switch (enModID) {
case RK_ID_VENC:
if (s32ChnID < 0 || s32ChnID >= VENC_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_venc_chns[s32ChnID];
target_mutex = &g_venc_mtx;
break;
case RK_ID_AENC:
if (s32ChnID < 0 || s32ChnID > AENC_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_aenc_chns[s32ChnID];
target_mutex = &g_aenc_mtx;
break;
case RK_ID_ALGO_MD:
if (s32ChnID < 0 || s32ChnID > ALGO_MD_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_algo_md_chns[s32ChnID];
target_mutex = &g_algo_md_mtx;
break;
case RK_ID_ALGO_OD:
if (s32ChnID < 0 || s32ChnID > ALGO_OD_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_algo_od_chns[s32ChnID];
target_mutex = &g_algo_od_mtx;
break;
case RK_ID_ADEC:
if (s32ChnID < 0 || s32ChnID > ADEC_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_adec_chns[s32ChnID];
target_mutex = &g_adec_mtx;
break;
case RK_ID_AO:
if (s32ChnID < 0 || s32ChnID > AO_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_ao_chns[s32ChnID];
target_mutex = &g_ao_mtx;
break;
case RK_ID_RGA:
if (s32ChnID < 0 || s32ChnID > RGA_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_rga_chns[s32ChnID];
target_mutex = &g_rga_mtx;
break;
case RK_ID_VO:
if (s32ChnID < 0 || s32ChnID > VO_MAX_CHN_NUM)
return -RK_ERR_SYS_ILLEGAL_PARAM;
target_chn = &g_vo_chns[s32ChnID];
target_mutex = &g_vo_mtx;
break;
default:
return -RK_ERR_SYS_NOT_SUPPORT;
}
MEDIA_BUFFER_IMPLE *mb = (MEDIA_BUFFER_IMPLE *)buffer;
target_mutex->lock();
if (target_chn->rkmedia_flow) {
target_chn->rkmedia_flow->SendInput(mb->rkmedia_mb, 0);
} else {
target_mutex->unlock();
return -RK_ERR_SYS_NOT_PERM;
}
target_mutex->unlock();
return RK_ERR_SYS_OK;
参考资料:
- 实时流协议(RTSP)简介
- RTSP协议
- 海思音频初始化的一个小细节
- 初始化MMP系统 范例
- RK3399 PRO快速开发 - 人脸特征点定位
- 海思 Hi35系列 MPP平台
- 嵌入式视频方案学习第四篇——系统基本控制
- 海思多媒体(MPP)开发(2)——视频输入(VI)
- 海思多媒体(MPP)开发(8)——获取VI中的YUV数据
- V536/V316项目Region模块用户指南/V1.0
- DRM(Direct Rendering Manager)学习简介
- Linux graphic subsystem(2)_DRI介绍
- Linux_GUI加速(2)_Linux中的DRM-KMS分析
- Linux DRM(二)基本概念和特性
- Linux DRM - Direct Rendering Manager
- 如何查看rtsp 视音频的编码 codec类型
- 海思HI35xx平台软件开发快速入门之H264解码实例
- 海思HiMPP视频编解码开发文档
- HiMPP V3.0 媒体处理软件笔记(二)
- 海思媒体处理平台VI视频输入模块
- sample_venc解析
- MPP模块及sample_venc分析
- 嵌入式视频方案学习第七篇——视频输入VI一般的初始化流程
- NTP和RTP时间戳
- RTP音视频同步中NTP的作用
- va_start和va_end使用详解
- RKMedia性能测试总结
- mpp头文件整理&缩写说明