原文地址:http://blog.csdn.net/qq_41051855/article/details/79051667
ffmpeg的解码过程在前面已经稍微总结了下,这里主要是测试一下用ffmpeg如何进行实时的解码。
在解码之前,我们先做好准备工作,调用摄像头。编码的过程中,进行入队出队操作,出队后的数据交给解码器,进行解码。
接下来依次介绍各个模块。
1.调用摄像头:
- VideoCapture capture(0);
- int w = capture.get(CV_CAP_PROP_FRAME_WIDTH);
- int h = capture.get(CV_CAP_PROP_FRAME_HEIGHT);
- int yuv_bufLen = w * h * 3 / 2;
- unsigned char* pYuvBuf = new unsigned char[yuv_bufLen];
- cout << "Frame size : " << w << " x " << h << endl;
- namedWindow("opencamera", CV_WINDOW_AUTOSIZE);
- while (1)
- {
- Mat frame;
- capture >> frame;
- imshow("opencamera", frame);
- if (waitKey(30) == 27) break;
- }
2.编码过程:
- DWORD WINAPI x264_encode(LPVOID lparam)
- {
- VideoCapture capture(0);
- if (!capture.isOpened())
- {
- cout << "Cannot open the video cam" << endl;
- return -1;
- }
- int w = capture.get(CV_CAP_PROP_FRAME_WIDTH);
- int h = capture.get(CV_CAP_PROP_FRAME_HEIGHT);
- result_link_type* result_link = (result_link_type*)lparam;
- int yuv_bufLen = w * h * 3 / 2;
- //unsigned char* pYuvBuf = new unsigned char[yuv_bufLen];
- //int fps =25;
- size_t yuv_size = w * h * 3 / 2;
- x264_t *encoder;
- x264_picture_t pic_in, pic_out;
- uint8_t *yuv_buffer;
- x264_param_t param;
- //x264_param_default_preset(¶m, "veryfast", "zerolatency"); //为结构体param赋默认值
- x264_param_default_preset(¶m, "veryfast", "animation");
- //param.i_threads = 1; //并行编码多帧
- param.i_width = w; //视频图像的宽
- param.i_height = h;
- // param.i_fps_num = fps; //帧率分子
- //param.i_fps_den = 1; //帧率分母 , fps_num / fps_den = 帧率
- //param.i_keyint_max = 50; //IDR帧之间的间隔
- //param.b_intra_refresh = 1; //是否使用周期帧内刷新IDR帧
- //param.b_annexb = 1; //加前缀码0x00000001
- //param.rc.b_mb_tree = 0; //实时编码必须为0,否则有延迟
- //param.b_sliced_threads = 0;
- //x264_param_apply_profile(¶m, "baseline"); //编码器的参数,使用baseline编码,可以跟上面的参数做冲突比较
- encoder = x264_encoder_open(¶m); //打开编码器,初始化param
- #if 1
- x264_picture_alloc(&pic_in, X264_CSP_I420, w, h); //为pic_in分配内存
- yuv_buffer = (uint8_t*)malloc(yuv_size); //给yuv_buffer分配内存
- pic_in.img.plane[0] = yuv_buffer; //pic_in的三通道分别赋值
- pic_in.img.plane[1] = pic_in.img.plane[0] + w * h;
- pic_in.img.plane[2] = pic_in.img.plane[1] + w * h / 4;
- int64_t i_pts = 0;
- x264_nal_t *nals;
- int nnal;
- FILE *fp_out = fopen("test.h264", "wb");
- if (!fp_out)
- {
- printf("Could not open output 264 file\n");
- return -1;
- }
- #if 1
- FILE* pFileOut = fopen("test.yuv", "w+");
- if (!pFileOut)
- {
- printf("Could not open input yuv file\n");
- return -1;
- }
- #endif
- cout << "Frame size : " << w << " x " << h << endl;
- namedWindow("opencamera", CV_WINDOW_AUTOSIZE);
- Mat frame;
- while (1)
- {
- capture >> frame; //摄像头处抓取一帧
- imshow("opencamera", frame); //显示
- //if (waitKey(30) == 27) break;
- waitKey(1);
- cv::Mat yuvImg;
- cv::cvtColor(frame, yuvImg, CV_BGR2YUV_I420); //YUV转RGB
- memcpy(yuv_buffer, yuvImg.data, yuv_bufLen*sizeof(unsigned char)); //YUV数据复制到yuv_buffer中
- //fwrite(yuv_buffer, yuv_bufLen*sizeof(unsigned char), 1, pFileOut); //YUV写入本地
- //while (fread(yuv_buffer, 1, yuv_size, inf) > 0)
- //{
- pic_in.i_pts = i_pts++;
- x264_encoder_encode(encoder, &nals, &nnal, &pic_in, &pic_out); //编码一帧数据
- x264_nal_t *nal;
- int j = 0;
- struct result_node_datatype *result_node = new struct result_node_datatype;
- result_node->result = new unsigned char[800000];
- memset(result_node->result, '\0', 800000);
- result_node->size = 0;
- for (nal = nals; nal < nals + nnal; nal++)
- {
- //fwrite(nal->p_payload, 1, nal->i_payload, fp_out); //产生的NAL保存在本地
- //result_node->size += nal->i_payload;
- //memcpy(result_node->result, nal->p_payload, nal->i_payload);
- //cout << "nal->i_payload = " <<nal->i_payload<< endl;
- //j = j + nal->i_payload;
- //result_push(result_link, result_node);
- //cout << "in for(nal): j = "<<j << endl;
- memcpy(result_node->result + j, nal->p_payload, nal->i_payload);
- j = j + nal->i_payload;
- }
- result_node->size = j;
- cout << "result_node->size = " << result_node->size << endl;
- result_push(result_link, result_node);
- }
- x264_encoder_close(encoder); //关闭编码器
- //fclose(inf);
- //free(yuv_buffer);
- //fclose(pFileOut);
- //delete[] pYuvBuf;
- //Sleep(100);
- #endif
- return NULL;
- }
X264编码的过程可以参考这里:点击打开链接。
需要注意的是,我们定义了一个为0的值j。编码产生后的NAL单元个数是nnal,编码后数据的起始地址是nal->p_payload,长度是nal->i_payload。增加j的原因是想把得到的一个个NAL单元累加在一起,组成一个完整帧的数据,最后一帧的长度就是j,然后将得到的一帧数据与长度送入队列,这是一个线程函数。对解码器来说,只有接收到完整的一帧,才能成功解码。
3.队列函数:
- void result_push(result_link_type* result_link, result_node_datatype * result_node) //入队操作
- {
- if (result_link->head == NULL)
- {
- result_link->head = result_node;
- result_link->end = result_link->head;
- result_link->result_num++;
- // cout << "0: result_link->result_num++" << endl;
- }
- else
- {
- result_link->end->next = result_node;
- result_link->end = result_node;
- result_link->result_num++;
- // cout << "1: result_link->result_num++" << endl;
- }
- }
- struct result_node_datatype* result_pop(result_link_type* result_link) //出队操作
- {
- struct result_node_datatype* tmp_node;
- if (result_link->head == NULL)
- return NULL;
- else if (result_link->head == result_link->end)
- {
- // cout << "result_link->head == result_link->end " << endl;
- return NULL;
- }
- else
- {
- tmp_node = result_link->head;
- result_link->head = result_link->head->next;
- result_link->result_num--;
- //cout << "result_link->result_num--" << endl;
- return tmp_node;
- }
- }
4.解码过程:
解码之前,要添加标志位0001。
- bool get_h264_data(uchar* buf,int in_len,uchar* out_buf, int &out_len)
- {
- char nalu[4] = { 0x00, 0x00, 0x00, 0x01 };
- memcpy(out_buf, nalu, 4);
- out_buf += 4;
- memcpy(out_buf, buf, in_len);
- out_len = in_len + 4;
- // cout << "out_len = " <<out_len<< endl;
- return true;
- }
解码过程:
- int main(int argc, char* argv[])
- {
- HANDLE thread1;
- result_link_type *result_link = new result_link_type;
- result_link->head = result_link->end = NULL;
- result_link->result_num = 0;
- thread1 = CreateThread(NULL, 0, x264_encode, (LPVOID)result_link, 0, NULL);
- Sleep(1);
- //system("pause");
- #if 1
- Mat pCvMat;
- AVCodec *pCodec;
- AVCodecContext *pCodecCtx = NULL;
- AVCodecParserContext *pCodecParserCtx = NULL;
- int frame_count;
- FILE *fp_in;
- FILE *fp_out;
- AVFrame *pFrame, *pFrameYUV;
- uint8_t *out_buffer;
- // const int in_buffer_size = 4096;
- const int in_buffer_size = 800000;
- //uint8_t in_buffer[in_buffer_size + FF_INPUT_BUFFER_PADDING_SIZE] = { 0 };
- uint8_t in_buffer[in_buffer_size];
- memset(in_buffer, 0, sizeof(in_buffer));
- uint8_t *cur_ptr;
- int cur_size;
- AVPacket packet;
- int ret, got_picture;
- int y_size;
- AVCodecID codec_id = AV_CODEC_ID_H264;
- // char filepath_in[] = "test.h264";
- // char filepath_out[] = "1.yuv";
- int first_time = 1;
- struct SwsContext *img_convert_ctx;
- //av_log_set_level(AV_LOG_DEBUG);
- avcodec_register_all();
- pCodec = avcodec_find_decoder(codec_id);
- if (!pCodec) {
- printf("Codec not found\n");
- return -1;
- }
- pCodecCtx = avcodec_alloc_context3(pCodec);
- if (!pCodecCtx){
- printf("Could not allocate video codec context\n");
- return -1;
- }
- pCodecParserCtx = av_parser_init(codec_id);
- if (!pCodecParserCtx){
- printf("Could not allocate video parser context\n");
- return -1;
- }
- if (pCodec->capabilities&CODEC_CAP_TRUNCATED)
- pCodecCtx->flags |= CODEC_FLAG_TRUNCATED; /* we do not send complete frames */
- if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {
- printf("Could not open codec\n");
- return -1;
- }
- #if 0
- //Input File
- fp_in = fopen(filepath_in, "rb");
- if (!fp_in) {
- printf("Could not open input stream\n");
- return -1;
- }
- //Output File
- fp_out = fopen(filepath_out, "wb");
- if (!fp_out) {
- printf("Could not open output YUV file\n");
- return -1;
- }
- #endif
- pFrame = av_frame_alloc();
- av_init_packet(&packet);
- AVFrame* pFrameBGR = av_frame_alloc(); //存储解码后转换的RGB数据
- // 保存BGR,opencv中是按BGR来保存的
- int size;
- //cout << "pCodecCtx->width = " << pCodecCtx->width << "\npCodecCtx->height = " << pCodecCtx->height << endl;
- //pCvMat.create(cv::Size(pCodecCtx->width, pCodecCtx->height), CV_8UC3);
- struct result_node_datatype *result_node2 = NULL;
- int out_len;
- while (1)
- {
- // cur_size = fread(in_buffer, 1, in_buffer_size, fp_in);
- // cout << "result_link->size = " << result_link->result_num << endl;
- result_node2 = result_pop(result_link);
- if (result_node2 == NULL)
- {
- Sleep(1);
- // cout << "result_node2 is NULL" << endl;
- continue;
- }
- //cur_size = result_node2->size;
- //cout<<"after result_pop()" << endl;
- get_h264_data(result_node2->result, result_node2->size, in_buffer, out_len);
- //cur_size = result_node2->size;
- cur_size = out_len;
- cout << "cur_size = " << cur_size << endl;
- if (cur_size == 0)
- break;
- cur_ptr = in_buffer;
- //cur_ptr = result_node2->result;
- while (cur_size>0){
- int len = av_parser_parse2(
- pCodecParserCtx, pCodecCtx,
- &packet.data, &packet.size,
- cur_ptr, cur_size,
- AV_NOPTS_VALUE, AV_NOPTS_VALUE, AV_NOPTS_VALUE);
- cur_ptr += len;
- cur_size -= len;
- if (packet.size == 0)
- continue;
- //Some Info from AVCodecParserContext
- printf("Packet Size:%6d\t", packet.size);
- switch (pCodecParserCtx->pict_type){
- case AV_PICTURE_TYPE_I: printf("Type: I\t"); break;
- case AV_PICTURE_TYPE_P: printf("Type: P\t"); break;
- case AV_PICTURE_TYPE_B: printf("Type: B\t"); break;
- default: printf("Type: Other\t"); break;
- }
- printf("Output Number:%4d\t", pCodecParserCtx->output_picture_number);
- printf("Offset:%8ld\n", pCodecParserCtx->cur_offset);
- ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, &packet);
- if (ret < 0) {
- printf("Decode Error.(解码错误)\n");
- return ret;
- }
- if (got_picture) {
- if (first_time){
- printf("\nCodec Full Name:%s\n", pCodecCtx->codec->long_name);
- printf("width:%d\nheight:%d\n\n", pCodecCtx->width, pCodecCtx->height);
- //SwsContext
- //img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt,
- // pCodecCtx->width, pCodecCtx->height, PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL);
- img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_BGR24, SWS_BICUBIC, NULL, NULL, NULL);
- //pFrameYUV = av_frame_alloc();
- //out_buffer = (uint8_t *)av_malloc(avpicture_get_size(PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height));
- //avpicture_fill((AVPicture *)pFrameYUV, out_buffer, PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height);
- //y_size = pCodecCtx->width*pCodecCtx->height;
- //size = avpicture_get_size(AV_PIX_FMT_BGR24, pCodecCtx->width, pCodecCtx->height);
- size = avpicture_get_size(AV_PIX_FMT_BGR24, pCodecCtx->width, pCodecCtx->height);
- out_buffer = (uint8_t *)av_malloc(size);
- avpicture_fill((AVPicture *)pFrameBGR, out_buffer, AV_PIX_FMT_BGR24, pCodecCtx->width, pCodecCtx->height); // allocator memory for BGR buffer
- cout << "pCodecCtx->width = " << pCodecCtx->width << "\npCodecCtx->height = " << pCodecCtx->height << endl;
- pCvMat.create(cv::Size(pCodecCtx->width, pCodecCtx->height), CV_8UC3);
- first_time = 0;
- }
- printf("Succeed to decode 1 frame!\n");
- //sws_scale(img_convert_ctx, (const uint8_t* const*)pFrame->data, pFrame->linesize, 0, pCodecCtx->height,pFrameYUV->data, pFrameYUV->linesize);
- sws_scale(img_convert_ctx, (const uint8_t* const*)pFrame->data, pFrame->linesize, 0, pCodecCtx->height, pFrameBGR->data, pFrameBGR->linesize);
- //fwrite(pFrameYUV->data[0], 1, y_size, fp_out); //Y
- //fwrite(pFrameYUV->data[1], 1, y_size / 4, fp_out); //U
- //fwrite(pFrameYUV->data[2], 1, y_size / 4, fp_out); //V
- cout << "size = " << size << endl;
- memcpy(pCvMat.data, out_buffer, size);
- imshow("RGB", pCvMat);
- waitKey(1);
- }
- }
- }
- system("pause");
- //Flush Decoder
- packet.data = NULL;
- packet.size = 0;
- #if 0
- while (1){
- ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, &packet);
- if (ret < 0) {
- printf("Decode Error.(解码错误)\n");
- return ret;
- }
- if (!got_picture)
- break;
- if (got_picture) {
- printf("Flush Decoder: Succeed to decode 1 frame!\n");
- sws_scale(img_convert_ctx, (const uint8_t* const*)pFrame->data, pFrame->linesize, 0, pCodecCtx->height,
- pFrameYUV->data, pFrameYUV->linesize);
- fwrite(pFrameYUV->data[0], 1, y_size, fp_out); //Y
- fwrite(pFrameYUV->data[1], 1, y_size / 4, fp_out); //U
- fwrite(pFrameYUV->data[2], 1, y_size / 4, fp_out); //V
- }
- }
- #endif
- // fclose(fp_in);
- // fclose(fp_out);
- sws_freeContext(img_convert_ctx);
- av_parser_close(pCodecParserCtx);
- //av_frame_free(&pFrameYUV);
- av_frame_free(&pFrameBGR);
- av_frame_free(&pFrame);
- avcodec_close(pCodecCtx);
- av_free(pCodecCtx);
- #endif
- return 0;
- }
解码流程以及基本参数和函数的意义,可以参考链接:点击打开链接。
解码成功后,got_picture为非零值,在这个判断里面,解码出的数据转为RGB格式,并用opencv显示出来。
运行效果图:
运行的过程中,对比上面的效果图,发现解码后的图像比摄像头要延迟几秒钟,这是因为在编码的过程,我们使用了参数animation,编码开始前会缓存几帧,再开始,这样帧间编码效果好,压缩效率高。如果使用zerolatency(零延时),基本不会有延时效果,但是压缩效果不太好。
完整测试项目的代码下载地址:点击打开链接