1. 概述
前面记录了一篇基于RTP协议的H264的推流器、接收器的实现过程,但是没有加上解码播放,这里记录一下如何实现解码和播放,也是在前面的基础之上实现的。前一篇的记录为【开源项目】基于RTP协议的H264码流发送器和接收器
在前文中,接收器将接收到的一系列数据包进行解析,并分成了一个个完整的帧,存储在内存之中。下面要将这些完整的帧进行解码成为yuv,并且播放。因此,需要添加解码和播放部分的代码。工程的代码结构为
工程的核心函数是udp_receive_packet(),这个函数的主要工作流程为:
(1)使用recvfrom()来接收数据流
(2)使用check_fragment()对数据流进行解析,并且拷贝到本地内存中
(3)使用h264_parse_packet()来解码所获取的数据,并且使用SDL进行播放
在处理接收的数据流时,我是按照一整个压缩帧进行存储的,例如存储一个完整的Intra帧或者一个完整的P帧,不包含后续帧的信息。然而,使用av_parser_parse2()进行分析时,会首先去寻找下一帧的起始地址来确定当前帧是否完整的输入了,如果没有找到,则很多分析流程不会执行。我在这里用了一个小技巧,在数据内存的最末尾加上一个伪起始码,让av_parser_parse2()确认已经接受了所有的数据,从而进行后续的分析
PS:不过其实这样写并不通用,只是为了配合我的整帧存储方式进行的微调。如果要实现通用的解码,在接收时去掉RTP的头,送入到av_parser_parse2()就可以了
2.工程
头文件的定义中,包括rtp header和rtp packet的定义,还定义了一个全局上下文信息结构体rtp_sdl_context_t
#pragma once
#include <stdio.h>
#include <WinSock2.h>
#include <string.h>
#include <stdint.h>
#include <inttypes.h>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavutil/imgutils.h"
#include "SDL2/SDL.h"
};
#undef main
#define RECV_DATA_SIZE 10000
#define MAX_BUFF_SIZE 32 * 1024 * 1024
#define RTP_MAX_PKT_SIZE 1400 // RTP数据包最大为
#define RTP_HEADER_SIZE 12
#define RTP_PADDING_SIZE 64
#define RTP_PACKET_START 1
#define RTP_PACKET_FRAGMENT 2
#define RTP_PACKET_END 3
#define RECV_STREAM_DOWNLOAD 0
#define RECV_YUV_DOWNLOAD 0
typedef struct rtp_header
{
// 存储时高位存储的是version
/* byte 0 */
uint8_t csrc_len : 4; /* expect 0 */
uint8_t extension : 1; /* expect 1 */
uint8_t padding : 1; /* expect 0 */
uint8_t version : 2; /* expect 2 */
/* byte 1 */
uint8_t payload_type : 7;
uint8_t marker : 1; /* expect 1 */
/* bytes 2, 3 */
uint16_t seq_num;
/* bytes 4-7 */
uint32_t timestamp;
/* bytes 8-11 */
uint32_t ssrc; /* stream number is used here. */
}rtp_header_t;
typedef struct rtp_packet
{
rtp_header_t rtp_h;
uint8_t rtp_data[RTP_MAX_PKT_SIZE + RTP_PADDING_SIZE];
}rtp_packet_t;
typedef struct rtp_context
{
int rtp_packet_cnt;
int rtp_buffer_size;
int rtp_frame_cnt;
int packet_loc; //
uint8_t* rtp_buffer_data;
}rtp_context_t;
typedef struct rtp_sdl_context
{
// video param
const AVCodec* codec;
AVCodecContext* codec_ctx;
AVCodecParserContext* parser_ctx ;
AVFrame* frame;
SwsContext* img_convert_ctx;
// SDL param
SDL_Window* window;
SDL_Renderer* render;
SDL_Texture* texture;
SDL_Rect rect;
}rtp_sdl_context_t;
cpp文件的定义和前文类似,只是增加了一些FFmpeg解码函数和SDL播放函数,重要部分有注释
#pragma warning(disable:4996)
#pragma comment(lib,"ws2_32.lib")
#include "include/udp_rtp_decode_sdl.h"
FILE* fp_yuv;
//int avc_init(const AVCodec* codec, AVCodecContext* codec_ctx, AVCodecParserContext* parser, AVFrame* frame)
int avc_init(rtp_sdl_context_t* rsc)
{
AVCodecID codec_id = AV_CODEC_ID_H264;
rsc->codec = avcodec_find_decoder(codec_id);
if (!rsc->codec)
{
printf("find decoder failed\n");
return -1;
}
rsc->codec_ctx = avcodec_alloc_context3(rsc->codec);
if (!rsc->codec_ctx)
{
printf("alloc context3 failed\n");
return -1;
}
rsc->parser_ctx = av_parser_init(codec_id);
if (!rsc->parser_ctx)
{
printf("parser ctx init failed\n");
return -1;
}
rsc->frame = av_frame_alloc();
if (!rsc->frame)
{
printf("alloc frame failed\n");
return -1;
}
if (avcodec_open2(rsc->codec_ctx, rsc->codec, NULL) < 0)
{
printf("Could not open codec\n");
return -1;
}
return 0;
}
int sdl_init(rtp_sdl_context_t* rsc)
{
if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {
printf("could not init sdl\n");
return -1;
}
const int screen_w = 1280, screen_h = 720;
const int pixel_w = 1280, pixel_h = 720;
//SDL 2.0 Support for multiple windows
rsc->window = SDL_CreateWindow("Play", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
screen_w, screen_h, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
if (!rsc->window) {
printf("SDL: could not create window - exiting:%s\n", SDL_GetError());
return -1;
}
rsc->render = SDL_CreateRenderer(rsc->window, -1, 0);
int pixformat = SDL_PIXELFORMAT_IYUV;
rsc->texture = SDL_CreateTexture(rsc->render, pixformat, SDL_TEXTUREACCESS_STREAMING, pixel_w, pixel_h);
int border = 0;
rsc->rect.x = 0 + border;
rsc->rect.y = 0 + border;
rsc->rect.w = screen_w - border * 2;
rsc->rect.h = screen_h - border * 2;
}
void av_free_all(rtp_sdl_context_t* rsc)
{
avcodec_free_context(&rsc->codec_ctx);
av_parser_close(rsc->parser_ctx);
av_frame_free(&rsc->frame);
}
int check_nalu_header(uint8_t data0)
{
int forbidden_zero_bit = data0 & 0x80; // 1bit
int nal_ref_idc = data0 & 0x60; // 2 bit
int nal_unit_type = data0 & 0x1F; // 5bit
if ((data0 & 0x80) == 1)
{
printf("forbidden zero bit should be 0\n");
return -1;
}
// printf("forbidden_zero_bit:%d, nal_ref_idc:%d, nal_unit_type:%d\n", forbidden_zero_bit, nal_ref_idc, nal_unit_type);
return nal_unit_type;
}
int check_fragment_nalu_header(rtp_context_t* rtp_ctx, uint8_t data0, uint8_t data1)
{
int nal_unit_type = check_nalu_header(data0);
int s, e, type;
int pos;
if (nal_unit_type == 28) // H264
{
s = data1 & 0x80; // S
e = data1 & 0x40; // E
type = data1 & 0x1F; // type
pos = data1 & 0xC0; // 1100 0000
switch (pos)
{
case 0x80:
rtp_ctx->packet_loc = RTP_PACKET_START;
break;
case 0x40:
rtp_ctx->packet_loc = RTP_PACKET_END;
break;
case 0x00:
rtp_ctx->packet_loc = RTP_PACKET_FRAGMENT;
break;
default: // error
printf("invalid packet loc\n");
return -1;
break;
}
}
return 0;
}
int find_nal_unit(uint8_t* buf, int size, int* nal_start, int* nal_end)
{
int i;
// find start
*nal_start = 0;
*nal_end = 0;
i = 0;
while ( //( next_bits( 24 ) != 0x000001 && next_bits( 32 ) != 0x00000001 )
(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01) &&
(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0 || buf[i + 3] != 0x01)
)
{
i++; // skip leading zero
if (i + 4 >= size) { return 0; } // did not find nal start
}
if (buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01) // ( next_bits( 24 ) != 0x000001 )
{
i++;
}
if (buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01) { /* error, should never happen */ return 0; }
i += 3;
*nal_start = i;
while ( //( next_bits( 24 ) != 0x000000 && next_bits( 24 ) != 0x000001 )
(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0) &&
(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01)
)
{
i++;
// FIXME the next line fails when reading a nal that ends exactly at the end of the data
if (i + 3 >= size) { *nal_end = size; return -1; } // did not find nal end, stream ended first
}
*nal_end = i;
return (*nal_end - *nal_start);
}
void set_default_rtp_context(rtp_context_t* rtp_ctx)
{
memset(rtp_ctx->rtp_buffer_data, 0, sizeof(rtp_ctx->rtp_buffer_size));
rtp_ctx->rtp_packet_cnt = 0;
rtp_ctx->rtp_buffer_size = 0;
rtp_ctx->packet_loc = 0;
}
// Check the data is fragment or not, if fragment, try to concate
int check_fragment(rtp_context_t* rtp_ctx, rtp_packet_t* rtp_pkt, uint8_t* data, int size)
{
int nal_start, nal_end;
int ret = 0;
int data_size = size - RTP_HEADER_SIZE;
find_nal_unit(data, data_size, &nal_start, &nal_end); // check NALU split pos
uint8_t data0 = data[nal_start];
uint8_t data1 = data[nal_start + 1];
uint8_t fu_indicator, fu_header;
if (nal_start > 0 && nal_start < 5) // single-fragment, maybe SPS, PPS or small size frame
{
fu_indicator = 0;
fu_header = 0;
ret = check_nalu_header(data0); // update nalu_type
rtp_ctx->rtp_buffer_data = (uint8_t*)realloc(rtp_ctx->rtp_buffer_data, (rtp_ctx->rtp_buffer_size + data_size) * sizeof(uint8_t));
memcpy(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, data, data_size);
#if STREAM_DOWNLOAD
fwrite(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, 1, data_size, fp_in);
#endif
fprintf(stdout, "rtp_ctx frame cnt:%d, frame_size:%d\n", rtp_ctx->rtp_frame_cnt, data_size);
rtp_ctx->rtp_frame_cnt++;
rtp_ctx->rtp_buffer_size += data_size;
}
else // multi-fragment
{
fu_indicator = data[0];
fu_header = data[1];
ret = check_fragment_nalu_header(rtp_ctx, fu_indicator, fu_header);
if (ret < 0)
{
printf("invalid nalu header\n");
return -1;
}
int real_data_size = data_size - 2;
rtp_ctx->rtp_buffer_data = (uint8_t*)realloc(rtp_ctx->rtp_buffer_data, (rtp_ctx->rtp_buffer_size + real_data_size) * sizeof(uint8_t));
if (!rtp_ctx->rtp_buffer_data)
{
printf("realloc rtp_buffer_data failed\n");
return -1;
}
memcpy(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, data + 2, real_data_size); // plus 2 to skip fu_indicator and fu_header
#if STREAM_DOWNLOAD
fwrite(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, 1, real_data_size, fp_in);
fflush(fp_in);
#endif
rtp_ctx->rtp_packet_cnt++;
rtp_ctx->rtp_buffer_size += real_data_size;
if (rtp_ctx->packet_loc == RTP_PACKET_END) // end of packet
{
fprintf(stdout, "rtp_ctx frame cnt:%d, frame_size:%d\n", rtp_ctx->rtp_frame_cnt, rtp_ctx->rtp_buffer_size);
rtp_ctx->rtp_frame_cnt++;
}
}
return 0;
}
// 伪造起始码
int forge_end_code(uint8_t* data, int size)
{
data = (uint8_t*)realloc(data, (size + 6) * sizeof(uint8_t));
if (!data)
{
printf("realloc end code failed\n");
return -1;
}
data[size] = 0x00;
data[size + 1] = 0x00;
data[size + 2] = 0x00;
data[size + 3] = 0x01;
data[size + 4] = 0x41;
data[size + 5] = 0x9A;
size += 6;
return size;
}
int h264_parse_packet(rtp_sdl_context_t* rsc, rtp_context_t* rtp_ctx, rtp_packet_t* rtp_pkt)
{
AVPacket* packet;
int ret = 0;
packet = av_packet_alloc();
if (!packet)
{
printf("alloc packet failed\n");
return -1;
}
// 添加伪起始码
uint8_t* buf_data = rtp_ctx->rtp_buffer_data;
int data_size = rtp_ctx->rtp_buffer_size;
data_size = forge_end_code(buf_data, data_size);
ret = av_parser_parse2(rsc->parser_ctx, rsc->codec_ctx, &packet->data, &packet->size,
buf_data, data_size, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
if (ret < 0) {
printf("parse packet failed, err:%d\n", ret);
return -1;
}
ret = avcodec_send_packet(rsc->codec_ctx, packet);
if (ret < 0)
{
printf("send packet failed\n");
return -1;
}
rsc->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
rsc->img_convert_ctx = sws_getContext(rsc->codec_ctx->width, rsc->codec_ctx->height, rsc->codec_ctx->pix_fmt,
rsc->codec_ctx->width, rsc->codec_ctx->height, AV_PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL);
while (ret >= 0) {
ret = avcodec_receive_frame(rsc->codec_ctx, rsc->frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return -1;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
exit(1);
}
//printf("saving frame:%d\n", rsc->codec_ctx->frame_num);
fflush(stdout);
#if RECV_YUV_DOWNLOAD
int size = rsc->frame->width * rsc->frame->height;
fwrite(rsc->frame->data[0], 1, size, fp_yuv);//Y
fwrite(rsc->frame->data[1], 1, size / 4, fp_yuv);//U
fwrite(rsc->frame->data[2], 1, size / 4, fp_yuv);//V
fflush(fp_yuv);
#endif
sws_scale(rsc->img_convert_ctx, (const unsigned char* const*)rsc->frame->data, rsc->frame->linesize, 0, rsc->codec_ctx->height,
rsc->frame->data, rsc->frame->linesize);
// SDL播放
SDL_UpdateYUVTexture(rsc->texture, &rsc->rect,
rsc->frame->data[0], rsc->frame->linesize[0],
rsc->frame->data[1], rsc->frame->linesize[1],
rsc->frame->data[2], rsc->frame->linesize[2]);
SDL_RenderClear(rsc->render);
SDL_RenderCopy(rsc->render, rsc->texture, NULL, &rsc->rect);
SDL_RenderPresent(rsc->render);
SDL_Delay(40); // delay 40ms
}
return 0;
}
// 接收数据包
int udp_recevie_packet(rtp_sdl_context_t* rsc, const char* url, int port)
{
WSADATA wsaData;
WORD sockVersion = MAKEWORD(2, 2);
int cnt = 0;
if (WSAStartup(sockVersion, &wsaData) != 0)
{
return 0;
}
SOCKET ser_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (ser_socket == INVALID_SOCKET)
{
ERROR("Invalid socket");
return -1;
}
int on = 1;
setsockopt(ser_socket, SOL_SOCKET, SO_REUSEADDR, (const char*)& on, sizeof(on));
sockaddr_in ser_addr;
ser_addr.sin_family = AF_INET;
ser_addr.sin_port = htons(port);
ser_addr.sin_addr.s_addr = inet_addr(url);
if (bind(ser_socket, (sockaddr*)& ser_addr, sizeof(ser_addr)) == SOCKET_ERROR)
{
printf("Bind socket addr error\n");
closesocket(ser_socket);
return -1;
}
sockaddr_in remote_addr;
int addr_len = sizeof(remote_addr);
char recv_data[RECV_DATA_SIZE];
rtp_context_t* rtp_ctx = (rtp_context_t*)calloc(1, sizeof(rtp_context_t));
if (!rtp_ctx)
{
printf("alloc rtp_ctx failed\n");
return -1;
}
rtp_packet_t* rtp_pkt = (rtp_packet_t*)calloc(1, sizeof(rtp_packet_t));
if (!rtp_pkt)
{
printf("alloc rtp_pkt failed\n");
return -1;
}
fprintf(stdout, "Listening on port:%d\n", port);
while (1)
{
// recvfrom接收传输过来的数据
int pkt_size = recvfrom(ser_socket, recv_data, RECV_DATA_SIZE, 0, (sockaddr*)& remote_addr, &addr_len);
if (pkt_size > 0)
{
memcpy(rtp_pkt, recv_data, pkt_size);
check_fragment(rtp_ctx, rtp_pkt, rtp_pkt->rtp_data, pkt_size); // check pkt data is fragment or not
rtp_header_t rtp_h = rtp_pkt->rtp_h;
char payload = rtp_h.payload_type;
if (rtp_ctx->packet_loc == RTP_PACKET_END)
{
switch (payload)
{
case 33: // mpegts
// mpegts_packet_parse((uint8_t*)rtp_data, parse_mpegts, payload, rtp_data_size); // TODO: add mpegts parser
printf("MPEGTS type\n");
break;
case 96: // h264
//printf("payload type:%s\n", "H264");
// 进行h264码流的解析
h264_parse_packet(rsc, rtp_ctx, rtp_pkt);
break;
default:
printf("Unknown type\n");
break;
}
// printf("[RTP PKT] %5d| %5s | %10u| %5d| %5d\n", cnt, payload_str, timestamp, seq_num, pkt_size);
set_default_rtp_context(rtp_ctx); // set default rtp ctx value
}
}
}
}
int main()
{
rtp_sdl_context_t* rsc = (rtp_sdl_context_t*)malloc(sizeof(rtp_sdl_context_t));
if (!rsc)
{
printf("malloc rsc failed\n");
return -1;
}
memset(rsc, 0, sizeof(rtp_sdl_context_t));
// 初始化参数
avc_init(rsc);
sdl_init(rsc);
// 如果要存储yuv信息可以设置为1
#if RECV_YUV_DOWNLOAD
fp_yuv = fopen("rtp_receive_yuv.yuv", "wb");
#endif
// 开始接收数据包
udp_recevie_packet(rsc, "127.0.0.1", 8880);
av_free_all(rsc);
#if RECV_YUV_DOWNLOAD
fclose(fp_yuv);
#endif
return 0;
}
3.测试
发送端
接收端
接收端播放正常,感觉可以后续改一改SDL的逻辑,让窗口变成可移动和可缩放的
4.小结
总体来说,这个功能的实现是比较简单的,不过使用了一个小的技巧,伪造了一个起始地址,如果代码格式要求不严格,可以凑合着用。如果要做成大的工程,应该将多个packet直接送入解码器,这样比较合理,也更符合FFmpeg的设计原则