【流媒体】基于RTP协议的H264播放器

1. 概述

前面记录了一篇基于RTP协议的H264的推流器、接收器的实现过程，但是没有加上解码播放，这里记录一下如何实现解码和播放，也是在前面的基础之上实现的。前一篇的记录为【开源项目】基于RTP协议的H264码流发送器和接收器

在前文中，接收器将接收到的一系列数据包进行解析，并分成了一个个完整的帧，存储在内存之中。下面要将这些完整的帧进行解码成为yuv，并且播放。因此，需要添加解码和播放部分的代码。工程的代码结构为

工程的核心函数是udp_receive_packet()，这个函数的主要工作流程为：
（1）使用recvfrom()来接收数据流
（2）使用check_fragment()对数据流进行解析，并且拷贝到本地内存中
（3）使用h264_parse_packet()来解码所获取的数据，并且使用SDL进行播放

在处理接收的数据流时，我是按照一整个压缩帧进行存储的，例如存储一个完整的Intra帧或者一个完整的P帧，不包含后续帧的信息。然而，使用av_parser_parse2()进行分析时，会首先去寻找下一帧的起始地址来确定当前帧是否完整的输入了，如果没有找到，则很多分析流程不会执行。我在这里用了一个小技巧，在数据内存的最末尾加上一个伪起始码，让av_parser_parse2()确认已经接受了所有的数据，从而进行后续的分析

PS：不过其实这样写并不通用，只是为了配合我的整帧存储方式进行的微调。如果要实现通用的解码，在接收时去掉RTP的头，送入到av_parser_parse2()就可以了

2.工程

头文件的定义中，包括rtp header和rtp packet的定义，还定义了一个全局上下文信息结构体rtp_sdl_context_t

#pragma once

#include <stdio.h>
#include <WinSock2.h>
#include <string.h>
#include <stdint.h>
#include <inttypes.h>

extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavutil/imgutils.h"
#include "SDL2/SDL.h"
};

#undef main

#define RECV_DATA_SIZE			10000
#define MAX_BUFF_SIZE			32 * 1024 * 1024

#define RTP_MAX_PKT_SIZE        1400	// RTP数据包最大为
#define RTP_HEADER_SIZE			12
#define RTP_PADDING_SIZE		64

#define RTP_PACKET_START		1
#define RTP_PACKET_FRAGMENT		2
#define RTP_PACKET_END			3

#define RECV_STREAM_DOWNLOAD	0
#define RECV_YUV_DOWNLOAD		0

typedef struct rtp_header
{
	// 存储时高位存储的是version
	/* byte 0 */
	uint8_t csrc_len : 4;		/* expect 0 */
	uint8_t extension : 1;		/* expect 1 */
	uint8_t padding : 1;        /* expect 0 */
	uint8_t version : 2;        /* expect 2 */
	/* byte 1 */
	uint8_t payload_type : 7;
	uint8_t marker : 1;        /* expect 1 */
	/* bytes 2, 3 */
	uint16_t seq_num;
	/* bytes 4-7 */
	uint32_t timestamp;
	/* bytes 8-11 */
	uint32_t ssrc;            /* stream number is used here. */
}rtp_header_t;

typedef struct rtp_packet
{
	rtp_header_t rtp_h;
	uint8_t rtp_data[RTP_MAX_PKT_SIZE + RTP_PADDING_SIZE];
}rtp_packet_t;

typedef struct rtp_context
{
	int rtp_packet_cnt;
	int rtp_buffer_size;
	int rtp_frame_cnt;
	int packet_loc;				//
	uint8_t* rtp_buffer_data;
}rtp_context_t;

typedef struct rtp_sdl_context
{
	// video param
	const AVCodec* codec;
	AVCodecContext* codec_ctx;
	AVCodecParserContext* parser_ctx ;
	AVFrame* frame;
	SwsContext* img_convert_ctx;

	// SDL param
	SDL_Window* window;
	SDL_Renderer* render;
	SDL_Texture* texture;
	SDL_Rect rect;
}rtp_sdl_context_t;

cpp文件的定义和前文类似，只是增加了一些FFmpeg解码函数和SDL播放函数，重要部分有注释

#pragma warning(disable:4996)
#pragma comment(lib,"ws2_32.lib")

#include "include/udp_rtp_decode_sdl.h"

FILE* fp_yuv;

//int avc_init(const AVCodec* codec, AVCodecContext* codec_ctx, AVCodecParserContext* parser, AVFrame* frame)
int avc_init(rtp_sdl_context_t* rsc)
{
	AVCodecID codec_id = AV_CODEC_ID_H264;
	rsc->codec = avcodec_find_decoder(codec_id);
	if (!rsc->codec)
	{
		printf("find decoder failed\n");
		return -1;
	}

	rsc->codec_ctx = avcodec_alloc_context3(rsc->codec);
	if (!rsc->codec_ctx)
	{
		printf("alloc context3 failed\n");
		return -1;
	}

	rsc->parser_ctx = av_parser_init(codec_id);
	if (!rsc->parser_ctx)
	{
		printf("parser ctx init failed\n");
		return -1;
	}

	rsc->frame = av_frame_alloc();
	if (!rsc->frame)
	{
		printf("alloc frame failed\n");
		return -1;
	}

	if (avcodec_open2(rsc->codec_ctx, rsc->codec, NULL) < 0)
	{
		printf("Could not open codec\n");
		return -1;
	}

	return 0;
}

int sdl_init(rtp_sdl_context_t* rsc)
{
	if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {
		printf("could not init sdl\n");
		return -1;
	}

	const int screen_w = 1280, screen_h = 720;
	const int pixel_w = 1280, pixel_h = 720;

	//SDL 2.0 Support for multiple windows
	rsc->window = SDL_CreateWindow("Play", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,
		screen_w, screen_h, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
	if (!rsc->window) {
		printf("SDL: could not create window - exiting:%s\n", SDL_GetError());
		return -1;
	}
	rsc->render = SDL_CreateRenderer(rsc->window, -1, 0);

	int pixformat = SDL_PIXELFORMAT_IYUV;
	rsc->texture = SDL_CreateTexture(rsc->render, pixformat, SDL_TEXTUREACCESS_STREAMING, pixel_w, pixel_h);

	int border = 0;
	rsc->rect.x = 0 + border;
	rsc->rect.y = 0 + border;
	rsc->rect.w = screen_w - border * 2;
	rsc->rect.h = screen_h - border * 2;
}

void av_free_all(rtp_sdl_context_t* rsc)
{
	avcodec_free_context(&rsc->codec_ctx);
	av_parser_close(rsc->parser_ctx);
	av_frame_free(&rsc->frame);
}

int check_nalu_header(uint8_t data0)
{
	int forbidden_zero_bit = data0 & 0x80; // 1bit
	int nal_ref_idc = data0 & 0x60; // 2 bit
	int nal_unit_type = data0 & 0x1F; // 5bit
	if ((data0 & 0x80) == 1)
	{
		printf("forbidden zero bit should be 0\n");
		return -1;
	}
	// printf("forbidden_zero_bit:%d, nal_ref_idc:%d, nal_unit_type:%d\n", forbidden_zero_bit, nal_ref_idc, nal_unit_type);
	return nal_unit_type;
}

int check_fragment_nalu_header(rtp_context_t* rtp_ctx, uint8_t data0, uint8_t data1)
{
	int nal_unit_type = check_nalu_header(data0);
	int s, e, type;
	int pos;
	if (nal_unit_type == 28) // H264
	{
		s = data1 & 0x80; // S
		e = data1 & 0x40; // E
		type = data1 & 0x1F; // type

		pos = data1 & 0xC0; // 1100 0000
		switch (pos)
		{
		case 0x80:
			rtp_ctx->packet_loc = RTP_PACKET_START;
			break;
		case 0x40:
			rtp_ctx->packet_loc = RTP_PACKET_END;
			break;
		case 0x00:
			rtp_ctx->packet_loc = RTP_PACKET_FRAGMENT;
			break;
		default: // error
			printf("invalid packet loc\n");
			return -1;
			break;
		}
	}
	return 0;
}

int find_nal_unit(uint8_t* buf, int size, int* nal_start, int* nal_end)
{
	int i;
	// find start
	*nal_start = 0;
	*nal_end = 0;

	i = 0;
	while (   //( next_bits( 24 ) != 0x000001 && next_bits( 32 ) != 0x00000001 )
		(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01) &&
		(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0 || buf[i + 3] != 0x01)
		)
	{
		i++; // skip leading zero
		if (i + 4 >= size) { return 0; } // did not find nal start
	}

	if (buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01) // ( next_bits( 24 ) != 0x000001 )
	{
		i++;
	}

	if (buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01) { /* error, should never happen */ return 0; }
	i += 3;
	*nal_start = i;

	while (   //( next_bits( 24 ) != 0x000000 && next_bits( 24 ) != 0x000001 )
		(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0) &&
		(buf[i] != 0 || buf[i + 1] != 0 || buf[i + 2] != 0x01)
		)
	{
		i++;
		// FIXME the next line fails when reading a nal that ends exactly at the end of the data
		if (i + 3 >= size) { *nal_end = size; return -1; } // did not find nal end, stream ended first
	}

	*nal_end = i;
	return (*nal_end - *nal_start);
}

void set_default_rtp_context(rtp_context_t* rtp_ctx)
{
	memset(rtp_ctx->rtp_buffer_data, 0, sizeof(rtp_ctx->rtp_buffer_size));
	rtp_ctx->rtp_packet_cnt = 0;
	rtp_ctx->rtp_buffer_size = 0;
	rtp_ctx->packet_loc = 0;
}

// Check the data is fragment or not, if fragment, try to concate
int check_fragment(rtp_context_t* rtp_ctx, rtp_packet_t* rtp_pkt, uint8_t* data, int size)
{
	int nal_start, nal_end;
	int ret = 0;
	int data_size = size - RTP_HEADER_SIZE;
	find_nal_unit(data, data_size, &nal_start, &nal_end); // check NALU split pos

	uint8_t data0 = data[nal_start];
	uint8_t data1 = data[nal_start + 1];
	uint8_t fu_indicator, fu_header;

	if (nal_start > 0 && nal_start < 5) // single-fragment, maybe SPS, PPS or small size frame
	{
		fu_indicator = 0;
		fu_header = 0;
		ret = check_nalu_header(data0); // update nalu_type
		rtp_ctx->rtp_buffer_data = (uint8_t*)realloc(rtp_ctx->rtp_buffer_data, (rtp_ctx->rtp_buffer_size + data_size) * sizeof(uint8_t));
		memcpy(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, data, data_size);

#if STREAM_DOWNLOAD
		fwrite(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, 1, data_size, fp_in);
#endif

		fprintf(stdout, "rtp_ctx frame cnt:%d, frame_size:%d\n", rtp_ctx->rtp_frame_cnt, data_size);
		rtp_ctx->rtp_frame_cnt++;
		rtp_ctx->rtp_buffer_size += data_size;
	}
	else // multi-fragment
	{
		fu_indicator = data[0];
		fu_header = data[1];
		ret = check_fragment_nalu_header(rtp_ctx, fu_indicator, fu_header);
		if (ret < 0)
		{
			printf("invalid nalu header\n");
			return -1;
		}
		int real_data_size = data_size - 2;
		rtp_ctx->rtp_buffer_data = (uint8_t*)realloc(rtp_ctx->rtp_buffer_data, (rtp_ctx->rtp_buffer_size + real_data_size) * sizeof(uint8_t));
		if (!rtp_ctx->rtp_buffer_data)
		{
			printf("realloc rtp_buffer_data failed\n");
			return -1;
		}
		memcpy(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, data + 2, real_data_size); // plus 2 to skip fu_indicator and fu_header
#if STREAM_DOWNLOAD
		fwrite(rtp_ctx->rtp_buffer_data + rtp_ctx->rtp_buffer_size, 1, real_data_size, fp_in);
		fflush(fp_in);
#endif
		rtp_ctx->rtp_packet_cnt++;
		rtp_ctx->rtp_buffer_size += real_data_size;

		if (rtp_ctx->packet_loc == RTP_PACKET_END) // end of packet
		{
			fprintf(stdout, "rtp_ctx frame cnt:%d, frame_size:%d\n", rtp_ctx->rtp_frame_cnt, rtp_ctx->rtp_buffer_size);
			rtp_ctx->rtp_frame_cnt++;
		}
	}
	return 0;
}
// 伪造起始码
int forge_end_code(uint8_t* data, int size)
{
	data = (uint8_t*)realloc(data, (size + 6) * sizeof(uint8_t));
	if (!data)
	{
		printf("realloc end code failed\n");
		return -1;
	}

	data[size] = 0x00;
	data[size + 1] = 0x00;
	data[size + 2] = 0x00;
	data[size + 3] = 0x01;
	data[size + 4] = 0x41;
	data[size + 5] = 0x9A;
	size += 6;
	return size;
}

int h264_parse_packet(rtp_sdl_context_t* rsc, rtp_context_t* rtp_ctx, rtp_packet_t* rtp_pkt)
{
	AVPacket* packet;
	int ret = 0;
	
	packet = av_packet_alloc();
	if (!packet)
	{
		printf("alloc packet failed\n");
		return -1;
	}

	// 添加伪起始码
	uint8_t* buf_data = rtp_ctx->rtp_buffer_data;
	int data_size = rtp_ctx->rtp_buffer_size;
	data_size = forge_end_code(buf_data, data_size);

	ret = av_parser_parse2(rsc->parser_ctx, rsc->codec_ctx, &packet->data, &packet->size,
		buf_data, data_size, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
	if (ret < 0) {
		printf("parse packet failed, err:%d\n", ret);
		return -1;
	}

	ret = avcodec_send_packet(rsc->codec_ctx, packet);
	if (ret < 0)
	{
		printf("send packet failed\n");
		return -1;
	}

	rsc->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
	rsc->img_convert_ctx = sws_getContext(rsc->codec_ctx->width, rsc->codec_ctx->height, rsc->codec_ctx->pix_fmt,
		rsc->codec_ctx->width, rsc->codec_ctx->height, AV_PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL);

	while (ret >= 0) {
		ret = avcodec_receive_frame(rsc->codec_ctx, rsc->frame);
		if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
			return -1;
		else if (ret < 0) {
			fprintf(stderr, "Error during decoding\n");
			exit(1);
		}

		//printf("saving frame:%d\n", rsc->codec_ctx->frame_num);
		fflush(stdout);

#if RECV_YUV_DOWNLOAD
		int size = rsc->frame->width * rsc->frame->height;
		fwrite(rsc->frame->data[0], 1, size, fp_yuv);//Y
		fwrite(rsc->frame->data[1], 1, size / 4, fp_yuv);//U
		fwrite(rsc->frame->data[2], 1, size / 4, fp_yuv);//V
		fflush(fp_yuv);
#endif

		sws_scale(rsc->img_convert_ctx, (const unsigned char* const*)rsc->frame->data, rsc->frame->linesize, 0, rsc->codec_ctx->height,
			rsc->frame->data, rsc->frame->linesize);
		// SDL播放
		SDL_UpdateYUVTexture(rsc->texture, &rsc->rect,
			rsc->frame->data[0], rsc->frame->linesize[0],
			rsc->frame->data[1], rsc->frame->linesize[1],
			rsc->frame->data[2], rsc->frame->linesize[2]);

		SDL_RenderClear(rsc->render);
		SDL_RenderCopy(rsc->render, rsc->texture, NULL, &rsc->rect);	
		SDL_RenderPresent(rsc->render);								
		SDL_Delay(40); // delay 40ms
	}
	return 0;
}
// 接收数据包
int udp_recevie_packet(rtp_sdl_context_t* rsc, const char* url, int port)
{
	WSADATA wsaData;
	WORD sockVersion = MAKEWORD(2, 2);
	int cnt = 0;

	if (WSAStartup(sockVersion, &wsaData) != 0)
	{
		return 0;
	}

	SOCKET ser_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
	if (ser_socket == INVALID_SOCKET)
	{
		ERROR("Invalid socket");
		return -1;
	}
	int on = 1;
	setsockopt(ser_socket, SOL_SOCKET, SO_REUSEADDR, (const char*)& on, sizeof(on));

	sockaddr_in ser_addr;
	ser_addr.sin_family = AF_INET;
	ser_addr.sin_port = htons(port);
	ser_addr.sin_addr.s_addr = inet_addr(url);

	if (bind(ser_socket, (sockaddr*)& ser_addr, sizeof(ser_addr)) == SOCKET_ERROR)
	{
		printf("Bind socket addr error\n");
		closesocket(ser_socket);
		return -1;
	}
	sockaddr_in remote_addr;
	int addr_len = sizeof(remote_addr);

	char recv_data[RECV_DATA_SIZE];
	rtp_context_t* rtp_ctx = (rtp_context_t*)calloc(1, sizeof(rtp_context_t));
	if (!rtp_ctx)
	{
		printf("alloc rtp_ctx failed\n");
		return -1;
	}
	rtp_packet_t* rtp_pkt = (rtp_packet_t*)calloc(1, sizeof(rtp_packet_t));
	if (!rtp_pkt)
	{
		printf("alloc rtp_pkt failed\n");
		return -1;
	}

	fprintf(stdout, "Listening on port:%d\n", port);
	while (1)
	{
		// recvfrom接收传输过来的数据
		int pkt_size = recvfrom(ser_socket, recv_data, RECV_DATA_SIZE, 0, (sockaddr*)& remote_addr, &addr_len);
		if (pkt_size > 0)
		{
			memcpy(rtp_pkt, recv_data, pkt_size);
			check_fragment(rtp_ctx, rtp_pkt, rtp_pkt->rtp_data, pkt_size); // check pkt data is fragment or not
			rtp_header_t rtp_h = rtp_pkt->rtp_h;
			char payload = rtp_h.payload_type;

			if (rtp_ctx->packet_loc == RTP_PACKET_END)
			{
				switch (payload)
				{
				case 33: // mpegts
					// mpegts_packet_parse((uint8_t*)rtp_data, parse_mpegts, payload, rtp_data_size); // TODO: add mpegts parser
					printf("MPEGTS type\n");
					break;
				case 96: // h264
					//printf("payload type:%s\n", "H264");
					// 进行h264码流的解析
					h264_parse_packet(rsc, rtp_ctx, rtp_pkt);
					break;
				default:
					printf("Unknown type\n");
					break;
				}
				// printf("[RTP PKT] %5d| %5s | %10u| %5d| %5d\n", cnt, payload_str, timestamp, seq_num, pkt_size);
				set_default_rtp_context(rtp_ctx); // set default rtp ctx value
			}
		}
	}
}

int main()
{
	rtp_sdl_context_t* rsc = (rtp_sdl_context_t*)malloc(sizeof(rtp_sdl_context_t));
	if (!rsc)
	{
		printf("malloc rsc failed\n");
		return -1;
	}
	memset(rsc, 0, sizeof(rtp_sdl_context_t));

	// 初始化参数
	avc_init(rsc);
	sdl_init(rsc);
	
	// 如果要存储yuv信息可以设置为1
#if RECV_YUV_DOWNLOAD
	fp_yuv = fopen("rtp_receive_yuv.yuv", "wb");
#endif
	// 开始接收数据包
	udp_recevie_packet(rsc, "127.0.0.1", 8880);
	av_free_all(rsc);

#if RECV_YUV_DOWNLOAD
	fclose(fp_yuv);
#endif

	return 0;
}

3.测试

发送端

接收端

接收端播放正常，感觉可以后续改一改SDL的逻辑，让窗口变成可移动和可缩放的

4.小结

总体来说，这个功能的实现是比较简单的，不过使用了一个小的技巧，伪造了一个起始地址，如果代码格式要求不严格，可以凑合着用。如果要做成大的工程，应该将多个packet直接送入解码器，这样比较合理，也更符合FFmpeg的设计原则