Windows平台如何实现多路RTSP|RTMP流合成后录像或转发RTMP服务

技术背景

我们在对接Windows平台RTSP|RTMP直播播放模块的时候，有开发者提出来这样的技术需求，他们做驾考、全景摄像头、多路会议录制等场景的时候，希望把多路视频流数据，合并到一路保存或者对外推送到RTMP服务。

技术实现

多路RTSP|RTMP流合流，实际上我们2016年就有这块demo，当时合流的数据是本地采集的摄像头或屏幕数据，和外部RTSP、RTMP流，合成后输出（类似于传统意义的连麦操作）。这里大概说下思路，外部的RTSP|RTMP流数据，解码后，把YUV或RGB数据回调上来，然后，按照图层的形式，分别贴摄像头、屏幕数据或解码后的流数据。

本次以四路RTSP摄像头数据合流为例：

开始播放：

/*
 * SmartPlayerDemo.cs
 * Author: daniusdk.com
 * QQ: 89030985
 */
private void btn_play1_Click(object sender, EventArgs e)
{
	if (btn_play1.Text == "播放")
	{
		String url = textBox_url1.Text;
		if (!InitCommonSDKParam(player1_handle_, url))
		{
			MessageBox.Show("设置参数错误!");
			return;
		}

		bool is_support_d3d_render = false;
		Int32 in_support_d3d_render = 0;

		if (NT.NTBaseCodeDefine.NT_ERC_OK == NTSmartPlayerSDK.NT_SP_IsSupportD3DRender(player1_handle_, playWnd1.Handle, ref in_support_d3d_render))
		{
			if (1 == in_support_d3d_render)
			{
				is_support_d3d_render = true;
			}
		}

		if (is_support_d3d_render)
		{
			// 支持d3d绘制的话，就用D3D绘制
			NTSmartPlayerSDK.NT_SP_SetRenderWindow(player1_handle_, playWnd1.Handle);

			if (btn_check_render_scale_mode.Checked)
				NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player1_handle_, 1);
			else
				NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player1_handle_, 0);

		}

		video_frame_call_back1_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
		NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player1_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back1_);

		UInt32 ret_start = NTSmartPlayerSDK.NT_SP_StartPlay(player1_handle_);

		if (ret_start != 0)
		{
			MessageBox.Show("播放失败..");
			return;
		}

		btn_play1.Text = "停止";
	}
	else
	{
		StopPlayback1();
	}
}

其中InitCommonSDKParam()主要完成一些初始化参数设置：

private bool InitCommonSDKParam(IntPtr handle, String url) {
	if (IntPtr.Zero == handle)
		return false;

	if (String.IsNullOrEmpty(url))
		return false;

	Int32 buffer_time = int.Parse(textBox_buffer_time.Text);

	NTSmartPlayerSDK.NT_SP_SetBuffer(handle, buffer_time);

	// 设置rtsp tcp模式，rtmp不使用, 可以不设置
	if (checkBox_rtsp_tcp.Checked)
		NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(handle, 1);
	else
		NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(handle, 0);

	//RTSP timeout设置
	Int32 rtsp_timeout = 10;
	NTSmartPlayerSDK.NT_SP_SetRtspTimeout(handle, rtsp_timeout);

	//RTSP TCP/UDP自动切换设置
	Int32 is_auto_switch_tcp_udp = 1;
	NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(handle, is_auto_switch_tcp_udp);

	if (checkBox_mute.Checked)
		NTSmartPlayerSDK.NT_SP_SetMute(handle, 1);
	else
		NTSmartPlayerSDK.NT_SP_SetMute(handle, 0);

	if (checkBox_fast_startup.Checked)
		NTSmartPlayerSDK.NT_SP_SetFastStartup(handle, 1);
	else
		NTSmartPlayerSDK.NT_SP_SetFastStartup(handle, 0);

	if (checkBox_hardware_decoder.Checked)
	{
		NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(handle, is_support_h264_hardware_decoder_ ? 1 : 0, 0);
		NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(handle, is_support_h265_hardware_decoder_ ? 1 : 0, 0);
	}
	else
	{
		NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(handle, 0, 0);
		NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(handle, 0, 0);
	}

	// 设置是否只解码关键帧
	if (btn_check_only_decode_video_key_frame.Checked)
		NTSmartPlayerSDK.NT_SP_SetOnlyDecodeVideoKeyFrame(handle, 1);
	else
		NTSmartPlayerSDK.NT_SP_SetOnlyDecodeVideoKeyFrame(handle, 0);

	// 设置低延迟模式
	if (checkBox_low_latency.Checked)
		NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(handle, 1);
	else
		NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(handle, 0);

	NTSmartPlayerSDK.NT_SP_SetRotation(handle, rotate_degrees_);

	NTSmartPlayerSDK.NT_SP_SetAudioVolume(handle, slider_audio_volume.Value);

	NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(handle, 1, 1);

	NTSmartPlayerSDK.NT_SP_SetURL(handle, url);

	return true;
}

开始播放之前，我们设置YUV数据回调：

video_frame_call_back1_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player1_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back1_);

回调处理如下，如果是多个图层，通过推送端，把yuv或rgb数据，投递给推送端，video frame数据回调，可以根据handle区分不同的图层或实例：

public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
	if (frame == IntPtr.Zero)
		return;

	NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));

	if (publisher_wrapper_ != null) {

		int video_layer_index;
		if (handle == player_handle_)
			video_layer_index = publisher_wrapper_.get_external_video_layer0_index();
		else if (handle == player1_handle_)
			video_layer_index = publisher_wrapper_.get_external_video_layer1_index();
		else if (handle == player2_handle_)
			video_layer_index = publisher_wrapper_.get_external_video_layer2_index();
		else if (handle == player3_handle_)
			video_layer_index = publisher_wrapper_.get_external_video_layer3_index();
		else
			video_layer_index = -1;

		if (video_layer_index > -1)
		{
			publisher_wrapper_.post_i420_layer_image(video_layer_index, video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
				  video_frame.plane2_, video_frame.stride2_,
				  video_frame.width_, video_frame.height_);
		}
	}

}

推送端，目前以四路合成为例，另外加个实时文字水印，图层设计如下：

public bool config_layers(bool is_add_rgbx_zero_layer)
{
	if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
		return false;

	if (is_empty_handle())
		return false;

	int w = video_width_;
	int h = video_height_;

	if ((w & 0x1) != 0)
		--w;

	if ((h & 0x1) != 0)
		--h;

	if (w < 2 || h < 2)
		return false;

	NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero);

	int type, index = 0;
	if (is_add_rgbx_zero_layer)
	{
		NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig();
		type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
		fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h);
		rgba_layer.red_ = 0;
		rgba_layer.green_ = 0;
		rgba_layer.blue_ = 0;
		rgba_layer.alpha_ = 255;
		if (add_layer_config(rgba_layer, type))
			index++;
	}

	NT_PB_ExternalVideoFrameLayerConfig external_video_layer0 = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(external_video_layer0, out external_video_layer0.base_, type, index, true, 0, 0, w/2, h/2);
	if (add_layer_config(external_video_layer0, type))
		external_video_layer0_index_ = index++;

	NT_PB_ExternalVideoFrameLayerConfig external_video_layer1 = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(external_video_layer1, out external_video_layer1.base_, type, index, true, w / 2, 0, w / 2, h / 2);
	if (add_layer_config(external_video_layer1, type))
		external_video_layer1_index_ = index++;

	NT_PB_ExternalVideoFrameLayerConfig external_video_layer2 = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(external_video_layer2, out external_video_layer2.base_, type, index, true, 0, h / 2, w / 2, h / 2);
	if (add_layer_config(external_video_layer2, type))
		external_video_layer2_index_ = index++;

	NT_PB_ExternalVideoFrameLayerConfig external_video_layer3 = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(external_video_layer3, out external_video_layer3.base_, type, index, true, w / 2, h / 2, w / 2, h / 2);
	if (add_layer_config(external_video_layer3, type))
		external_video_layer3_index_ = index++;


	//叠加的文本层
	NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(text_layer, out text_layer.base_, type, index, false, w / 2, h / 2, 64, 64);
	if (add_layer_config(text_layer, type))
		text_layer_index_ = index++;

	return index > 0;
}

合成后数据，可以对外推送到RTMP服务，也可以注入到本地RTSP服务，或者本地直接录制MP4文件,录制出来四宫格效果如下：

总结

多路RTSP|RTMP数据合流，在多媒体处理、实时监控、驾考、教育等各个行业，应用非常广泛，除了视频外，音频如果需要合成，可以以采集系统扬声器的形式合流出来。多路合流，可以事先做好排版编辑，如果期间不希望显示某一路数据，可以点隐藏图层，实时对图层进行隐藏。感兴趣的开发者，可以单独跟我沟通交流。