技术背景
随着物联网等行业的崛起,越来越多的传统行业如虚拟仿真、航天工业、工业仿真、城市规划等,对Linux下的生态构建,有了更大的期望,Linux平台下,可选的直播推拉流解决方案相对Windows和移动端,非常少,基于Unity的Linux推送方案,更是几无参考。本文以Unity3d环境下Linux平台推送Unity窗体和Unity采集的音频,然后编码推送到RTMP服务器为例,大概说下实现过程。
技术实现
本文以采集Unity窗体数据为例,如果需要对接摄像头和屏幕亦可。简单来说,高效率的获取到原始的窗体Texture,拿到对应的RGB数据,传给现有的RTMP推送模块,音频的话,只要获取到AudioClip数据,实时读取然后传PCM数据。如需2路PCM数据混音,可以采集两路AudioClip数据,分别调用原生的Linux推送RTMP模块的PCM数据投递接口,编码打包推送即可,无图无真相:
下图以Linux平台下,unity3d环境采集窗体(为方便测试,加了个实时时间显示)和unity声音为例,整体延迟可达毫秒级。
音频的话,可以循环采集播放、也可以播放完毕后,加实时静音。
AudioClip数据采集,如需audiosource数据播放完毕后,填充静音帧,可调用FillPcmMuteData()接口。
/// <summary>
/// 获取AudioClip数据
/// </summary>
private void PostUnityAudioClipData()
{
//重新初始化
TimerManager.UnRegister(this);
audio_clip_info_ = new AudioClipInfo();
audio_clip_info_.audio_source_ = GameObject.Find("Canvas/Panel/AudioSource").GetComponent<AudioSource>();
if (audio_clip_info_.audio_source_ == null)
{
Debug.LogError("audio source is null..");
return;
}
if (audio_clip_info_.audio_source_.clip == null)
{
Debug.LogError("audio clip is null..");
return;
}
audio_clip_info_.audio_clip_ = audio_clip_info_.audio_source_.clip;
audio_clip_info_.audio_clip_offset_ = 0;
audio_clip_info_.audio_clip_length_ = audio_clip_info_.audio_clip_.samples * audio_clip_info_.audio_clip_.channels;
pcm_mute_data_ = FillPcmMuteData(audio_clip_info_.audio_clip_);
TimerManager.Register(this, 0.019999f, (PostPCMData), false);
}
采集后的数据,定时调用原生接口,发送到原生接口:
var sample_length = sizeof(float) * pcm_sample.Length;
pcm_data.data_ = Marshal.AllocHGlobal(sample_length);
Marshal.Copy(pcm_sample, 0, pcm_data.data_, pcm_sample.Length);
pcm_data.size_ = (uint)sample_length;
publisher_wrapper_.OnPostAudioPCMFloatData(pcm_data.data_,
pcm_data.size_,
pcm_time_stamp_,
pcm_data.sample_rate_,
pcm_data.channels_,
pcm_data.per_channel_sample_number_);
Marshal.FreeHGlobal(pcm_data.data_);
pcm_data.data_ = IntPtr.Zero;
pcm_data = null;
pcm_time_stamp_ += 10; //时间戳自增10毫秒
视频窗体数据采集:
if (video_push_type_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
return;
if (texture_ == null || video_width_ != Screen.width || video_height_ != Screen.height)
{
Debug.Log("OnPostRender screen changed++ scr_width: " + Screen.width + " scr_height: " + Screen.height);
if (screen_image_ != IntPtr.Zero)
{
Marshal.FreeHGlobal(screen_image_);
screen_image_ = IntPtr.Zero;
}
if (texture_ != null)
{
UnityEngine.Object.Destroy(texture_);
texture_ = null;
}
video_width_ = Screen.width;
video_height_ = Screen.height;
texture_ = new Texture2D(video_width_, video_height_, TextureFormat.BGRA32, false);
screen_image_ = Marshal.AllocHGlobal(video_width_ * 4 * video_height_);
Debug.Log("OnPostRender screen changed--");
return;
}
从Texture拿到数据,然后调用OnPostRGBAData()数据传递到原生接口即可。
原生接口模块初始化:
/*
* nt_publisher_wrapper.cs
* Github:https://github.com/daniulive/SmarterStreaming
*
* InitSDK主要完成原生模块的初始化工作
*/
private bool InitSDK()
{
if (!is_pusher_sdk_init_)
{
// 设置日志路径(请确保目录存在)
Debug.LogError("NT_SL_SetPath++");
String log_path = "./";
NTSmartLog.NT_SL_SetPath(log_path);
UInt32 isInited = NTSmartPublisherSDK.NT_PB_Init(0, IntPtr.Zero);
if (isInited != 0)
{
Debug.Log("调用NT_PB_Init失败..");
return false;
}
is_pusher_sdk_init_ = true;
}
return true;
}
RTMP推送模块基础参数配置:
private void SetCommonOptionToPublisherSDK()
{
if (!IsPublisherHandleAvailable())
{
Debug.Log("SetCommonOptionToPublisherSDK, publisher handle with null..");
return;
}
NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0,
0, IntPtr.Zero);
if (video_option_ == (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
{
// 第0层填充RGBA矩形, 目的是保证帧率, 颜色就填充全黑
int red = 0;
int green = 0;
int blue = 0;
int alpha = 255;
NT_PB_RGBARectangleLayerConfig rgba_layer_c0 = new NT_PB_RGBARectangleLayerConfig();
rgba_layer_c0.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
rgba_layer_c0.base_.index_ = 0;
rgba_layer_c0.base_.enable_ = 1;
rgba_layer_c0.base_.region_.x_ = 0;
rgba_layer_c0.base_.region_.y_ = 0;
rgba_layer_c0.base_.region_.width_ = video_width_;
rgba_layer_c0.base_.region_.height_ = video_height_;
rgba_layer_c0.base_.offset_ = Marshal.OffsetOf(rgba_layer_c0.GetType(), "base_").ToInt32();
rgba_layer_c0.base_.cb_size_ = (uint)Marshal.SizeOf(rgba_layer_c0);
rgba_layer_c0.red_ = System.BitConverter.GetBytes(red)[0];
rgba_layer_c0.green_ = System.BitConverter.GetBytes(green)[0];
rgba_layer_c0.blue_ = System.BitConverter.GetBytes(blue)[0];
rgba_layer_c0.alpha_ = System.BitConverter.GetBytes(alpha)[0];
IntPtr rgba_conf = Marshal.AllocHGlobal(Marshal.SizeOf(rgba_layer_c0));
Marshal.StructureToPtr(rgba_layer_c0, rgba_conf, true);
UInt32 rgba_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
rgba_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE,
0, IntPtr.Zero);
Marshal.FreeHGlobal(rgba_conf);
NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 = new NT_PB_ExternalVideoFrameLayerConfig();
external_layer_c1.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
external_layer_c1.base_.index_ = 1;
external_layer_c1.base_.enable_ = 1;
external_layer_c1.base_.region_.x_ = 0;
external_layer_c1.base_.region_.y_ = 0;
external_layer_c1.base_.region_.width_ = video_width_;
external_layer_c1.base_.region_.height_ = video_height_;
external_layer_c1.base_.offset_ = Marshal.OffsetOf(external_layer_c1.GetType(), "base_").ToInt32();
external_layer_c1.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c1);
IntPtr external_layer_conf = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1));
Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true);
UInt32 external_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,
0, IntPtr.Zero);
Marshal.FreeHGlobal(external_layer_conf);
}
else if (video_option_ == (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA)
{
CameraInfo camera = cameras_[cur_sel_camera_index_];
NT_PB_VideoCaptureCapability cap = camera.capabilities_[cur_sel_camera_resolutions_index_];
SetVideoCaptureDeviceBaseParameter(camera.id_.ToString(), cap.width_, cap.height_);
}
SetFrameRate((uint)video_fps_);
Int32 type = 0; //软编码
Int32 encoder_id = 1;
UInt32 codec_id = (UInt32)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;
Int32 param1 = 0;
SetVideoEncoder(type, encoder_id, codec_id, param1);
SetVideoQuality(CalVideoQuality(video_width_, video_height_, is_h264_encoder_));
SetVideoBitRate(CalBitRate(video_fps_, video_width_, video_height_));
SetVideoMaxBitRate((CalMaxKBitRate(video_fps_, video_width_, video_height_, false)));
SetVideoKeyFrameInterval((key_frame_interval_));
if (is_h264_encoder_)
{
SetVideoEncoderProfile(1);
}
SetVideoEncoderSpeed(CalVideoEncoderSpeed(video_width_, video_height_, is_h264_encoder_));
// 音频相关设置
SetAuidoInputDeviceId(0);
SetPublisherAudioCodecType(1);
SetPublisherMute(is_mute_);
SetEchoCancellation(0, 0);
SetNoiseSuppression(0);
SetAGC(0);
SetVAD(0);
SetInputAudioVolume(Convert.ToSingle(audio_input_volume_));
}
开始和停止预览:
public bool StartPreview()
{
if(CheckPublisherHandleAvailable() == false)
return false;
video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);
if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero))
{
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
return false;
}
publisher_handle_count_++;
is_previewing_ = true;
return true;
}
public void StopPreview()
{
if (is_previewing_ == false) return;
is_previewing_ = false;
publisher_handle_count_--;
NTSmartPublisherSDK.NT_PB_StopPreview(publisher_handle_);
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
}
预览数据回调:
//预览数据回调
public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
{
NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));
NT_VideoFrame pVideoFrame = new NT_VideoFrame();
pVideoFrame.width_ = pb_image.width_;
pVideoFrame.height_ = pb_image.height_;
pVideoFrame.stride_ = pb_image.stride_[0];
Int32 argb_size = pb_image.stride_[0] * pb_image.height_;
pVideoFrame.plane_data_ = new byte[argb_size];
if (argb_size > 0)
{
Marshal.Copy(pb_image.plane_[0],pVideoFrame.plane_data_,0, argb_size);
}
{
cur_image_ = pVideoFrame;
}
}
开始推送和停止推送:
public bool StartPublisher(String url)
{
if (CheckPublisherHandleAvailable() == false) return false;
if (publisher_handle_ == IntPtr.Zero)
{
return false;
}
if (!String.IsNullOrEmpty(url))
{
NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero);
}
if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero))
{
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
is_publishing_ = false;
return false;
}
publisher_handle_count_++;
is_publishing_ = true;
return true;
}
public void StopPublisher()
{
if (is_publishing_ == false) return;
publisher_handle_count_--;
NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_);
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
is_publishing_ = false;
}
对应Unity层调用:
public void btn_publish_Click()
{
if (publisher_wrapper_.IsPublishing())
{
return;
}
String url = rtmp_pusher_url.text;
if (url.Length < 8)
{
publisher_wrapper_.Close();
Debug.Log("请输入RTMP推送地址");
return;
}
publisher_wrapper_.SetVideoPushType(video_push_type_);
publisher_wrapper_.SetAudioPushType(audio_push_type_);
if (!publisher_wrapper_.StartPublisher(url))
{
Debug.LogError("调用StartPublisher失败..");
return;
}
startPublishBtn.interactable = false;
stopPublishBtn.interactable = !startPublishBtn.interactable;
videoOptionSel.interactable = false;
audioOptionSel.interactable = false;
if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA
|| audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER)
{
PostUnityAudioClipData();
}
}
public void btn_stop_publish_Click()
{
if (!publisher_wrapper_.IsPublishing())
{
return;
}
StopAudioSource();
if (texture_ != null)
{
UnityEngine.Object.Destroy(texture_);
texture_ = null;
}
if (screen_image_ != IntPtr.Zero)
{
Marshal.FreeHGlobal(screen_image_);
screen_image_ = IntPtr.Zero;
}
publisher_wrapper_.StopPublisher();
videoOptionSel.interactable = true;
audioOptionSel.interactable = true;
startPublishBtn.interactable = true;
stopPublishBtn.interactable = !startPublishBtn.interactable;
}
总结
以上是Unity环境下,以采集Unity窗体和unity声音为例,对接Linux平台RTMP推送,整体延迟也可以达到毫秒级,基本满足常用场景了,感兴趣的开发者可酌情参考。