Windows平台Unity下实现camera场景推送RTMP|轻量级RTSP服务|实时录像

技术背景

我们在对接Unity平台camera场景采集的时候，除了常规的RTMP推送、录像外，还有一些开发者，需要能实现轻量级RTSP服务，对外提供个拉流的RTSP URL。

目前我们在Windows平台Unity下数据源可采集到以下部分：

• 采集Unity camera场景；
• 采集摄像头；
• 采集屏幕；
• 采集Unity声音；
• 采集麦克风；
• 采集扬声器；
• Unity PCM混音；

对外提供的技术能力有：

• RTMP直播推送；
• 轻量级RTSP服务；
• 实时录像、暂停|恢复录像；
• 实时预览。

以下录制下来的MP4文件是采集Unity camera场景，音频是unity声音。

Unity平台实现camera场景实时录像

技术实现

实际上，在实现Unity平台音视频能力之前，我们原生模块已经有非常成熟的技术积累，Unity下还是调用的原生的推送模块，不同的是，数据源需要采集Unity的audio、video，然后高效的投递到底层模块，底层模块负责编码打包，并投递到RTMP或RTSP服务。

先说支持的音视频类型：

    public void SelVideoPushType(int type)
    {
        switch (type)
        {
            case 0:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER;    //采集Unity窗体
                break;
            case 1:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA;   //采集摄像头
                break;
            case 2:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_SCREEN;   //采集屏幕
                break;
            case 3:
                video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_NO_VIDEO; //不采集视频
                break;
        }

        Debug.Log("SelVideoPushType type: " + type + " video_push_type: " + video_push_type_);
    }

    public void SelAudioPushType(int type)
    {
        switch (type)
        {
            case 0:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA;    //采集Unity声音
                break;
            case 1:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_MIC;  //采集麦克风
                break;
            case 2:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER;  //采集扬声器
                break;
            case 3:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER;  //两路Unity AudioClip混音测试
                break;
            case 4:
                audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_NO_AUDIO;   //不采集音频
                break;
        }

        Debug.Log("SelAudioPushType type: " + type + " audio_push_type: " + audio_push_type_);
    }

采集音视频数据：

    private void StartCaptureAvData()
    {
        if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA
            || audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER)
        {
            PostUnityAudioClipData();
        }

        textures_poll_ = new TexturesPool();
        post_image_worker_ = new PostImageWorker(textures_poll_, publisher_wrapper_);
        post_worker_thread_ = new Thread(post_image_worker_.run);
        post_worker_thread_.Start();
    }

    private void StopCaptureAvData()
    {
        if (post_image_worker_ != null)
        {
            post_image_worker_.requestStop();
            post_image_worker_ = null;
        }

        if (post_worker_thread_ != null)
        {
            post_worker_thread_.Join();
            post_worker_thread_ = null;
        }

        if (textures_poll_ != null)
        {
            textures_poll_.clear();
            textures_poll_ = null;
        }

        StopAudioSource();
    }

RTMP推送：

    public void btn_start_rtmp_pusher_Click()
    {
        if (publisher_wrapper_.IsPushingRtmp())
        {
            StopPushRTMP();
            btn_rtmp_pusher_.GetComponentInChildren<Text>().text = "推送RTMP";

            return;
        }

        String url = rtmp_pusher_url_.text;

        if (url.Length < 8)
        {
            publisher_wrapper_.Close();

            Debug.LogError("请输入RTMP推送地址");
            return;
        }

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
        {
            publisher_wrapper_.SetVideoPushType(video_push_type_);
            publisher_wrapper_.SetAudioPushType(audio_push_type_);
        }

        if (!publisher_wrapper_.StartRtmpPusher(url))
        {
            Debug.LogError("调用StartPublisher失败..");
            return;
        }

        btn_rtmp_pusher_.GetComponentInChildren<Text>().text = "停止推送";

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
        {
            StartCaptureAvData();
            coroutine_ = StartCoroutine(OnPostVideo());
        }
    }

轻量级RTSP服务相关调用：

    public void btn_rtsp_service_Click()
    {
        if (publisher_wrapper_.IsRtspServiceRunning())
        {
            publisher_wrapper_.StopRtspService();
            btn_rtsp_service_.GetComponentInChildren<Text>().text = "启动RTSP服务";

            btn_rtsp_publisher_.interactable = false;
            return;
        }

        if (!publisher_wrapper_.StartRtspService())
        {
            Debug.LogError("调用StartRtspService失败..");
            return;
        }

        btn_rtsp_publisher_.interactable = true;

        btn_rtsp_service_.GetComponentInChildren<Text>().text = "停止RTSP服务";
    }

    public void btn_rtsp_publisher_Click()
    {
        if (publisher_wrapper_.IsRtspPublisherRunning())
        {
            publisher_wrapper_.StopRtspStream();

            if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
            {
                StopCaptureAvData();

                if (coroutine_ != null)
                {
                    StopCoroutine(coroutine_);
                    coroutine_ = null;
                }
            }

            btn_rtsp_service_.interactable = true;

            btn_rtsp_publisher_.GetComponentInChildren<Text>().text = "发布RTSP";
        }
        else
        {
            if (!publisher_wrapper_.IsRtspServiceRunning())
            {
                Debug.LogError("RTSP service is not running..");
                return;
            }


            if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
            {
                publisher_wrapper_.SetVideoPushType(video_push_type_);
                publisher_wrapper_.SetAudioPushType(audio_push_type_);
            }

            publisher_wrapper_.StartRtspStream();

            if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
            {
                StartCaptureAvData();
                coroutine_ = StartCoroutine(OnPostVideo());
            }

            btn_rtsp_publisher_.GetComponentInChildren<Text>().text = "停止RTSP";

            btn_rtsp_service_.interactable = false;
        }
    }

    public void btn_get_rtsp_session_numbers_Click()
    {
        if (publisher_wrapper_.IsRtspServiceRunning())
        {
            btn_get_rtsp_session_numbers_.GetComponentInChildren<Text>().text = "RTSP会话数:" + publisher_wrapper_.GetRtspSessionNumbers();
        }
    }

实时录像调用：

    public void btn_record_Click()
    {
        if (publisher_wrapper_.IsRecording())
        {
            StopRecord();
            btn_record_.GetComponentInChildren<Text>().text = "开始录像";

            return;
        }

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
        {
            publisher_wrapper_.SetVideoPushType(video_push_type_);
            publisher_wrapper_.SetAudioPushType(audio_push_type_);
        }

        if (!publisher_wrapper_.StartRecorder())
        {
            Debug.LogError("调用StartRecorder失败..");
            return;
        }

        btn_record_.GetComponentInChildren<Text>().text = "停止录像";

        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
        {
            StartCaptureAvData();
            coroutine_ = StartCoroutine(OnPostVideo());
        }
    }

    public void StopRecord()
    {
        if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
        {
            StopCaptureAvData();

            if (coroutine_ != null)
            {
                StopCoroutine(coroutine_);
                coroutine_ = null;
            }
        }

        publisher_wrapper_.StopRecorder();
    }

    private void btn_pause_record_Click()
    {
        if (!publisher_wrapper_.IsPublisherHandleAvailable())
        {
            return;
        }

        String btn_pause_rec_text = btn_pause_record_.GetComponentInChildren<Text>().text;

        if ("暂停录像" == btn_pause_rec_text)
        {
            UInt32 ret = publisher_wrapper_.PauseRecorder(true);

            if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
            {
                Debug.LogError("暂停录像失败, 请重新尝试!");
                return;
            }
            else if (NTBaseCodeDefine.NT_ERC_OK == ret)
            {
                btn_pause_record_.GetComponentInChildren<Text>().text = "恢复录像";
            }
        }
        else
        {
            UInt32 ret = publisher_wrapper_.PauseRecorder(false);
            if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
            {
                Debug.LogError("恢复录像失败, 请重新尝试!");
                return;
            }
            else if (NTBaseCodeDefine.NT_ERC_OK == ret)
            {
                btn_pause_record_.GetComponentInChildren<Text>().text = "暂停录像";
            }
        }
    }

实时预览相关：

    public void btn_preview_Click()
    {
        if (btn_preview_.GetComponentInChildren<Text>().text.Equals("本地预览"))
        {
            if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
            {
                publisher_wrapper_.SetVideoPushType(video_push_type_);
            }

            if (publisher_wrapper_.StartPreview())
            {
                btn_preview_.GetComponentInChildren<Text>().text = "停止预览";
            }

            if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
            {
                StartCaptureAvData();
                coroutine_ = StartCoroutine(OnPostVideo());
            }
        }
        else
        {
            if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
            {
                StopCaptureAvData();

                if (coroutine_ != null)
                {
                    StopCoroutine(coroutine_);
                    coroutine_ = null;
                }
            }

            publisher_wrapper_.StopPreview();
            btn_preview_.GetComponentInChildren<Text>().text = "本地预览";
        }
    }

总结

Unity平台下RTMP推送、录像、轻量级RTSP服务，在虚拟仿真、医疗、教育等场景下，应用非常广泛。要实现低延迟，除了需要高效率的音视频数据采集，编码和数据投递外，还需要好的直播播放器支持。配合我们的SmartPlayer，可轻松实现毫秒级体验，满足绝大多数应用场景技术诉求。