视频采集 via Media Foundation

Media Foundation 简介

Media Foundation （简称 MF）是微软在 Windows Vista上 推出的新一代多媒体应用库，目的是提供 Windows 平台一个统一的多媒体影音解决方案，开发者可以通过 MF 播放视频或声音文件、进行多媒体文件格式转码，或者将一连串图片编码为视频等等。

MF 是 DirectShow 为主的旧式多媒体应用程序接口的替代者与继承者，在微软的计划下将逐步汰换 DirectShow 技术。MF 要求 Windows Vista 或更高版本，不支持较早期的 Windows 版本，特别是 Windows XP。
MF 长于高质量的音频和视频播放，高清内容（如 HDTV，高清电视）和数字版权管理（DRM）访问控制。MF 在不同的 Windows 版本上能力不同，如 Windows 7 上就添加了 h.264 编码支持。Windows 8 上则提供数种更高质量的设置。

MF 提供了两种编程模型，第一种是以 Media Session 为主的 Media pipeline 模型，但是该模型太过复杂，且曝露过多底层细节，故微软于 Windows 7 上推出第二种编程模型，内含 SourceReader、Transcode API 、SinkWriter 及 MFPlay 等高度封装模块，大大简化了 MF 的使用难度。

# 本文使用了第二种（简单的）编程模型。

Media Foundation 采集视频

采集流程图

MF 采集视频代码

以下是整个 MF 采集过程的概要代码，略去设备枚举和 CMFCapture 类的实现

hr = CoInitializeEx(NULL, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE);
hr = MFStartup(MF_VERSION);

// Enumerate the capture devices.
hr = UpdateDeviceList(hDlg, true);

EncodingParameters vidEncParam;
vidEncParam.subType = _getSubType(hDlg, false);
vidEncParam.bitRate = TARGET_VID_BIT_RATE;
CComPtr<IMFActivate> pVidActivate = NULL;
hr = GetSelectedDevice(hDlg, &pVidActivate, true);

hr = CMFCapture::CreateInstance(hDlg, &g_pCapture);
hr = g_pCapture->startCapture(pVidActivate, &vidEncParam, pszFile);
// Capturing ...  

hr = g_pCapture->stopCapture();
g_pCapture->Release();
MFShutdown();
CoUninitialize();

MMDeviceHelper::enumVidCapDevices 函数

此处使用了 _enumMFDevices 传参的形式获取视频设备，因为该函数还可以枚举音频设备。

HRESULT MMDeviceHelper::enumVidCapDevices()
{
    return _enumDevices(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
}

HRESULT MMDeviceHelper::_enumMFDevices(const GUID& devType)
{
    HRESULT hr = S_OK;
    CComPtr<IMFAttributes> pAttributes = NULL;
    clear();
    
    // Initialize an attribute store. We will use this to specify the enumeration parameters.
    hr = MFCreateAttributes(&pAttributes, 1);
    RETURN_IF_FAILED(hr);
    
    hr = pAttributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, devType);
    RETURN_IF_FAILED(hr);
    
    hr = MFEnumDeviceSources(pAttributes, &m_ppDevices, &m_cDevices);
    RETURN_IF_FAILED(hr);
    
    return hr;
}

CMFCapture::startCapture 函数

首先创建一个 Sink Writer 并开始写入，接着配置视频输入并开始读取 sample。

HRESULT CMFCapture::startCapture(IMFActivate *pVideoAct, EncodingParameters* pVidEncParam, LPCTSTR pszFileName)
{
    HRESULT hr = S_OK;
    SyncUtil::AutoLock lock(m_critsec);
    hr = MFCreateSinkWriterFromURL(pszFileName, NULL, NULL, &m_pWriter);
    RETURN_IF_MF_FAILED(hr);
    
    m_bFirstSample = TRUE;
    m_llBaseTime = 0;
    hr = _configVideoCapture(pVideoAct, pVidEncParam);
    GOTO_LABEL_IF_FAILED(hr, OnErr);
    
    hr = m_pWriter->BeginWriting();
    GOTO_LABEL_IF_FAILED(hr, OnErr);
    
    hr = m_pVideoReader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
    GOTO_LABEL_IF_FAILED(hr, OnErr);
    
    m_isCapturing = true;
    return hr;
OnErr:
    SAFE_RELEASE(m_pVideoReader);
    SAFE_RELEASE(m_pWriter);
    return hr;
}

CMFCapture::_configVideoCapture 函数

创建并配置视频 source reader 及 encoder。下面还注册了一个 ColorConvert 的 DMO，用来在需要时让 MF 自动转换视频帧的色彩空间。

HRESULT CMFCapture::_configVideoCapture( IMFActivate *pActivate, EncodingParameters* pEncParam )
{
    HRESULT hr = E_FAIL;
    CComPtr<IMFMediaSource> pSource = NULL;
    DWORD sinkStream = 0;
    CComPtr<IMFMediaType> pType = NULL;
    SyncUtil::AutoLock lock(m_critSec);
    
    hr = pActivate->ActivateObject(__uuidof(IMFMediaSource), (void**)&pSource);
    
    hr = createSrcReader(pSource, m_pVideoReader, this);
    hr = configSrcReader(m_pVideoReader, false);
    
    hr = m_pVideoReader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, &pType);
    hr = configVideoEncoder(pEncParam, pType, m_pWriter, &sinkStream);
    
    m_videoStreamIdx = sinkStream;
    hr = MFTRegisterLocalByCLSID(__uuidof(CColorConvertDMO), MFT_CATEGORY_VIDEO_PROCESSOR,
        _T(""), MFT_ENUM_FLAG_SYNCMFT, 0, NULL, 0, NULL);
        
    hr = m_pWriter->SetInputMediaType(sinkStream, pType, NULL);
    return S_OK;
}

createSrcReader 函数

创建 source reader 并指定回调接口。

HRESULT createSrcReader(IMFMediaSource *pSource, IMFSourceReader*& pReader, IUnknown* pCallback)
{
    HRESULT hr = S_OK;
    CComPtr<IMFAttributes> pAttributes = NULL;
    hr = MFCreateAttributes(&pAttributes, 2);
    RETURN_IF_FAILED(hr);
    
    hr = pAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, pCallback);
    RETURN_IF_FAILED(hr);
    
    /*
    By default, when the application releases the source reader, 
    the source reader shuts down the media source by calling IMFMediaSource::Shutdown on the media source. 
    At that point, the application can no longer use the media source.
    However, if the MF_SOURCE_READER_DISCONNECT_MEDIASOURCE_ON_SHUTDOWN attribute is TRUE, 
    the source reader does not shut down the media source. 
    That means the application can still use the media source after the application releases the source reader.
    */
    hr = pAttributes->SetUINT32(MF_SOURCE_READER_DISCONNECT_MEDIASOURCE_ON_SHUTDOWN, TRUE);
    RETURN_IF_FAILED(hr);
    
    hr = MFCreateSourceReaderFromMediaSource(pSource, pAttributes, &pReader);
    RETURN_IF_MF_FAILED(hr);
    
    return hr;
}

configSrcReader 函数

设置 source reader 的输出媒体类型。

    std::vector<GUID> subTypes;
    subTypes.push_back(MFVideoFormat_NV12);
    subTypes.push_back(MFVideoFormat_YUY2);
    subTypes.push_back(MFVideoFormat_RGB24);
    BOOL bUseNativeType = FALSE;
    
    for (int i = 0; ; ++i) {
        hr = pReader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, &pType);
        if (FAILED(hr)) 
            break;
            
        hr = pType->GetGUID(MF_MT_SUBTYPE, &subType);
        
        for (UINT32 i = 0; i < subTypes.size(); i++) {
            if (subType == subTypes[i]) {
                hr = pReader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, NULL, pType);
                if (SUCCEEDED(hr)) {
                    bUseNativeType = TRUE; 
                    break;
                }
            }
        }
        
        if (bUseNativeType)
            break;
        else 
            pType = NULL;
    }
    
    if (!bUseNativeType) {
        if (pType == NULL)
            hr = pReader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, &pType);
            
        for (UINT32 i = 0; i < subTypes.size(); i++) {
            hr = pType->SetGUID(MF_MT_SUBTYPE, subTypes[i]);
            hr = pReader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, NULL, pType);
            if (SUCCEEDED(hr)) 
                break;
        }
    }

configVideoEncoder 函数

设置编码器的一些基本参数，如码率、帧率、宽高等。

HRESULT configVideoEncoder( 
    EncodingParameters* params, 
    IMFMediaType *pType, 
    IMFSinkWriter *pWriter,
    DWORD *pdwStreamIndex )
{
    HRESULT hr = S_OK;
    CComPtr<IMFMediaType> pTargetType = NULL;
    hr = MFCreateMediaType(&pTargetType);   
    hr = pTargetType->SetGUID( MF_MT_MAJOR_TYPE, MFMediaType_Video );     
    hr = pTargetType->SetGUID(MF_MT_SUBTYPE, params->subType);
    hr = pTargetType->SetUINT32(MF_MT_AVG_BITRATE, params->bitRate);
    hr = copyAttribute(pType, pTargetType, MF_MT_FRAME_SIZE);
    hr = copyAttribute(pType, pTargetType, MF_MT_FRAME_RATE);    
    hr = copyAttribute(pType, pTargetType, MF_MT_PIXEL_ASPECT_RATIO);    
    hr = copyAttribute(pType, pTargetType, MF_MT_INTERLACE_MODE);    
    hr = pWriter->AddStream(pTargetType, pdwStreamIndex);
    
    return hr;
}

IMFSourceReaderCallback::OnReadSample 回调函数

Source reader 读取到一个 sample 后回调此函数，设置完时间戳后即交给 Sink Writer 编码并写入文件。

HRESULT CMFCapture::OnReadSample(HRESULT hrStatus, DWORD, DWORD, LONGLONG llTimeStamp, IMFSample *pSample)
{
    if (!isCapturing())
        return S_OK;
        
    SyncUtil::AutoLock lock(m_critsec);
    if (NULL == m_pWriter)
        return S_OK;
        
    HRESULT hr = S_OK;
    RETURN_IF_FAILED(hrStatus);
    
    if (NULL != pSample) {
        if (m_bFirstSample) {
            m_llBaseTime = llTimeStamp;
            m_bFirstSample = FALSE;
        }
        
        llTimeStamp -= m_llBaseTime;
        hr = pSample->SetSampleTime(llTimeStamp);
        hr = m_pWriter->WriteSample(m_videoStreamIdx, pSample);
        RETURN_IF_FAILED(hr);
    }
    
    hr = m_pVideoReader->ReadSample((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
    RETURN_IF_FAILED(hr);
    
    return S_OK;
}

CMFCapture::stopCapture 函数

在 Flush Source Reader 的时候，如果不等待 IMFSourceReaderCallback::OnFlush 回调完成，可能会造成死锁。

HRESULT CMFCapture::stopCapture()
{ 
    HRESULT hr = S_OK;
    m_isCapturing = false;
    SyncUtil::AutoLock lock(m_critsec);
    
    if (NULL != m_pWriter) {
        m_pWriter->Flush(m_videoStreamIdx);
        
        hr = m_pWriter->Finalize();
        SAFE_RELEASE(m_pWriter);
    }
    
    if (NULL != m_pVideoReader) {
        m_pVideoReader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM);
        WaitForSingleObject(m_hFlushedEvent, 3000);
        SAFE_RELEASE(m_pVideoReader);
    }
    
    return hr;
}

TDMETHODIMP OnFlush(DWORD dwStreamIndex)
{
    SetEvent(m_hFlushedEvent);
    return S_OK;
}

Tools

请参考我的另一篇关于 MF 的文章：音频采集 via Media Foundation

其他框架下的采集

请参考对应的文章。

• FFmpeg
• DirectShow

– EOF –