C++编程-2：使用Android MediaCodec 硬解码延时问题分析

      最近做项目用到Android native层的MediaCodec的接口对H264进行解码，通过在解码前和解码后加打印日志，发现解码耗时200多ms，和IOS的解码耗时10ms相比实在是延时好大。后来研究了两周也没能解决延时问题，很悲惨……不过还是将这过程中分析的思路记录下来，说不定以后万一灵感来了就解决了呢。

     起初在https://software.intel.com/en-us/forums/android-applications-on-intel-architecture/topic/536355和https://software.intel.com/en-us/android/articles/android-hardware-codec-mediacodec这两个网址中找到问题原因和一种解决方法。

      如果编码器类型是H.264，MediaCodec可能会有几帧的延迟（对于IA平台，它是7帧，而另一些则是3-5帧）。如果帧速率为30 FPS，则7帧将具有200毫秒的延迟。什么导致了硬件解码器的延迟？原因可能是主或高配置文件有一个B帧，并且如果当B帧引用后续缓冲区时解码器没有缓冲区，则该播放会短暂停止。英特尔已经考虑了硬件解码器的这种情况，并为主要或高配置文件提供了7个缓冲区，但是由于基线没有B帧，因此将其减少到基线的零缓冲区。其他供应商可能为所有配置文件使用4个缓冲区。
     因此，在基于IA的Android平台上，解决方案是将编码器更改为基线配置文件。如果编码器配置文件无法更改，则可行的解决方法是更改​​解码器端的配置数据。通常，配置文件位于第五个字节中，因此将基准配置文件更改为66会在IA平台上产生最低延迟。也就是说要想低延时，必须：

    - 不要设置MediaFormat.KEY_MAX_WIDTH和MediaFormat.KEY_MAX_HEIGHT。

     - 将第一个SPS中的配置文件切换到基线，然后在第一个PPS后以正确的配置文件重新提交。

      先附上解码部分代码：

 

int OpenMetaCUDAVideoDecoderContext::OnVideoH264(uint8_t * frame, uint32_t frameSize){

    DP_MSG("onVideoData=%6d, %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X\n",
           frameSize,
           frame[0],frame[1],frame[2],frame[3],frame[4],frame[5],frame[6],frame[7]
    );

    int  pos = 0;
    if( (frame[4] & 0X1F ) == 0X09 ){
        // this is an iframe
        frame     += 6;
        frameSize -= 6;
    }

    if( (frame[4] & 0X1F ) == 0X07 ){
        // this is an iframe
        iFrameFound = true;
    }
    else if(!iFrameFound){
        // iframe not found, do nothing
        return -1;
    }

    DP_MSG("video264::::%d",frameSize)

    uint8_t *data = NULL;
    uint8_t *pps = NULL;
    uint8_t *sps = NULL;

    // I know what my H.264 data source's NALUs look like so I know start code index is always 0.
    // if you don't know where it starts, you can use a for loop similar to how i find the 2nd and 3rd start codes
    int currentIndex = 0;
    long blockLength = 0;

    int nalu_type = (frame[currentIndex + 4] & 0x1F);
    // if we havent already set up our format description with our SPS PPS parameters, we
    // can't process any frames except type 7 that has our parameters
    if (nalu_type != 7 && formatDesc == NULL)
    {
        return -1;
    }

    if (nalu_type == 7)
    {
        currentIndex = H264_findStartCode(frame,frameSize,currentIndex+4);
        size_t spsSize = currentIndex;

        // find what the second NALU type is
        nalu_type = (frame[currentIndex + 4] & 0x1F);

        DP_MSG("onVideoDataSPs=%6d, %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X\n",
               frameSize,
               frame[0],frame[1],frame[2],frame[3],frame[4],frame[5],frame[6],frame[7]
        );

        // type 8 is the PPS parameter NALU
        if(nalu_type == 8){
            int nextIndex = H264_findStartCode(frame,frameSize,currentIndex+4);
            size_t ppsSize = nextIndex - currentIndex;

            DP_MSG("onVideoDataPPs=%6d, %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X\n",
                   frameSize,
                   frame[0],frame[1],frame[2],frame[3],frame[4],frame[5],frame[6],frame[7],
                   frame[spsSize+1],frame[spsSize+2],frame[spsSize+3],frame[spsSize+4]
            );

            if(decompressionSession == NULL) {
                formatDesc = AMediaFormat_new();
                AMediaFormat_setString(formatDesc, "mime", "video/avc");
                AMediaFormat_setInt32(formatDesc, AMEDIAFORMAT_KEY_WIDTH, 1920); // 视频宽度
                AMediaFormat_setInt32(formatDesc, AMEDIAFORMAT_KEY_HEIGHT, 1088); // 视频高度

                sps = (uint8_t *)malloc(spsSize);
                pps = (uint8_t *)malloc(ppsSize);
                memcpy (sps, &frame[0], spsSize);
                memcpy (pps, &frame[spsSize], ppsSize);

                AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize); // sps
                AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps

                DP_MSG("createDecompSession ========= ")
                this->createDecompSession();

                if(sps != NULL)
                {
                    free(sps);
                    sps = NULL;
                }
                if(pps != NULL)
                {
                    free(pps);
                    pps = NULL;
                }

            }

            // now lets handle the IDR frame that (should) come after the parameter sets
            // I say "should" because that's how I expect my H264 stream to work, YMMV
            currentIndex = nextIndex;
            nalu_type = (frame[currentIndex + 4] & 0x1F);

            while(currentIndex != -1 && nalu_type != 5 && nalu_type != 1){
                currentIndex = H264_findStartCode(frame,frameSize,currentIndex+4);
                nalu_type = (frame[currentIndex + 4] & 0x1F);
            }
        }
    }

    int32_t afout = 0;
       nalu_type = 5; // temp
    // type 5 is an IDR frame NALU.  The SPS and PPS NALUs should always be followed by an IDR (or IFrame) NALU, as far as I know
    if(nalu_type == 5)
    {

        DP_MSG("jniPlayTs size=%d and %d \n",AMediaFormat_getInt32(formatDesc,AMEDIAFORMAT_KEY_COLOR_FORMAT,&afout),afout);
        //DP_MSG("theNativeWindow format %p and %d \n",theNativeWindow,ANativeWindow_getFormat(theNativeWindow));
        // find the offset, or where the SPS and PPS NALUs end and the IDR frame NALU begins
        int offset = currentIndex;//
        blockLength = frameSize - offset;

        data = &frame[offset];

        DP_MSG("nalu_type ===5 ")
        //DP_MSG("nalu_type ===5 %s /n ",AMediaFormat_toString(formatDesc))

        if(decompressionSession == nullptr)
        {
            DP_MSG("decompressionSession is null")
        }

        DP_MSG("OpenMetaCUDAVideoDecoder Start decoding: frameIdentifier=%lld",frameIdentifier);
        avx_info("OpenMetaCUDAVideoDecoder Start decoding: ","frameIdentifier=%lld",frameIdentifier);

        static int64_t  llPts = 0;

        ssize_t index = -1;
        index = AMediaCodec_dequeueInputBuffer(decompressionSession, 10000);
        if(index>=0)
        {
            size_t bufsize;
            auto buf = AMediaCodec_getInputBuffer(decompressionSession, index, &bufsize);
            if(buf!= nullptr )
            {
                DP_MSG("buf!= nullptr5======ww")
                memcpy(buf,data,blockLength);

                AMediaCodec_queueInputBuffer(decompressionSession, index, 0, blockLength, (uint64_t)frameIdentifier, 0);

                llPts += 3000;
            }

        }

        AMediaCodecBufferInfo info;
        auto status = AMediaCodec_dequeueOutputBuffer(decompressionSession, &info, 0);
        if (status >= 0) {
            if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                DP_MSG("output EOS");
            }

            size_t outsize = 0;
            u_int8_t * outData = AMediaCodec_getOutputBuffer(decompressionSession,status,&outsize);
            auto formatType = AMediaCodec_getOutputFormat(decompressionSession);
            DP_MSG("format formatType to: %s", AMediaFormat_toString(formatType));
            int colorFormat = 0;
            int width = 0;
            int height = 0;
            AMediaFormat_getInt32(formatType,"color-format",&colorFormat);
            AMediaFormat_getInt32(formatType,"width",&width);
            AMediaFormat_getInt32(formatType,"height",&height);
            DP_MSG("format color-format : %d width ：%d height :%d", colorFormat,width,height);

			AMediaFormat_delete(formatType);
            DP_MSG("OpenMetaCUDAVideoDecoder End   decoding: frameIdentifier=%lld,%d,%d,%p",info.presentationTimeUs,info.size,outsize,outData)
			if(this->kController){
                //OpenMetaPixelBuffer _kPixelBuffer(pixelBuffer,sizeof(pixelBuffer));
                OpenMetaPixelBuffer _kPixelBuffer(outData,sizeof(outData));
                _kPixelBuffer.kLine[0]  = outData;
                _kPixelBuffer.kLine[1]  = outData + width * height;
                _kPixelBuffer.kSizeX    = width;
                _kPixelBuffer.kSizeY    = height;
                _kPixelBuffer.kDataType = 1;
                _kPixelBuffer.kTimeStamp = info.presentationTimeUs;

                if(this!=NULL && this->kController!=NULL)
                {
                    this->kController->OnVideoImage(&_kPixelBuffer);
                }

            }
            avx_info("OpenMetaCUDAVideoDecoder End   decoding:","%lld,%d,%d,%p",info.presentationTimeUs,info.size,outsize,outData);
            AMediaCodec_releaseOutputBuffer(decompressionSession, status, info.size != 0);

        } else if (status == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
            DP_MSG("output buffers changed");
        } else if (status == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
            auto format = AMediaCodec_getOutputFormat(decompressionSession);
            DP_MSG("format changed to: %s", AMediaFormat_toString(format));
            AMediaFormat_delete(format);
        } else if (status == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
            DP_MSG("no output buffer right now");
        } else {
            DP_MSG("unexpected info code: %zd", status);
        }
      
    }

    return 0;

}

 

      针对查到的解决办法，对以上代码进行了修改，分别是：

 

 

//将第一个sps的改为基线配置
sps = (uint8_t *)malloc(spsSize);
sps[0+5] = 66; 
memcpy (sps, &frame[0], spsSize);

pps = (uint8_t *)malloc(ppsSize);
memcpy (pps, &frame[spsSize], ppsSize);

AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps

 

//拷贝两份sps，将第一个sps的改为基线配置
sps = (uint8_t *)malloc(spsSize * 2);
memcpy (sps, &frame[0], spsSize);
sps[0+5] = 66; 
memcpy (sps + spsSize, &frame[0], spsSize);

pps = (uint8_t *)malloc(ppsSize);
memcpy (pps, &frame[spsSize], ppsSize);

AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize * 2); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps

//拷贝两份sps，将第二份的第一个sps的改为基线配置
sps = (uint8_t *)malloc(spsSize * 2);
memcpy (sps, &frame[0], spsSize);
memcpy (sps + spsSize, &frame[0], spsSize);
sps[spsSize+5] = 66; 

pps = (uint8_t *)malloc(ppsSize);
memcpy (pps, &frame[spsSize], ppsSize);

AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize * 2); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps

     通过查看打印的开始解码与结束解码的时间发现，几种方法的时间差距并不是很大，依旧是200-250ms左右。