Android Multimedia框架总结（二十七）MediaCodec回顾

Android Multimedia框架总结（二十七）MediaCodec回顾

2017年06月04日 15:35:34 逆流的鱼yuiop 阅读数：2968 标签： androidmediacodec更多

个人分类： MultiMediaFramework

所属专栏： MultiMedia框架总结(基于6.0源码)

版权声明：我已委托“维权骑士”（rightknights.com）为我的文章进行维权行动.转载务必转载所有，且须注明出处。否则保留追究法律责任 https://blog.csdn.net/hejjunlin/article/details/72859142

Android App 通过 MediaCodec Java API 获得的编解码器，实际上是由 StageFright 媒体框架提供。android.media.MediaCodec 调用 libmedia_jni.so 中 JNI native 函数，这些 JNI 函数再去调用 libstagefright.so 库获得 StageFright 框架中的编解码器。StageFright再调用OMX组件进行解码。这是之前梳理的流程。这次再读主要是搞明白他们的调用过程。

先看Java层代码：

mediaCodec = MediaCodec.createByCodecName(codecName);//创建Codec
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MINE_TYPE, width, height);

mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bit);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, fps);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); // 关键帧间隔时间// 单位s
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,                                    
                MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar);
                // mediaFormat.setInteger(MediaFormat.KEY_PROFILE,
                // CodecProfileLevel.AVCProfileBaseline);
                // mediaFormat.setInteger(MediaFormat.KEY_LEVEL,
                // CodecProfileLevel.AVCLevel52);
                mediaCodec.configure(mediaFormat, null, null,
                        MediaCodec.CONFIGURE_FLAG_ENCODE);
                mediaCodec.start();

Java层： 
createByName->New MediaCode()->setup(native声明方法) 
JNI层： 
1、映射（android_media_MediaCodec.cpp 中）

    { "native_setup", "(Ljava/lang/String;ZZ)V",
      (void *)android_media_MediaCodec_native_setup },

2、setup->android_media_MediaCodec_native_setup

static void android_media_MediaCodec_native_setup(
        JNIEnv *env, jobject thiz,
        jstring name, jboolean nameIsType, jboolean encoder) {
    if (name == NULL) {
        //这里可以学习，jni层如何抛出异常的。使用jniThrowException
        jniThrowException(env, "java/lang/NullPointerException", NULL);
        return;
    }

    const char *tmp = env->GetStringUTFChars(name, NULL);

    if (tmp == NULL) {
        return;
    }

    sp<JMediaCodec> codec = new JMediaCodec(env, thiz, tmp, nameIsType, encoder);

    const status_t err = codec->initCheck();
    if (err == NAME_NOT_FOUND) {
        // fail and do not try again.
        jniThrowException(env, "java/lang/IllegalArgumentException",
                String8::format("Failed to initialize %s, error %#x", tmp, err));
        env->ReleaseStringUTFChars(name, tmp);
        return;
    } if (err == NO_MEMORY) {
        throwCodecException(env, err, ACTION_CODE_TRANSIENT,
                String8::format("Failed to initialize %s, error %#x", tmp, err));
        env->ReleaseStringUTFChars(name, tmp);
        return;
    } else if (err != OK) {
        // believed possible to try again
        jniThrowException(env, "java/io/IOException",
                String8::format("Failed to find matching codec %s, error %#x", tmp, err));
        env->ReleaseStringUTFChars(name, tmp);
        return;
    }

    env->ReleaseStringUTFChars(name, tmp);

    codec->registerSelf();

    setMediaCodec(env,thiz, codec);
}

3、重点是 sp codec = new JMediaCodec(env, thiz, tmp, nameIsType, encoder); 
sp是智能指针中强指针（Strong Pointer），智能指针主要灵活管理内存相关。还有wp（Weak Pointer），弱引用指针对象，在 、system\core\include\utils\StrongPointer.h 下，直接可以拷贝这套google的东西，运用到项目也是可以的。

4、看下构造函数，

JMediaCodec::JMediaCodec(
        JNIEnv *env, jobject thiz,
        const char *name, bool nameIsType, bool encoder)
    : mClass(NULL),
      mObject(NULL) {
    jclass clazz = env->GetObjectClass(thiz);//获取class
    CHECK(clazz != NULL);

    mClass = (jclass)env->NewGlobalRef(clazz);//全局引用
    mObject = env->NewWeakGlobalRef(thiz);//弱全局引用

    cacheJavaObjects(env);

    mLooper = new ALooper;
    mLooper->setName("MediaCodec_looper");

    mLooper->start(
            false,      // runOnCallingThread
            true,       // canCallJava
            PRIORITY_FOREGROUND);

    if (nameIsType) {
        mCodec = MediaCodec::CreateByType(mLooper, name, encoder, &mInitStatus);
    } else {
        mCodec = MediaCodec::CreateByComponentName(mLooper, name, &mInitStatus);
    }
    CHECK((mCodec != NULL) != (mInitStatus != OK));
}

•  

mCodec = MediaCodec::CreateByType(mLooper, name, encoder, &mInitStatus)，//这里的MedCodec就是stagefright中MediaCodec，位于 \frameworks\av\media\libstagefright\MediaCodec.cpp，隶属于libstagefright.so

5、进入MediaCodec.cpp中

// static
sp<MediaCodec> MediaCodec::CreateByType(
        const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err, pid_t pid) {
    sp<MediaCodec> codec = new MediaCodec(looper, pid);

    const status_t ret = codec->init(mime, true /* nameIsType */, encoder);
    if (err != NULL) {
        *err = ret;
    }
    return ret == OK ? codec : NULL; // NULL deallocates codec.
}

6、构造

MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid)
    : mState(UNINITIALIZED),
      mReleasedByResourceManager(false),
      mLooper(looper),
      mCodec(NULL),
      mReplyID(0),
      mFlags(0),
      mStickyError(OK),
      mSoftRenderer(NULL),
      mResourceManagerClient(new ResourceManagerClient(this)),
      mResourceManagerService(new ResourceManagerServiceProxy(pid)),
      mBatteryStatNotified(false),
      mIsVideo(false),
      mVideoWidth(0),
      mVideoHeight(0),
      mRotationDegrees(0),
      mDequeueInputTimeoutGeneration(0),
      mDequeueInputReplyID(0),
      mDequeueOutputTimeoutGeneration(0),
      mDequeueOutputReplyID(0),
      mHaveInputSurface(false),
      mHavePendingInputBuffers(false) {
}

这里有点意思，直接把MediaCodec.h中头文件中变量进行初始化，注意：（冒号），看MediaCodec.h中，第一句就是struct MediaCodec : public AHandler，也就是说MediaCodec是一个结构体，这个结构体继承AHandler（也是一个结构体），AHandler继承utils/RefBase.h，为什么大量使用结构体，而不是class？

• 1、结构体在堆栈中创建，是值类型，速度回比较快
• 2、也考虑到和C兼容，C也有结构体。
• 3、通常用来处理作为基类型对待的小对象
• 4、c++ 里面结构体是可以继承的（纠正网上很多说结构体，不能继承的说法，这里源码也可以得到印证，结构体不仅能继承其他，也能被其他继承）

struct 与class本质上应该是相同的，只是默认的访问权限不同（struct默认是public，class默认是private ），网上也有人说结构体不能定义虚函数，这种说法也不对，如下代码也是MediaCodec.h中的，不仅可以定义虚函数，还可以定义友元。

protected:
    virtual ~MediaCodec();
    virtual void onMessageReceived(const sp<AMessage> &msg);

private:
    // used by ResourceManagerClient
    status_t reclaim(bool force = false);
    friend struct ResourceManagerClient;

最后疑惑点

在阅读时，还发现有NdkMediaCodec及NdkMediaCodec.cpp这些个class, 和上面几个class的区别是什么？有什么关系？为什么要这么设计？ 
frameworks\av\include\ndk\NdkMediaCodec.h

/*
 * This file defines an NDK API.
 * Do not remove methods.
 * Do not change method signatures.
 * Do not change the value of constants.
 * Do not change the size of any of the classes defined in here.
 * Do not reference types that are not part of the NDK.
 * Do not #include files that aren't part of the NDK.
 */

#ifndef _NDK_MEDIA_CODEC_H
#define _NDK_MEDIA_CODEC_H

#include <android/native_window.h>

#include "NdkMediaCrypto.h"
#include "NdkMediaError.h"
#include "NdkMediaFormat.h"

#ifdef __cplusplus
extern "C" {
#endif


struct AMediaCodec;
typedef struct AMediaCodec AMediaCodec;

struct AMediaCodecBufferInfo {
    int32_t offset;
    int32_t size;
    int64_t presentationTimeUs;
    uint32_t flags;
};
typedef struct AMediaCodecBufferInfo AMediaCodecBufferInfo;
typedef struct AMediaCodecCryptoInfo AMediaCodecCryptoInfo;

enum {
    AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM = 4,
    AMEDIACODEC_CONFIGURE_FLAG_ENCODE = 1,
    AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED = -3,
    AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED = -2,
    AMEDIACODEC_INFO_TRY_AGAIN_LATER = -1
};

/**
 * Create codec by name. Use this if you know the exact codec you want to use.
 * When configuring, you will need to specify whether to use the codec as an
 * encoder or decoder.
 */
AMediaCodec* AMediaCodec_createCodecByName(const char *name);

/**
 * Create codec by mime type. Most applications will use this, specifying a
 * mime type obtained from media extractor.
 */
AMediaCodec* AMediaCodec_createDecoderByType(const char *mime_type);

/**
 * Create encoder by name.
 */
AMediaCodec* AMediaCodec_createEncoderByType(const char *mime_type);

/**
 * delete the codec and free its resources
 */
media_status_t AMediaCodec_delete(AMediaCodec*);

/**
 * Configure the codec. For decoding you would typically get the format from an extractor.
 */
media_status_t AMediaCodec_configure(
        AMediaCodec*,
        const AMediaFormat* format,
        ANativeWindow* surface,
        AMediaCrypto *crypto,
        uint32_t flags);

/**
 * Start the codec. A codec must be configured before it can be started, and must be started
 * before buffers can be sent to it.
 */
media_status_t AMediaCodec_start(AMediaCodec*);

/**
 * Stop the codec.
 */
media_status_t AMediaCodec_stop(AMediaCodec*);

/*
 * Flush the codec's input and output. All indices previously returned from calls to
 * AMediaCodec_dequeueInputBuffer and AMediaCodec_dequeueOutputBuffer become invalid.
 */
media_status_t AMediaCodec_flush(AMediaCodec*);

/**
 * Get an input buffer. The specified buffer index must have been previously obtained from
 * dequeueInputBuffer, and not yet queued.
 */
uint8_t* AMediaCodec_getInputBuffer(AMediaCodec*, size_t idx, size_t *out_size);

/**
 * Get an output buffer. The specified buffer index must have been previously obtained from
 * dequeueOutputBuffer, and not yet queued.
 */
uint8_t* AMediaCodec_getOutputBuffer(AMediaCodec*, size_t idx, size_t *out_size);

/**
 * Get the index of the next available input buffer. An app will typically use this with
 * getInputBuffer() to get a pointer to the buffer, then copy the data to be encoded or decoded
 * into the buffer before passing it to the codec.
 */
ssize_t AMediaCodec_dequeueInputBuffer(AMediaCodec*, int64_t timeoutUs);

/**
 * Send the specified buffer to the codec for processing.
 */
media_status_t AMediaCodec_queueInputBuffer(AMediaCodec*,
        size_t idx, off_t offset, size_t size, uint64_t time, uint32_t flags);

/**
 * Send the specified buffer to the codec for processing.
 */
media_status_t AMediaCodec_queueSecureInputBuffer(AMediaCodec*,
        size_t idx, off_t offset, AMediaCodecCryptoInfo*, uint64_t time, uint32_t flags);

/**
 * Get the index of the next available buffer of processed data.
 */
ssize_t AMediaCodec_dequeueOutputBuffer(AMediaCodec*, AMediaCodecBufferInfo *info,
        int64_t timeoutUs);
AMediaFormat* AMediaCodec_getOutputFormat(AMediaCodec*);

/**
 * If you are done with a buffer, use this call to return the buffer to
 * the codec. If you previously specified a surface when configuring this
 * video decoder you can optionally render the buffer.
 */
media_status_t AMediaCodec_releaseOutputBuffer(AMediaCodec*, size_t idx, bool render);

/**
 * If you are done with a buffer, use this call to update its surface timestamp
 * and return it to the codec to render it on the output surface. If you
 * have not specified an output surface when configuring this video codec,
 * this call will simply return the buffer to the codec.
 *
 * For more details, see the Java documentation for MediaCodec.releaseOutputBuffer.
 */
media_status_t AMediaCodec_releaseOutputBufferAtTime(
        AMediaCodec *mData, size_t idx, int64_t timestampNs);


typedef enum {
    AMEDIACODECRYPTOINFO_MODE_CLEAR = 0,
    AMEDIACODECRYPTOINFO_MODE_AES_CTR = 1
} cryptoinfo_mode_t;

/**
 * Create an AMediaCodecCryptoInfo from scratch. Use this if you need to use custom
 * crypto info, rather than one obtained from AMediaExtractor.
 *
 * AMediaCodecCryptoInfo describes the structure of an (at least
 * partially) encrypted input sample.
 * A buffer's data is considered to be partitioned into "subsamples",
 * each subsample starts with a (potentially empty) run of plain,
 * unencrypted bytes followed by a (also potentially empty) run of
 * encrypted bytes.
 * numBytesOfClearData can be null to indicate that all data is encrypted.
 * This information encapsulates per-sample metadata as outlined in
 * ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base media file format files".
 */
AMediaCodecCryptoInfo *AMediaCodecCryptoInfo_new(
        int numsubsamples,
        uint8_t key[16],
        uint8_t iv[16],
        cryptoinfo_mode_t mode,
        size_t *clearbytes,
        size_t *encryptedbytes);

/**
 * delete an AMediaCodecCryptoInfo created previously with AMediaCodecCryptoInfo_new, or
 * obtained from AMediaExtractor
 */
media_status_t AMediaCodecCryptoInfo_delete(AMediaCodecCryptoInfo*);

/**
 * The number of subsamples that make up the buffer's contents.
 */
size_t AMediaCodecCryptoInfo_getNumSubSamples(AMediaCodecCryptoInfo*);

/**
 * A 16-byte opaque key
 */
media_status_t AMediaCodecCryptoInfo_getKey(AMediaCodecCryptoInfo*, uint8_t *dst);

/**
 * A 16-byte initialization vector
 */
media_status_t AMediaCodecCryptoInfo_getIV(AMediaCodecCryptoInfo*, uint8_t *dst);

/**
 * The type of encryption that has been applied,
 * one of AMEDIACODECRYPTOINFO_MODE_CLEAR or AMEDIACODECRYPTOINFO_MODE_AES_CTR.
 */
cryptoinfo_mode_t AMediaCodecCryptoInfo_getMode(AMediaCodecCryptoInfo*);

/**
 * The number of leading unencrypted bytes in each subsample.
 */
media_status_t AMediaCodecCryptoInfo_getClearBytes(AMediaCodecCryptoInfo*, size_t *dst);

/**
 * The number of trailing encrypted bytes in each subsample.
 */
media_status_t AMediaCodecCryptoInfo_getEncryptedBytes(AMediaCodecCryptoInfo*, size_t *dst);

#ifdef __cplusplus
} // extern "C"
#endif

#endif //_NDK_MEDIA_CODEC_H

在AMediaCodec中，有一个createAMediaCodec，如下：

static AMediaCodec * createAMediaCodec(const char *name, bool name_is_type, bool encoder) {
    AMediaCodec *mData = new AMediaCodec();
    mData->mLooper = new ALooper;
    mData->mLooper->setName("NDK MediaCodec_looper");
    status_t ret = mData->mLooper->start(
            false,      // runOnCallingThread
            true,       // canCallJava XXX
            PRIORITY_FOREGROUND);
    if (name_is_type) {
        mData->mCodec = android::MediaCodec::CreateByType(mData->mLooper, name, encoder);
    } else {
        mData->mCodec = android::MediaCodec::CreateByComponentName(mData->mLooper, name);
    }
    if (mData->mCodec == NULL) {  // failed to create codec
        AMediaCodec_delete(mData);
        return NULL;
    }
    mData->mHandler = new CodecHandler(mData);
    mData->mLooper->registerHandler(mData->mHandler);
    mData->mGeneration = 1;
    mData->mRequestedActivityNotification = false;
    mData->mCallback = NULL;

    return mData;
}

其中也有一个android::MediaCodec::CreateByType(mData->mLooper, name, encoder);调用的是MediaCodec的CreateByType函数。这个类作用暂时不明确，看Google标识是勿动，勿改，勿乱搞，这是个NDK API。