Android-video-视频软硬件编解码器的集成

　安卓机器中都有很多视频软硬件编解码器，例如: h264/hevc/h263/mpeg4……等等。但是如果现在有一个新的符合标准的编码器给你，需要怎么集成呢？阅读本文你将会找到答案。

1. 软件编码器的集成:

　　集成之前需要先了解软件编码器的加载过程:

1.1 软件编码器的加载过程

// frameworks/av/media/libstagefright/ACodec.cpp
step1: ACodec收到消息kWhatAllocateComponent或者在onSetup的时候就会调用onAllocateComponent。

step2: ACodec调用omx去创建组件
bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) {
    ALOGV("onAllocateComponent");

    CHECK(mCodec->mOMXNode == NULL);

    sp<AMessage> notify = new AMessage(kWhatOMXDied, mCodec);

    sp<RefBase> obj;
    CHECK(msg->findObject("codecInfo", &obj));
    sp<MediaCodecInfo> info = (MediaCodecInfo *)obj.get(); // 获取codec信息
    AString owner = "default";
    AString componentName;
    CHECK(msg->findString("componentName", &componentName)); // 获取componentName

    //　确认使用的是平台的codec
    if (!(componentName.find("qcom", 0) > 0 || componentName.find("qti", 0) > 0) ||
          componentName.find("video", 0) > 0 || componentName.find("flac", 0) > 0) {
        if (info == nullptr) {
            ALOGE("Unexpected nullptr for codec information");
            mCodec->signalError(OMX_ErrorUndefined, UNKNOWN_ERROR);
            return false;
        }
        owner = (info->getOwnerName() == nullptr) ? "default" : info->getOwnerName();
    }

    sp<CodecObserver> observer = new CodecObserver; // 创建CodecObserver
    sp<IOMX> omx;
    sp<IOMXNode> omxNode;

    status_t err = NAME_NOT_FOUND;
    OMXClient client;
    if (client.connect(owner.c_str()) != OK) { // 能否找到对应的codec
        mCodec->signalError(OMX_ErrorUndefined, NO_INIT);
        return false;
    }
    omx = client.interface(); // 给omx赋值???

    pid_t tid = gettid();
    int prevPriority = androidGetThreadPriority(tid);
    androidSetThreadPriority(tid, ANDROID_PRIORITY_FOREGROUND); // 设置线程优先级
    err = omx->allocateNode(componentName.c_str(), observer, &omxNode); // 调用allocateNode，把CodecObserver和omxNode联系起来
    androidSetThreadPriority(tid, prevPriority); //　线程优先级恢复

    if (err != OK) {
        ALOGE("Unable to instantiate codec '%s' with err %#x.", componentName.c_str(), err);

        mCodec->signalError((OMX_ERRORTYPE)err, makeNoSideEffectStatus(err));
        return false;
    }

    mDeathNotifier = new DeathNotifier(notify); // mDeathNotifier的作用？？？
    auto tOmxNode = omxNode->getHalInterface();
    if (!tOmxNode->linkToDeath(mDeathNotifier, 0)) {
        mDeathNotifier.clear();
    }

    notify = new AMessage(kWhatOMXMessageList, mCodec);
    notify->setInt32("generation", ++mCodec->mNodeGeneration);
    observer->setNotificationMessage(notify);

    mCodec->mComponentName = componentName;
    mCodec->mRenderTracker.setComponentName(componentName);
    mCodec->mFlags = 0;

    if (componentName.endsWith(".secure")) {
        mCodec->mFlags |= kFlagIsSecure;
        mCodec->mFlags |= kFlagIsGrallocUsageProtected;
        mCodec->mFlags |= kFlagPushBlankBuffersToNativeWindowOnShutdown;
    }

    mCodec->mOMX = omx;
    mCodec->mOMXNode = omxNode;
    mCodec->mCallback->onComponentAllocated(mCodec->mComponentName.c_str());
    mCodec->changeState(mCodec->mLoadedState);

    return true;
}

// frameworks/av/media/libstagefright/omx/1.0/Omx.cpp
Return<void> Omx::allocateNode(
        const hidl_string& name,
        const sp<IOmxObserver>& observer,
        allocateNode_cb _hidl_cb) {

    using ::android::IOMXNode;
    using ::android::IOMXObserver;

    sp<OMXNodeInstance> instance;
    {
        Mutex::Autolock autoLock(mLock);
        if (mLiveNodes.size() == kMaxNodeInstances) {
            ..... // 处理异常
        }

        instance = new OMXNodeInstance( // 新建OMXNodeInstance
                this, new LWOmxObserver(observer), name.c_str());

        OMX_COMPONENTTYPE *handle;
        // 调用OMXMaster的makeComponentInstance
        OMX_ERRORTYPE err = mMaster->makeComponentInstance(
                name.c_str(), &OMXNodeInstance::kCallbacks,
                instance.get(), &handle);

        if (err != OMX_ErrorNone) {
            .....　// 处理异常
        }
        instance->setHandle(handle); // setHandle

        // Find quirks from mParser
        const auto& codec = mParser.getCodecMap().find(name.c_str());
        if (codec == mParser.getCodecMap().cend()) {
            LOG(WARNING) << "Failed to obtain quirks for omx component "
                    "'" << name.c_str() << "' "
                    "from XML files";
        } else {
            uint32_t quirks = 0;
            for (const auto& quirk : codec->second.quirkSet) {
                if (quirk == "requires-allocate-on-input-ports") {
                    quirks |= OMXNodeInstance::
                            kRequiresAllocateBufferOnInputPorts;
                }
                if (quirk == "requires-allocate-on-output-ports") {
                    quirks |= OMXNodeInstance::
                            kRequiresAllocateBufferOnOutputPorts;
                }
            }
            instance->setQuirks(quirks);
        }

        mLiveNodes.add(observer.get(), instance);
        mNode2Observer.add(instance.get(), observer.get());
    }
    observer->linkToDeath(this, 0);

    _hidl_cb(toStatus(OK), new TWOmxNode(instance));
    return Void();
}

// frameworks/av/media/libstagefright/omx/OMXMaster.cpp
OMX_ERRORTYPE OMXMaster::makeComponentInstance(
        const char *name,
        const OMX_CALLBACKTYPE *callbacks,
        OMX_PTR appData,
        OMX_COMPONENTTYPE **component) {
    ALOGI("makeComponentInstance(%s) in %s process", name, mProcessName);
    Mutex::Autolock autoLock(mLock);

    *component = NULL;

    ssize_t index = mPluginByComponentName.indexOfKey(String8(name));

    if (index < 0) {
        return OMX_ErrorInvalidComponentName;
    }

    OMXPluginBase *plugin = mPluginByComponentName.valueAt(index);
    // 如果是软件编解码器，调用SoftOMXPlugin的makeComponentInstance,如果是硬件就调用QComOMXPlugin的makeComponentInstance。
    OMX_ERRORTYPE err =
        plugin->makeComponentInstance(name, callbacks, appData, component);

    if (err != OMX_ErrorNone) {
        return err;
    }

    mPluginByInstance.add(*component, plugin);

    return err;
}


// frameworks/av/media/libstagefright/omx/SoftOMXPlugin.cpp
OMX_ERRORTYPE SoftOMXPlugin::makeComponentInstance(
        const char *name,
        const OMX_CALLBACKTYPE *callbacks,
        OMX_PTR appData,
        OMX_COMPONENTTYPE **component) {
    ALOGV("makeComponentInstance '%s'", name);

    for (size_t i = 0; i < kNumComponents; ++i) {
        if (strcmp(name, kComponents[i].mName)) {
            continue;
        }　// 此处是根据名字在kComponents中循环去找到匹配的编解码器

        AString libName = "libstagefright_soft_";　// 编解码器name补全
        libName.append(kComponents[i].mLibNameSuffix);
        libName.append(".so");

　　　　　// dlopen打开编解码器
        void *libHandle = dlopen(libName.c_str(), RTLD_NOW|RTLD_NODELETE);

        if (libHandle == NULL) {
            ALOGE("unable to dlopen %s: %s", libName.c_str(), dlerror());

            return OMX_ErrorComponentNotFound;
        }

        typedef SoftOMXComponent *(*CreateSoftOMXComponentFunc)(
                const char *, const OMX_CALLBACKTYPE *,
                OMX_PTR, OMX_COMPONENTTYPE **);

        CreateSoftOMXComponentFunc createSoftOMXComponent =
            (CreateSoftOMXComponentFunc)dlsym(
                    libHandle,
                    "_Z22createSoftOMXComponentPKcPK16OMX_CALLBACKTYPE"
                    "PvPP17OMX_COMPONENTTYPE");

        if (createSoftOMXComponent == NULL) {
            dlclose(libHandle);
            libHandle = NULL;

            return OMX_ErrorComponentNotFound;
        }

        sp<SoftOMXComponent> codec =
            (*createSoftOMXComponent)(name, callbacks, appData, component);

        if (codec == NULL) {
            dlclose(libHandle);
            libHandle = NULL;

            return OMX_ErrorInsufficientResources;
        }

        OMX_ERRORTYPE err = codec->initCheck();
        if (err != OMX_ErrorNone) {
            dlclose(libHandle);
            libHandle = NULL;

            return err;
        }

        codec->incStrong(this);
        codec->setLibHandle(libHandle);

        return OMX_ErrorNone;
    }

    return OMX_ErrorInvalidComponentName;
}

从上面的过程来看，最终就是根据软件编码器的name来加载的，在SoftOMXPlugin中首先根据codec name在kComponents中寻找对应的编解码器的名字，然后补齐即可。

kComponents内容如下:

static const struct {
    const char *mName;
    const char *mLibNameSuffix;
    const char *mRole;

} kComponents[] = {
    // two choices for aac decoding.
    // configurable in media/libstagefright/data/media_codecs_google_audio.xml
    // default implementation
    { "OMX.google.aac.decoder", "aacdec", "audio_decoder.aac" },
    // alternate implementation
    { "OMX.google.xaac.decoder", "xaacdec", "audio_decoder.aac" },
    { "OMX.google.aac.encoder", "aacenc", "audio_encoder.aac" },
    { "OMX.google.amrnb.decoder", "amrdec", "audio_decoder.amrnb" },
    { "OMX.google.amrnb.encoder", "amrnbenc", "audio_encoder.amrnb" },
    { "OMX.google.amrwb.decoder", "amrdec", "audio_decoder.amrwb" },
    { "OMX.google.amrwb.encoder", "amrwbenc", "audio_encoder.amrwb" },
    { "OMX.google.h264.decoder", "avcdec", "video_decoder.avc" },
    { "OMX.google.h264.encoder", "avcenc", "video_encoder.avc" },
    { "OMX.google.hevc.decoder", "hevcdec", "video_decoder.hevc" },
    { "OMX.google.g711.alaw.decoder", "g711dec", "audio_decoder.g711alaw" },
    { "OMX.google.g711.mlaw.decoder", "g711dec", "audio_decoder.g711mlaw" },
    { "OMX.google.mpeg2.decoder", "mpeg2dec", "video_decoder.mpeg2" },
    { "OMX.google.h263.decoder", "mpeg4dec", "video_decoder.h263" },
    { "OMX.google.h263.encoder", "mpeg4enc", "video_encoder.h263" },
    { "OMX.google.mpeg4.decoder", "mpeg4dec", "video_decoder.mpeg4" },
    { "OMX.google.mpeg4.encoder", "mpeg4enc", "video_encoder.mpeg4" },
    { "OMX.google.mp3.decoder", "mp3dec", "audio_decoder.mp3" },
    { "OMX.google.vorbis.decoder", "vorbisdec", "audio_decoder.vorbis" },
    { "OMX.google.opus.decoder", "opusdec", "audio_decoder.opus" },
    { "OMX.google.vp8.decoder", "vpxdec", "video_decoder.vp8" },
    { "OMX.google.vp9.decoder", "vpxdec", "video_decoder.vp9" },
    { "OMX.google.vp8.encoder", "vpxenc", "video_encoder.vp8" },
    { "OMX.google.vp9.encoder", "vpxenc", "video_encoder.vp9" },
    { "OMX.google.raw.decoder", "rawdec", "audio_decoder.raw" },
    { "OMX.google.flac.decoder", "flacdec", "audio_decoder.flac" },
    { "OMX.google.flac.encoder", "flacenc", "audio_encoder.flac" },
    { "OMX.google.gsm.decoder", "gsmdec", "audio_decoder.gsm" },
#ifdef QTI_FLAC_DECODER
    { "OMX.qti.audio.decoder.flac", "qtiflacdec", "audio_decoder.flac" },
#endif
};

1.2 软件编解码器的集成

　　看懂了上面的加载过程，集成就特别简单了。

　　step1: 我们的编解码器需要符合统一命名规则，即libstagefright_soft_xxx.so的格式。

　　step2: 把mLibNameSuffix即xxx对应的内容添加到kComponents中。(例如我们需要把"OMX.google.h264.encoder"对应的编码器改为libstagefright_soft_xxx.so则只需把kComponents的:{ "OMX.google.h264.encoder", "avcenc", "video_encoder.avc" },改为

{ "OMX.google.h264.encoder", "xxx", "video_encoder.avc" },)

　　step3: 重编libstagefright_omx.so，并将其push到system/lib system/lib/vndk-28 system/lib64 system/lib64/vndk-28 vendor/lib/vndk vendor/lib64/vndk目录下。

　　step4: 把需要替换的libstagefright_soft_xxx.so放到vendor/lib和vendor/lib64下。

　　step5: reboot之后即可使用。

 

2. 硬件编解码器的集成

　　上面介绍了软件编解码器的集成，实际从acodec到OmxMaster部分软硬件都是相同的，只是在plugin时软件使用的SoftOMXPlugin，硬件使用QComOMXPlugin。

2.1 硬解编解码器的plugin

// hardware/qcom/media/libstagefrighthw/QComOMXPlugin.cpp
OMX_ERRORTYPE QComOMXPlugin::makeComponentInstance(
        const char *name,
        const OMX_CALLBACKTYPE *callbacks,
        OMX_PTR appData,
        OMX_COMPONENTTYPE **component) {
    if (mLibHandle == NULL) { // 此处mLibHandle在结构体构造时就赋值dlopen("libOmxCore.so", RTLD_NOW)
        return OMX_ErrorUndefined;
    }

    return (*mGetHandle)( // mGetHandle = (GetHandleFunc)dlsym(mLibHandle, "OMX_GetHandle");
// 也就是mGetHandle对应的实现在libOmxCore.so的OMX_GetHandle这个函数。
            reinterpret_cast<OMX_HANDLETYPE *>(component),
            const_cast<char *>(name),
            appData, const_cast<OMX_CALLBACKTYPE *>(callbacks));
}

// /hardware/qcom/media/mm-core/src/common/qc_omx_core.c
OMX_API OMX_ERRORTYPE OMX_APIENTRY
OMX_GetHandle(OMX_OUT OMX_HANDLETYPE*     handle,
              OMX_IN OMX_STRING    componentName,
              OMX_IN OMX_PTR             appData,
              OMX_IN OMX_CALLBACKTYPE* callBacks)
{
  OMX_ERRORTYPE  eRet = OMX_ErrorNone;
  int cmp_index = -1;
  int hnd_index = -1;
  int vpp_cmp_index = -1;

  pthread_mutex_lock(&lock_core);
  if(handle)
  {
    *handle = NULL;
    char optComponentName[OMX_MAX_STRINGNAME_SIZE];
    strlcpy(optComponentName, componentName, OMX_MAX_STRINGNAME_SIZE);// 把componentName赋值给optComponentName

    if(strstr(componentName, "avc") && strstr(componentName, "decoder"))
    {　// 如果是avc decoder，需要判断是否有dsmode
      void *libhandle = dlopen("libOmxVideoDSMode.so", RTLD_NOW);
      if(libhandle)
      {
        int (*fn_ptr)()  = dlsym(libhandle, "isDSModeActive");

        if(fn_ptr == NULL)
        {
          DEBUG_PRINT_ERROR("Error: isDSModeActive Not Found %s\n",
                    dlerror());
        }
        else
        {
          int isActive = fn_ptr();
          char *pSubString = strstr(componentName, ".dsmode");
          if(pSubString)
          {
            optComponentName[pSubString - componentName] = 0;
          }
          else if(isActive)
          {
            strlcat(optComponentName, ".dsmode", OMX_MAX_STRINGNAME_SIZE);
          }
          cmp_index = get_cmp_index(optComponentName);
        }
        dlclose(libhandle);
      }
      else
      {
        DEBUG_PRINT_ERROR("Failed to load dsmode library");
      }
    }

    if(cmp_index < 0) 
    { // 先通过componentName找到cmp_index，在把componentName赋值给optComponentName。
      cmp_index = get_cmp_index(componentName);
      strlcpy(optComponentName, componentName, OMX_MAX_STRINGNAME_SIZE);
    }
    if(cmp_index >= 0)
    {
      DEBUG_PRINT("getting fn pointer\n");

      // Load VPP omx component for decoder if vpp property is enabled
      const char *hwDecLib = "libOmxVdec.so";　　// 硬件解码
      const char *swDecLib = "libOmxSwVdec.so";　// 软件解码
      if (!strncmp(core[cmp_index].so_lib_name, hwDecLib, strlen(hwDecLib)) ||
          !strncmp(core[cmp_index].so_lib_name, swDecLib, strlen(swDecLib))) {　// 找到了当前component对应的lib库
        bool isVppEnabled = false;
        if (isConfigStoreEnabled()) {
          getConfigStoreBool("vpp", "enable", &isVppEnabled, false);
        } else {
          char value[PROPERTY_VALUE_MAX];
          if ((property_get("vendor.media.vpp.enable", value, NULL))
               && (!strcmp("1", value) || !strcmp("true", value))) {
            isVppEnabled = true;
          }
        }
        if (isVppEnabled) {　// 如果enable了vpp
          DEBUG_PRINT("VPP property is enabled");
          vpp_cmp_index = get_cmp_index("OMX.qti.vdec.vpp");
          if (vpp_cmp_index < 0) {
            DEBUG_PRINT_ERROR("Unable to find VPP OMX lib in registry ");
          } else {
            DEBUG_PRINT("Loading vpp for vdec");
            cmp_index = vpp_cmp_index;
          }
        }
      }

       // 动态加载so库
      core[cmp_index].fn_ptr =
        omx_core_load_cmp_library(core[cmp_index].so_lib_name,
                                  &core[cmp_index].so_lib_handle);


      if(core[cmp_index].fn_ptr)
      {
        
        void* pThis = (*(core[cmp_index].fn_ptr))();
        if(pThis)
        {
          void *hComp = NULL;
          hComp = qc_omx_create_component_wrapper((OMX_PTR)pThis);// 创建hComp，hComp实际指向pThis->m_cmp，可用hComp调用OMX_COMPONENTTYPE里面的各种函数。
          if((eRet = qc_omx_component_init(hComp, optComponentName)) !=
                           OMX_ErrorNone)　// 初始化
          {
              DEBUG_PRINT("Component not created succesfully\n");
              pthread_mutex_unlock(&lock_core);
              return eRet;

          }
          qc_omx_component_set_callbacks(hComp,callBacks,appData); // 设置callBacks

          if (vpp_cmp_index >= 0)
          {
            hnd_index = get_comp_handle_index("OMX.qti.vdec.vpp");　// get_comp_handle_index
          }
          else
          {
            hnd_index = get_comp_handle_index(optComponentName);
          }

          if(hnd_index >= 0)
          {
            core[cmp_index].inst[hnd_index]= *handle = (OMX_HANDLETYPE) hComp;
          }
          else
          {
            DEBUG_PRINT("OMX_GetHandle:NO free slot available to store Component Handle\n");
            pthread_mutex_unlock(&lock_core);
            return OMX_ErrorInsufficientResources;
          }
          DEBUG_PRINT("Component %p Successfully created\n",*handle);
        }
        else
        {
          eRet = OMX_ErrorInsufficientResources;
          DEBUG_PRINT("Component Creation failed\n");
        }
      }
      else
      {
        eRet = OMX_ErrorNotImplemented;
        DEBUG_PRINT("library couldnt return create instance fn\n");
      }

    }
    else
    {
      eRet = OMX_ErrorNotImplemented;
      DEBUG_PRINT("ERROR: Already another instance active  ;rejecting \n");
    }
  }
  else
  {
    eRet =  OMX_ErrorBadParameter;
    DEBUG_PRINT("\n OMX_GetHandle: NULL handle \n");
  }
  pthread_mutex_unlock(&lock_core);
  return eRet;
}

上面介绍了component的注册过程，下面仔细看看每个函数的实现

// hardware/qcom/media/mm-core/src/common/omx_core_cmp.cpp
void * qc_omx_create_component_wrapper(OMX_PTR obj_ptr)
{
    qc_omx_component *pThis        = (qc_omx_component *)obj_ptr;
    OMX_COMPONENTTYPE* component   = &(pThis->m_cmp);
    memset(&pThis->m_cmp,0,sizeof(OMX_COMPONENTTYPE));

    component->nSize               = sizeof(OMX_COMPONENTTYPE);
    component->nVersion.nVersion   = OMX_SPEC_VERSION;
    component->pApplicationPrivate = 0;
    component->pComponentPrivate   = obj_ptr;

    component->AllocateBuffer      = &qc_omx_component_allocate_buffer;
    component->FreeBuffer          = &qc_omx_component_free_buffer;
    component->GetParameter        = &qc_omx_component_get_parameter;
    component->SetParameter        = &qc_omx_component_set_parameter;
    component->SendCommand         = &qc_omx_component_send_command;
    component->FillThisBuffer      = &qc_omx_component_fill_this_buffer;
    component->EmptyThisBuffer     = &qc_omx_component_empty_this_buffer;
    component->GetState            = &qc_omx_component_get_state;
    component->GetComponentVersion = &qc_omx_component_get_version;
    component->GetConfig           = &qc_omx_component_get_config;
    component->SetConfig           = &qc_omx_component_set_config;
    component->GetExtensionIndex   = &qc_omx_component_get_extension_index;
    component->ComponentTunnelRequest = &qc_omx_component_tunnel_request;
    component->UseBuffer           = &qc_omx_component_use_buffer;
    component->SetCallbacks        = &qc_omx_component_set_callbacks;
    component->UseEGLImage         = &qc_omx_component_use_EGL_image;
    component->ComponentRoleEnum   = &qc_omx_component_role_enum;
    component->ComponentDeInit     = &qc_omx_component_deinit;
    return (void *)component;
}

OMX_ERRORTYPE
qc_omx_component_init(OMX_IN OMX_HANDLETYPE hComp, OMX_IN OMX_STRING componentName)
{
  OMX_ERRORTYPE eRet = OMX_ErrorBadParameter;
  qc_omx_component *pThis = (hComp)? (qc_omx_component *)(((OMX_COMPONENTTYPE *)hComp)->pComponentPrivate):NULL;
  DEBUG_PRINT("OMXCORE: qc_omx_component_init %p\n", hComp);

  if(pThis)
  {
    // call the init fuction
    eRet = pThis->component_init(componentName); // 此处就会调用omx_vdec_v4l2.cpp里面的omx_vdec::component_init函数。后面的函数都一样，最终都会调用omx_vdec_v4l2.cpp里面对应的函数。

    if(eRet != OMX_ErrorNone)
    {
      //  in case of error, please destruct the component created
       delete pThis;
    }
  }
  return eRet;
}