多媒体框架中的音频offload

不管是不是offload模式（音频由dsp解码，不走omx框架），最终都会有AudioSink。有两类player，MediaPlayerInterface和MediaPlayerHWInterface。
andriod里面绝大数情况（phone，tv可能不是这样）用的就是MediaPlayerInterface，最终都会有AudioSink。
通过AudioSink和音频框架的交互来实现音频输出。


1.play的类型，决定p->hardwareOutput的放回值。MediaPlayerInterface返回为false，MediaPlayerHWInterface返回为true。
sp<MediaPlayerBase> MediaPlayerService::Client::setDataSource_pre(
        player_type playerType)
{
    ALOGV("player type = %d", playerType);


    // create the right type of player
    sp<MediaPlayerBase> p = createPlayer(playerType);
    if (p == NULL) {
        return p;
    }


    if (!p->hardwareOutput()) {
        Mutex::Autolock l(mLock);
        mAudioOutput = new AudioOutput(mAudioSessionId, IPCThreadState::self()->getCallingUid(),
                mPid, mAudioAttributes);
        static_cast<MediaPlayerInterface*>(p.get())->setAudioSink(mAudioOutput);
    }


    return p;
}


2.手机用的player
// Implement this class for media players that use the AudioFlinger software mixer
class MediaPlayerInterface : public MediaPlayerBase
{
public:
    virtual             ~MediaPlayerInterface() { }
    virtual bool        hardwareOutput() { return false; }
    virtual void        setAudioSink(const sp<AudioSink>& audioSink) { mAudioSink = audioSink; }
protected:
    sp<AudioSink>       mAudioSink;
};


// Implement this class for media players that output audio directly to hardware
class MediaPlayerHWInterface : public MediaPlayerBase
{
public:
    virtual             ~MediaPlayerHWInterface() {}
    virtual bool        hardwareOutput() { return true; }
    virtual status_t    setVolume(float leftVolume, float rightVolume) = 0;
    virtual status_t    setAudioStreamType(audio_stream_type_t streamType) = 0;
};


3.utils.cpp， 得到音频的info，和AduioPoliyService交互判断是否offload。
bool canOffloadStream(const sp<MetaData>& meta, bool hasVideo,
                      bool isStreaming, audio_stream_type_t streamType)
{
    const char *mime;
    if (meta == NULL) {
        return false;
    }
    CHECK(meta->findCString(kKeyMIMEType, &mime));


    audio_offload_info_t info = AUDIO_INFO_INITIALIZER;


    info.format = AUDIO_FORMAT_INVALID;
    if (mapMimeToAudioFormat(info.format, mime) != OK) {
        ALOGE(" Couldn't map mime type \"%s\" to a valid AudioSystem::audio_format !", mime);
        return false;
    } else {
        ALOGV("Mime type \"%s\" mapped to audio_format %d", mime, info.format);
    }
    info.format  = AVUtils::get()->updateAudioFormat(info.format, meta);
    if (AUDIO_FORMAT_INVALID == info.format) {
        // can't offload if we don't know what the source format is
        ALOGE("mime type \"%s\" not a known audio format", mime);
        return false;
    }


    if (AVUtils::get()->canOffloadAPE(meta) != true) {
        return false;
    }


    // Redefine aac format according to its profile
    // Offloading depends on audio DSP capabilities.
    int32_t aacaot = -1;
    if (meta->findInt32(kKeyAACAOT, &aacaot)) {
        bool isADTSSupported = false;
        isADTSSupported = AVUtils::get()->mapAACProfileToAudioFormat(meta, info.format,
                                  (OMX_AUDIO_AACPROFILETYPE) aacaot);
        if (!isADTSSupported) {
           mapAACProfileToAudioFormat(info.format,(OMX_AUDIO_AACPROFILETYPE) aacaot);
        }
    }


    int32_t srate = -1;
    if (!meta->findInt32(kKeySampleRate, &srate)) {
        ALOGV("track of type '%s' does not publish sample rate", mime);
    }
    info.sample_rate = srate;


    int32_t cmask = 0;
    if (!meta->findInt32(kKeyChannelMask, &cmask)) {
        ALOGV("track of type '%s' does not publish channel mask", mime);


        // Try a channel count instead
        int32_t channelCount;
        if (!meta->findInt32(kKeyChannelCount, &channelCount)) {
            ALOGV("track of type '%s' does not publish channel count", mime);
        } else {
            cmask = audio_channel_out_mask_from_count(channelCount);
        }
    }
    info.channel_mask = cmask;


    int64_t duration = 0;
    if (!meta->findInt64(kKeyDuration, &duration)) {
        ALOGV("track of type '%s' does not publish duration", mime);
    }
    info.duration_us = duration;


    int32_t brate = -1;
    if (!meta->findInt32(kKeyBitRate, &brate)) {
        ALOGV("track of type '%s' does not publish bitrate", mime);
     }
    info.bit_rate = brate;




    info.stream_type = streamType;
    info.has_video = hasVideo;
    info.is_streaming = isStreaming;


    // Check if offload is possible for given format, stream type, sample rate,
    // bit rate, duration, video and streaming
    return AudioSystem::isOffloadSupported(info);
}


3.audio在offload时创建DecoderPassThrough，与Decoder一样都是继承于DecoderBase。
status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) {
    // The audio decoder could be cleared by tear down. If still in shut down
    // process, no need to create a new audio decoder.
    if (*decoder != NULL || (audio && mFlushingAudio == SHUT_DOWN)) {
        return OK;
    }


    sp<AMessage> format = mSource->getFormat(audio);


    if (format == NULL) {
        return -EWOULDBLOCK;
    }


    format->setInt32("priority", 0 /* realtime */);


    if (!audio) {
        AString mime;
        CHECK(format->findString("mime", &mime));


        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, this);
        if (mCCDecoder == NULL) {
            mCCDecoder = new CCDecoder(ccNotify);
        }


        if (mSourceFlags & Source::FLAG_SECURE) {
            format->setInt32("secure", true);
        }


        if (mSourceFlags & Source::FLAG_PROTECTED) {
            format->setInt32("protected", true);
        }


        float rate = getFrameRate();
        if (rate > 0) {
            format->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);
        }
    }


    if (audio) {
        sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
        ++mAudioDecoderGeneration;
        notify->setInt32("generation", mAudioDecoderGeneration);


        determineAudioModeChange();


        if (AVNuUtils::get()->isRAWFormat(format)) {
            AVNuUtils::get()->setPCMFormat(format,
                    AVNuUtils::get()->getKeyPCMFormat(mSource->getFormatMeta(true /* audio */)));
        }
        if (mOffloadAudio && !ifDecodedPCMOffload()) {
            const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL);
            format->setInt32("has-video", hasVideo);
            *decoder = AVNuFactory::get()->createPassThruDecoder(notify, mSource, mRenderer);//audio在offload时创建PassThruDecoder
        } else {
            *decoder = AVNuFactory::get()->createDecoder(notify, mSource, mPID, mRenderer);
        }
    } else {
        sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
        ++mVideoDecoderGeneration;
        notify->setInt32("generation", mVideoDecoderGeneration);


        *decoder = new Decoder(
                notify, mSource, mPID, mRenderer, mSurface, mCCDecoder);


        // enable FRC if high-quality AV sync is requested, even if not
        // directly queuing to display, as this will even improve textureview
        // playback.
        {
            char value[PROPERTY_VALUE_MAX];
            if (property_get("persist.sys.media.avsync", value, NULL) &&
                    (!strcmp("1", value) || !strcasecmp("true", value))) {
                format->setInt32("auto-frc", 1);
            }
        }
    }
    (*decoder)->init();
    (*decoder)->configure(format);


    // allocate buffers to decrypt widevine source buffers
    if (!audio && (mSourceFlags & Source::FLAG_SECURE)) {
        Vector<sp<ABuffer> > inputBufs;
        CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK);


        Vector<MediaBuffer *> mediaBufs;
        for (size_t i = 0; i < inputBufs.size(); i++) {
            const sp<ABuffer> &buffer = inputBufs[i];
            MediaBuffer *mbuf = new MediaBuffer(buffer->data(), buffer->size());
            mediaBufs.push(mbuf);
        }


        status_t err = mSource->setBuffers(audio, mediaBufs);
        if (err != OK) {
            for (size_t i = 0; i < mediaBufs.size(); ++i) {
                mediaBufs[i]->release();
            }
            mediaBufs.clear();
            ALOGE("Secure source didn't support secure mediaBufs.");
            return err;
        }
    }
    return OK;
}


4.DecoderPassThrough到Renderer，并在在Renderer中openAudioSink
void NuPlayer::DecoderPassThrough::onConfigure(const sp<AMessage> &format) {
    ALOGV("[%s] onConfigure", mComponentName.c_str());
    mCachedBytes = 0;
    mPendingBuffersToDrain = 0;
    mReachedEOS = false;
    ++mBufferGeneration;


    onRequestInputBuffers();


    int32_t hasVideo = 0;
    format->findInt32("has-video", &hasVideo);


    // The audio sink is already opened before the PassThrough decoder is created.
    // Opening again might be relevant if decoder is instantiated after shutdown and
    // format is different.
    status_t err = mRenderer->openAudioSink( //在Renderer中openAudioSink
            format, true /* offloadOnly */, hasVideo,
            AUDIO_OUTPUT_FLAG_NONE /* flags */, NULL /* isOffloaded */, mSource->isStreaming());
    if (err != OK) {
        handleError(err);
    }
}


5.openAudioSink
status_t NuPlayer::Renderer::openAudioSink(
        const sp<AMessage> &format,
        bool offloadOnly,
        bool hasVideo,
        uint32_t flags,
        bool *isOffloaded,
        bool isStreaming) {
    sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
    msg->setMessage("format", format);
    msg->setInt32("offload-only", offloadOnly);
    msg->setInt32("has-video", hasVideo);
    msg->setInt32("flags", flags);
    msg->setInt32("isStreaming", isStreaming);


    sp<AMessage> response;
    status_t postStatus = msg->postAndAwaitResponse(&response);//postAndAwaitResponse


    int32_t err;
    if (postStatus != OK || !response->findInt32("err", &err)) {
        err = INVALID_OPERATION;
    } else if (err == OK && isOffloaded != NULL) {
        int32_t offload;
        CHECK(response->findInt32("offload", &offload));
        *isOffloaded = (offload != 0);
    }
    return err;
}


6.
        case kWhatOpenAudioSink:
        {
            sp<AMessage> format;
            CHECK(msg->findMessage("format", &format));


            int32_t offloadOnly;
            CHECK(msg->findInt32("offload-only", &offloadOnly));


            int32_t hasVideo;
            CHECK(msg->findInt32("has-video", &hasVideo));


            uint32_t flags;
            CHECK(msg->findInt32("flags", (int32_t *)&flags));


            uint32_t isStreaming;
            CHECK(msg->findInt32("isStreaming", (int32_t *)&isStreaming));


            status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming);//


            sp<AMessage> response = new AMessage;
            response->setInt32("err", err);
            response->setInt32("offload", offloadingAudio());


            sp<AReplyToken> replyID;
            CHECK(msg->senderAwaitsResponse(&replyID));
            response->postReply(replyID);


            break;
        }