作为一个audio工程师,需要了解一些Nuplayer框架和audio framework之间的联系!
先从熟悉的着手,先分析下Nuplayer对AudioTrack的使用
Nuplayer是在如何使用AudioTrack的?
先看一个类的继承关系
//MediaPlayerService.h
class MediaPlayerService : public BnMediaPlayerService
{
...
class AudioOutput : public MediaPlayerBase::AudioSink
...
sp<AudioTrack> mTrack;
...
}
可见,AudioTrack被封装到了MediaPlayerService的子类AudioOutput中。
而AudioOutput又是这样被使用的:
//sp<MediaPlayerBase> MediaPlayerService::Client::setDataSource_pre(
//player_type playerType)
mAudioOutput = new AudioOutput(mAudioSessionId, IPCThreadState::self()->getCallingUid(),mPid, mAudioAttributes);
static_cast<MediaPlayerInterface*>(p.get())->setAudioSink(mAudioOutput);
return p;
//这里创建的mAudioOutput通过setAudioSink设置给了
//NuPlayer的mAudioSink
这个Client对应着使用mediaplayer接口和mediaplayerservice交互的进程。每一个client都有自己独立的一个mAudioOutput来使用。MediaPlayerService对AudioTrack做了一层封装。其实最终还是客户端app进程在使用。
创建audiotrack对象的流程如下:
//MediaCodec发来类似于如下消息
//NuPlayerDecoder: [audio] kWhatCodecNotify: cbID = 4, paused = 0
//4对应着MediaCodec::CB_OUTPUT_FORMAT_CHANGED
NuPlayer::Decoder::onMessageReceived()
NuPlayer::Decoder::handleOutputFormatChange()
NuPlayer::Renderer::changeAudioFormat()
//发出消息kWhatChangeAudioFormat(发给自己)
//收到消息
NuPlayer::Renderer::onMessageReceived(kWhatChangeAudioFormat)
NuPlayer::Renderer::onChangeAudioFormat
ExtendedNuPlayerRenderer::onOpenAudioSink
NuPlayer::Renderer::onOpenAudioSink
NuPlayer::Renderer::onOpenAudioSink()
MediaPlayerService::AudioOutput::open()
这样就创建了AudioTrack.
接下来看看,AudioTrack对象(mAudioSink)都是怎么应用的
音频数据流的代码流程
NuPlayer::Decoder收到消息
NuPlayerDecoder: [audio] kWhatCodecNotify: cbID = 1, paused = 0
NuPlayerDecoder: [OMX.google.aac.decoder] onMessage: AMessage(what = 'cdcN', target = 8) = {
NuPlayerDecoder: int32_t callbackID = 2
NuPlayerDecoder: int32_t index = 0
NuPlayerDecoder: size_t offset = 0
NuPlayerDecoder: size_t size = 4096
NuPlayerDecoder: int64_t timeUs = 820464400
int32_t flags = 0
NuPlayerDecoder: }
callbackID是2,对应MediaCodec::CB_OUTPUT_AVAILABLE
//触发调用这个方法
NuPlayer::Decoder::handleAnOutputBuffer(index, offset, size, timeUs, flags);
/***方法大致流程如下***/
sp<MediaCodecBuffer> buffer;
//根据index取出对应buffer
//这里mCodec是MediaCodec的一个实例
//由此可见,这里NuPlayer::Decoder和MediaCode是
//成对的
mCodec->getOutputBuffer(index, &buffer);
if (index >= mOutputBuffers.size()) {
for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
//扩容
mOutputBuffers.add();
}
}
//赋值
mOutputBuffers.editItemAt(index) = buffer;
//设定数据的范围,偏移和大小
buffer->setRange(offset, size);
//设置meta
buffer->meta()->clear();
buffer->meta()->setInt64("timeUs", timeUs);
...
//消息发给自己,NuPlayer::Decoder::onMessageReceived处理消息
//最终转到onRenderBuffer
sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
reply->post();
//queueBuffer或者queueEOS
mRenderer->queueBuffer(mIsAudio, buffer, reply);
if (eos && !isDiscontinuityPending()) {
mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
}
先说kWhatRenderBuffer消息的处理:
void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
status_t err;
int32_t render;
size_t bufferIx;
int32_t eos;
//buffer index
CHECK(msg->findSize("buffer-ix", &bufferIx));
//视频
if (!mIsAudio) {
int64_t timeUs;
sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
buffer->meta()->findInt64("timeUs", &timeUs);
//mSelectedTrack合法的话就调用display去显示?
if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
mCCDecoder->display(timeUs);
}
}
...
if (mCodec == NULL) {
err = NO_INIT;
} else if (msg->findInt32("render", &render) && render) {
/* 消息示例
[OMX.qcom.video.decoder.avc] onMessage: AMessage(what = 'rndr', target = 31) = {
size_t buffer-ix = 9
int32_t generation = 1
int64_t timestampNs = 85634970014000
int32_t render = 1
}
*/
//纳秒级别的精度
//触发调用BufferChannel的renderOutputBuffer方法
int64_t timestampNs;
err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
} else {
//直接调用BufferChannel的discardBuffer方法
mNumOutputFramesDropped += !mIsAudio;
err = mCodec->releaseOutputBuffer(bufferIx);
}
...
//如果eos,通知NuPlayer关掉decoder或者rescan sources
if (msg->findInt32("eos", &eos) && eos
&& isDiscontinuityPending()) {
finishHandleDiscontinuity(true /* flushOnTimeChange */);
}
}
再接着看
queueBuffe
void NuPlayer::Renderer::queueBuffer(
bool audio,
const sp<MediaCodecBuffer> &buffer,
const sp<AMessage> ¬ifyConsumed) {
int64_t mediaTimeUs = -1;
buffer->meta()->findInt64("timeUs", &mediaTimeUs);
//注意这里的第二个参数是this指针,意思是消息的处理由
//NuPlayer::Renderer这个类构造的当前对象来完成
//可以在NuPlayerRenderer.h中发现
//struct NuPlayer::Renderer : public AHandler
//该类也实现了这个处理的方法
//void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg)
sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
msg->setInt32("queueGeneration", getQueueGeneration(audio));
msg->setInt32("audio", static_cast<int32_t>(audio));
msg->setObject("buffer", buffer);
msg->setMessage("notifyConsumed", notifyConsumed);
msg->post();
}
发了kWhatQueueBuffer消息之后,转到这个方法:
void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
int32_t audio;
CHECK(msg->findInt32("audio", &audio));
...
if (audio) {
mHasAudio = true;
} else {
mHasVideo = true;
}
//如果buffer里有视频
if (mHasVideo) {
if (mVideoScheduler == NULL) {
mVideoScheduler = new VideoFrameScheduler();
mVideoScheduler->init();
}
}
...
QueueEntry entry;
entry.mBuffer = buffer;//真正的音频/视频数据
//对应queueBuffer传入的reply,用来做notify的
entry.mNotifyConsumed = notifyConsumed;
entry.mOffset = 0;
entry.mFinalResult = OK;//正常情况下设置成OK
entry.mBufferOrdinal = ++mTotalBuffersQueued;
//将entry加入我们前面提到过的mAudioQueue/mVideoQueue
//然后调用postDrainAudio(Video)Queue
if (audio) {
Mutex::Autolock autoLock(mLock);
mAudioQueue.push_back(entry);
postDrainAudioQueue_l();
} else {
mVideoQueue.push_back(entry);
postDrainVideoQueue();
}
}
postDrainAudioQueue_l,对应着kWhatDrainAudioQueue
case kWhatDrainAudioQueue:
{
mDrainAudioQueuePending = false;
//如果这里进来的drainGeneration
//不是audio的,直接跳出
int32_t generation;
CHECK(msg->findInt32("drainGeneration", &generation));
if (generation != getDrainGeneration(true /* audio */)) {
break;
}
//这里要进入一个比较复杂的流程了
//做的事情主要是drain audio queue
//排泄audio队列
//***打几个星号强调一下***
//这个函数的主要返回值表达的意思是是否
//need reschedule(重新调度)
if (onDrainAudioQueue()) {
//如果需要重新调度,进入下面的逻辑
uint32_t numFramesPlayed;
//调用AudioSink的getPosition方法
//AudioSink是一个audio output的
//抽象层
//这里的audio output指的是
//libmediaplayerservice/MediaPlayerService.h
//中定义的AudioOutput类
//那这里的调用实际上是调用了
//MediaPlayerService::AudioOutput::getPosition
//当然最终调用的就是AudioTrack的getPosition
if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
...}
//已经写下去的数据量减去,已经播放掉的数据量
//就是还在队列中pending的数据量
uint32_t numFramesPendingPlayout =
mNumFramesWritten - numFramesPlayed;
// This is how long the audio sink will have data to