Alexa SDK庖丁解牛-第十回：SpeechSynthesizer模块分析

1.audio speak播放流程梳理：

bool MediaPlayer::init() {

if (false == gst_init_check(NULL, NULL, NULL)) {

ACSDK_ERROR(LX("initPlayerFailed").d("reason", "gstInitCheckFailed"));

return false;

}

 

if (!(m_mainLoop = g_main_loop_new(nullptr, false))) {

ACSDK_ERROR(LX("initPlayerFailed").d("reason", "gstMainLoopNewFailed"));

return false;

};

 

m_mainLoopThread = std::thread(g_main_loop_run, m_mainLoop);

 

if (!setupPipeline()) {

ACSDK_ERROR(LX("initPlayerFailed").d("reason", "setupPipelineFailed"));

return false;

}

 

return true;

}

 

bool MediaPlayer::setupPipeline() {

m_busWatchId = gst_bus_add_watch(bus, &MediaPlayer::onBusMessage, this);

}

 

gboolean MediaPlayer::onBusMessage(GstBus* bus, GstMessage* message, gpointer mediaPlayer) {

return static_cast<MediaPlayer*>(mediaPlayer)->handleBusMessage(message);

}

gboolean MediaPlayer::handleBusMessage(GstMessage* message) {

}

 

2.SpeechSynthesizer interface与speakMediaPlayer的关系

m_speechSynthesizer = capabilityAgents::speechSynthesizer::SpeechSynthesizer::create(

speakMediaPlayer,

m_connectionManager,

m_focusManager,

contextManager,

attachmentManager,

exceptionSender,

m_dialogUXStateAggregator);

在识别唤醒词后，client与server的交互过程会执行到onFocusChanged，具体可以参考导读一中的描述，首先调用void SpeechSynthesizer::startPlaying()设置读操作handle，以及数据更新回调函数，同时在handlePlay中会更新状态，触发handleBusMessage的执行；

void SpeechSynthesizer::onFocusChanged(FocusState newFocus)

{

executeStateChange();

}

void SpeechSynthesizer::executeStateChange() {

startPlaying();

或者

stopPlaying();

}

关键的地方就在这里，m_speechPlayer的初始化原型为MediaPlayer，我们需要去MediaPlayer的类定义去看下具体的描述和实现，MediaPlayer::handleBusMessage()函数就在这里被触发：

void SpeechSynthesizer::startPlaying() {

ACSDK_DEBUG9(LX("startPlaying"));

m_mediaSourceId = m_speechPlayer->setSource(std::move(m_currentInfo->attachmentReader));

if (MediaPlayerInterface::ERROR == m_mediaSourceId) {

ACSDK_ERROR(LX("startPlayingFailed").d("reason", "setSourceFailed"));

executePlaybackError(ErrorType::MEDIA_ERROR_INTERNAL_DEVICE_ERROR, "playFailed");

} else if (!m_speechPlayer->play(m_mediaSourceId)) {

executePlaybackError(ErrorType::MEDIA_ERROR_INTERNAL_DEVICE_ERROR, "playFailed");

} else {

// Execution of play is successful.

m_isAlreadyStopping = false;

}

}

这里我们需要特别关注两个执行单元：

1. m_mediaSourceId = m_speechPlayer->setSource(std::move(m_currentInfo->attachmentReader));

2.m_speechPlayer->play(m_mediaSourceId)

step1：

对于1我们来分析一下，MediaPlayer中关于setSource的原型有三类，根据此处的参数类型，应该是：

MediaPlayer::SourceId MediaPlayer::setSource(std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> reader) {

ACSDK_DEBUG9(LX("setSourceCalled").d("sourceType", "AttachmentReader"));

std::promise<MediaPlayer::SourceId> promise;

auto future = promise.get_future();

/* 回调函数lambda表达式*/

std::function<gboolean()> callback = [this, &reader, &promise]() {

handleSetAttachmentReaderSource(std::move(reader), &promise);

return false;

};

if (queueCallback(&callback) != UNQUEUED_CALLBACK) {

return future.get();

}

return ERROR_SOURCE_ID;

}

回调函数部分主要工作：

0）tearDownTransientPipelineElements();初始化、重置工作

1）std::shared_ptr<SourceInterface> source = AttachmentReaderSource::create(this, reader);

m_source = source;

2）g_signal_connect(m_pipeline.decoder, "pad-added", G_CALLBACK(onPadAdded), this)；

3）onPadAdded();=》void MediaPlayer::handlePadAdded(）:

gst_element_link(decoder, m_pipeline.decodedQueue);

其中source的创建，里面有很多东西需要我们来了解一下，

std::unique_ptr<AttachmentReaderSource> AttachmentReaderSource::create(

PipelineInterface* pipeline,

std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> attachmentReader) {

std::unique_ptr<AttachmentReaderSource> result(new AttachmentReaderSource(pipeline, attachmentReader));

if (result->init()) {

return result;

}

return nullptr;

};

AttachmentReaderSource的父类是BaseStreamSource，调用父类的方法init();

AttachmentReaderSource::AttachmentReaderSource(

PipelineInterface* pipeline,

std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> reader) :

BaseStreamSource{pipeline, "AttachmentReaderSource"},

m_reader{reader} {};

 

bool BaseStreamSource::init() {

//need data

//encough data

}

分析到这里，我们心中应该记住一条：mediaplayer-》AttachmentReader-》BaseStreamSource，抓住这条线理解问题就简单了。

BaseStreamSource::handleNeedData()

void BaseStreamSource::installOnReadDataHandler()

{

m_sourceId = g_idle_add(reinterpret_cast<GSourceFunc>(&onReadData), this);

}

gboolean BaseStreamSource::onReadData(gpointer pointer) {

return static_cast<BaseStreamSource*>(pointer)->handleReadData();

}

gboolean AttachmentReaderSource::handleReadData() {

}

=>

void BaseStreamSource::signalEndOfData() {

clearOnReadDataHandler();

}

step2：bool MediaPlayer::play(MediaPlayer::SourceId id) {

ACSDK_DEBUG9(LX("playCalled"));

if (!m_source) {

ACSDK_ERROR(LX("playFailed").d("reason", "sourceNotSet"));

return ERROR;

}

/* 设置URL地址给g_object_set(m_pipeline->getDecoder(), "uri", m_url.c_str(), "use-buffering", true, NULL);目前看调用的是空函数*/

m_source->preprocess();

 

std::promise<bool> promise;

auto future = promise.get_future();

/* 回调函数lambda表达式*/

std::function<gboolean()> callback = [this, id, &promise]() {

handlePlay(id, &promise);

return false;

};

 

if (queueCallback(&callback) != UNQUEUED_CALLBACK) {

return future.get();

}

return false;

}

handlePlay回调函数部分主要工作是设置状态更新，GST_STATE_PLAYING设置后触发播放MediaPlayer注册的onBusMessage：

GstState startingState = GST_STATE_PLAYING;

stateChange = gst_element_set_state(m_pipeline.pipeline, startingState);

 

3. MediaPlayer注册的onBusMessage，在监听m_pipeline.pipeline的状态更新：

GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(m_pipeline.pipeline));

m_busWatchId = gst_bus_add_watch(bus, &MediaPlayer::onBusMessage, this);

疑问：mediaplayer设置完URL后，如何管理音频数据，数据是否还存放在SDS模块，解码播放通过SDS读取；

触发MediaPlayer注册的onBusMessage后的重要工作是：sendPlaybackStarted()：

void MediaPlayer::sendPlaybackStarted() {

m_playerObserver->onPlaybackStarted(m_currentId);

}

 

void SpeechSynthesizer::onPlaybackStarted(SourceId id) {

executePlaybackStarted()；

}

 

void SpeechSynthesizer::executePlaybackStarted() {

1) setCurrentStateLocked();

2) m_waitOnStateChange.notify_one();

3) m_messageSender->sendMessage(request);

}

1) 状态更新

void SpeechSynthesizer::setCurrentStateLocked()

{

executeProvideState(m_currentState, 0);

.................

observer->onStateChanged(m_currentState);

}

executeProvideState(m_currentState, 0);

=>

std::chrono::milliseconds MediaPlayer::getOffset(MediaPlayer::SourceId id) {

}

2） 解阻：void SpeechSynthesizer::onFocusChanged(FocusState newFocus)，具体可以参考导读一中的介绍。

 

4. audioplayer的作用：

1). void AudioPlayer::playNextItem()

void AudioPlayer::executeOnFocusChanged(FocusState newFocus)

{

playNextItem();

}

void AudioPlayer::playNextItem()

{

m_sourceId = m_mediaPlayer->setSource(std::move(item.stream.reader));

或者

m_sourceId = m_mediaPlayer->setSource(item.stream.url, item.stream.offset);

m_mediaPlayer->play(m_sourceId);

}

2). void Renderer::executeStart()

void Renderer::start()

{

m_executor.submit(

[this, audioFactory, urls, loopCount, loopPause]() { executeStart(audioFactory, urls, loopCount, loopPause); });

}

5.onBusMessage注册了数据处理的回调函数，对应的回调函数实现在BaseStreamSource类的init函数中，下面我们一起来看看：

bool BaseStreamSource::init() {

//need data

//encough data

}

1）onNeedData函数负责读取数据，具体工作由callback函数handleNeedData完成

void BaseStreamSource::onNeedData(GstElement* pipeline, guint size, gpointer pointer) {

source->m_needDataCallbackId = source->m_pipeline->queueCallback(&source->m_handleNeedDataFunction);

}

=》

gboolean BaseStreamSource::handleNeedData() {

ACSDK_DEBUG9(LX("handleNeedDataCalled"));

std::lock_guard<std::mutex> lock(m_callbackIdMutex);

m_needDataCallbackId = 0;

installOnReadDataHandler();

return false;

}

void BaseStreamSource::installOnReadDataHandler() {

/* 初始化类成员变量，这里会关键的，初始化状态在后面会作为出错处理

重试机制的判断参数 */

m_sourceRetryCount = 0;

m_sourceId = g_idle_add(reinterpret_cast<GSourceFunc>(&onReadData), this);

}

gboolean BaseStreamSource::onReadData(gpointer pointer) {

return static_cast<BaseStreamSource*>(pointer)->handleReadData();

}

//重点来了，数据如何读取，读取后如何给gst都在这里找到了答案

gboolean AttachmentReaderSource::handleReadData() {

auto buffer = gst_buffer_new_allocate(nullptr, CHUNK_SIZE, nullptr);

gst_buffer_map(buffer, &info, GST_MAP_WRITE);

auto size = m_reader->read(info.data, info.size, &status);

auto flowRet = gst_app_src_push_buffer(getAppSrc(), buffer);

}

2）

void BaseStreamSource::onEnoughData(GstElement* pipeline, gpointer pointer) {

source->m_enoughDataCallbackId = source->m_pipeline->queueCallback(&source->m_handleEnoughDataFunction);

}