1.audio speak播放流程梳理:
bool MediaPlayer::init() {
if (false == gst_init_check(NULL, NULL, NULL)) {
ACSDK_ERROR(LX("initPlayerFailed").d("reason", "gstInitCheckFailed"));
return false;
}
if (!(m_mainLoop = g_main_loop_new(nullptr, false))) {
ACSDK_ERROR(LX("initPlayerFailed").d("reason", "gstMainLoopNewFailed"));
return false;
};
m_mainLoopThread = std::thread(g_main_loop_run, m_mainLoop);
if (!setupPipeline()) {
ACSDK_ERROR(LX("initPlayerFailed").d("reason", "setupPipelineFailed"));
return false;
}
return true;
}
bool MediaPlayer::setupPipeline() {
m_busWatchId = gst_bus_add_watch(bus, &MediaPlayer::onBusMessage, this);
}
gboolean MediaPlayer::onBusMessage(GstBus* bus, GstMessage* message, gpointer mediaPlayer) {
return static_cast<MediaPlayer*>(mediaPlayer)->handleBusMessage(message);
}
gboolean MediaPlayer::handleBusMessage(GstMessage* message) {
}
2.SpeechSynthesizer interface与speakMediaPlayer的关系
m_speechSynthesizer = capabilityAgents::speechSynthesizer::SpeechSynthesizer::create(
speakMediaPlayer,
m_connectionManager,
m_focusManager,
contextManager,
attachmentManager,
exceptionSender,
m_dialogUXStateAggregator);
在识别唤醒词后,client与server的交互过程会执行到onFocusChanged,具体可以参考导读一中的描述,首先调用void SpeechSynthesizer::startPlaying()设置读操作handle,以及数据更新回调函数,同时在handlePlay中会更新状态,触发handleBusMessage的执行;
void SpeechSynthesizer::onFocusChanged(FocusState newFocus)
{
executeStateChange();
}
void SpeechSynthesizer::executeStateChange() {
startPlaying();
或者
stopPlaying();
}
关键的地方就在这里,m_speechPlayer的初始化原型为MediaPlayer,我们需要去MediaPlayer的类定义去看下具体的描述和实现,MediaPlayer::handleBusMessage()函数就在这里被触发:
void SpeechSynthesizer::startPlaying() {
ACSDK_DEBUG9(LX("startPlaying"));
m_mediaSourceId = m_speechPlayer->setSource(std::move(m_currentInfo->attachmentReader));
if (MediaPlayerInterface::ERROR == m_mediaSourceId) {
ACSDK_ERROR(LX("startPlayingFailed").d("reason", "setSourceFailed"));
executePlaybackError(ErrorType::MEDIA_ERROR_INTERNAL_DEVICE_ERROR, "playFailed");
} else if (!m_speechPlayer->play(m_mediaSourceId)) {
executePlaybackError(ErrorType::MEDIA_ERROR_INTERNAL_DEVICE_ERROR, "playFailed");
} else {
// Execution of play is successful.
m_isAlreadyStopping = false;
}
}
这里我们需要特别关注两个执行单元:
1. m_mediaSourceId = m_speechPlayer->setSource(std::move(m_currentInfo->attachmentReader));
2.m_speechPlayer->play(m_mediaSourceId)
step1:
对于1我们来分析一下,MediaPlayer中关于setSource的原型有三类,根据此处的参数类型,应该是:
MediaPlayer::SourceId MediaPlayer::setSource(std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> reader) {
ACSDK_DEBUG9(LX("setSourceCalled").d("sourceType", "AttachmentReader"));
std::promise<MediaPlayer::SourceId> promise;
auto future = promise.get_future();
/* 回调函数lambda表达式*/
std::function<gboolean()> callback = [this, &reader, &promise]() {
handleSetAttachmentReaderSource(std::move(reader), &promise);
return false;
};
if (queueCallback(&callback) != UNQUEUED_CALLBACK) {
return future.get();
}
return ERROR_SOURCE_ID;
}
回调函数部分主要工作:
0)tearDownTransientPipelineElements();初始化、重置工作
1)std::shared_ptr<SourceInterface> source = AttachmentReaderSource::create(this, reader);
m_source = source;
2)g_signal_connect(m_pipeline.decoder, "pad-added", G_CALLBACK(onPadAdded), this);
3)onPadAdded();=》void MediaPlayer::handlePadAdded():
gst_element_link(decoder, m_pipeline.decodedQueue);
其中source的创建,里面有很多东西需要我们来了解一下,
std::unique_ptr<AttachmentReaderSource> AttachmentReaderSource::create(
PipelineInterface* pipeline,
std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> attachmentReader) {
std::unique_ptr<AttachmentReaderSource> result(new AttachmentReaderSource(pipeline, attachmentReader));
if (result->init()) {
return result;
}
return nullptr;
};
AttachmentReaderSource的父类是BaseStreamSource,调用父类的方法init();
AttachmentReaderSource::AttachmentReaderSource(
PipelineInterface* pipeline,
std::shared_ptr<avsCommon::avs::attachment::AttachmentReader> reader) :
BaseStreamSource{pipeline, "AttachmentReaderSource"},
m_reader{reader} {};
bool BaseStreamSource::init() {
//need data
//encough data
}
分析到这里,我们心中应该记住一条:mediaplayer-》AttachmentReader-》BaseStreamSource,抓住这条线理解问题就简单了。
BaseStreamSource::handleNeedData()
void BaseStreamSource::installOnReadDataHandler()
{
m_sourceId = g_idle_add(reinterpret_cast<GSourceFunc>(&onReadData), this);
}
gboolean BaseStreamSource::onReadData(gpointer pointer) {
return static_cast<BaseStreamSource*>(pointer)->handleReadData();
}
gboolean AttachmentReaderSource::handleReadData() {
}
=>
void BaseStreamSource::signalEndOfData() {
clearOnReadDataHandler();
}
step2:bool MediaPlayer::play(MediaPlayer::SourceId id) {
ACSDK_DEBUG9(LX("playCalled"));
if (!m_source) {
ACSDK_ERROR(LX("playFailed").d("reason", "sourceNotSet"));
return ERROR;
}
/* 设置URL地址给g_object_set(m_pipeline->getDecoder(), "uri", m_url.c_str(), "use-buffering", true, NULL);目前看调用的是空函数*/
m_source->preprocess();
std::promise<bool> promise;
auto future = promise.get_future();
/* 回调函数lambda表达式*/
std::function<gboolean()> callback = [this, id, &promise]() {
handlePlay(id, &promise);
return false;
};
if (queueCallback(&callback) != UNQUEUED_CALLBACK) {
return future.get();
}
return false;
}
handlePlay回调函数部分主要工作是设置状态更新,GST_STATE_PLAYING设置后触发播放MediaPlayer注册的onBusMessage:
GstState startingState = GST_STATE_PLAYING;
stateChange = gst_element_set_state(m_pipeline.pipeline, startingState);
3. MediaPlayer注册的onBusMessage,在监听m_pipeline.pipeline的状态更新:
GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(m_pipeline.pipeline));
m_busWatchId = gst_bus_add_watch(bus, &MediaPlayer::onBusMessage, this);
疑问:mediaplayer设置完URL后,如何管理音频数据,数据是否还存放在SDS模块,解码播放通过SDS读取;
触发MediaPlayer注册的onBusMessage后的重要工作是:sendPlaybackStarted():
void MediaPlayer::sendPlaybackStarted() {
m_playerObserver->onPlaybackStarted(m_currentId);
}
void SpeechSynthesizer::onPlaybackStarted(SourceId id) {
executePlaybackStarted();
}
void SpeechSynthesizer::executePlaybackStarted() {
1) setCurrentStateLocked();
2) m_waitOnStateChange.notify_one();
3) m_messageSender->sendMessage(request);
}
1) 状态更新
void SpeechSynthesizer::setCurrentStateLocked()
{
executeProvideState(m_currentState, 0);
.................
observer->onStateChanged(m_currentState);
}
executeProvideState(m_currentState, 0);
=>
std::chrono::milliseconds MediaPlayer::getOffset(MediaPlayer::SourceId id) {
}
2) 解阻:void SpeechSynthesizer::onFocusChanged(FocusState newFocus),具体可以参考导读一中的介绍。
4. audioplayer的作用:
1). void AudioPlayer::playNextItem()
void AudioPlayer::executeOnFocusChanged(FocusState newFocus)
{
playNextItem();
}
void AudioPlayer::playNextItem()
{
m_sourceId = m_mediaPlayer->setSource(std::move(item.stream.reader));
或者
m_sourceId = m_mediaPlayer->setSource(item.stream.url, item.stream.offset);
m_mediaPlayer->play(m_sourceId);
}
2). void Renderer::executeStart()
void Renderer::start()
{
m_executor.submit(
[this, audioFactory, urls, loopCount, loopPause]() { executeStart(audioFactory, urls, loopCount, loopPause); });
}
5.onBusMessage注册了数据处理的回调函数,对应的回调函数实现在BaseStreamSource类的init函数中,下面我们一起来看看:
bool BaseStreamSource::init() {
//need data
//encough data
}
1)onNeedData函数负责读取数据,具体工作由callback函数handleNeedData完成
void BaseStreamSource::onNeedData(GstElement* pipeline, guint size, gpointer pointer) {
source->m_needDataCallbackId = source->m_pipeline->queueCallback(&source->m_handleNeedDataFunction);
}
=》
gboolean BaseStreamSource::handleNeedData() {
ACSDK_DEBUG9(LX("handleNeedDataCalled"));
std::lock_guard<std::mutex> lock(m_callbackIdMutex);
m_needDataCallbackId = 0;
installOnReadDataHandler();
return false;
}
void BaseStreamSource::installOnReadDataHandler() {
/* 初始化类成员变量,这里会关键的,初始化状态在后面会作为出错处理
重试机制的判断参数 */
m_sourceRetryCount = 0;
m_sourceId = g_idle_add(reinterpret_cast<GSourceFunc>(&onReadData), this);
}
gboolean BaseStreamSource::onReadData(gpointer pointer) {
return static_cast<BaseStreamSource*>(pointer)->handleReadData();
}
//重点来了,数据如何读取,读取后如何给gst都在这里找到了答案
gboolean AttachmentReaderSource::handleReadData() {
auto buffer = gst_buffer_new_allocate(nullptr, CHUNK_SIZE, nullptr);
gst_buffer_map(buffer, &info, GST_MAP_WRITE);
auto size = m_reader->read(info.data, info.size, &status);
auto flowRet = gst_app_src_push_buffer(getAppSrc(), buffer);
}
2)
void BaseStreamSource::onEnoughData(GstElement* pipeline, gpointer pointer) {
source->m_enoughDataCallbackId = source->m_pipeline->queueCallback(&source->m_handleEnoughDataFunction);
}