目录
- 处理RTMP推流video message
- 处理RTMP推流audio message
- 处理RTMP推流onMetaData message
1. 服务器读取RTMP推流数据 RTMP推流、拉流创建连接说到,SrsRtmpConn::publishing会创建SrsPublishRecvThread协程来接收数据,然后在SrsRtmpConn::do_publishing开启接收协程。 而最终会执行到SrsRecvThread::do_cycle,负责读取RTMP推流数据。 srs_error_t SrsRecvThread::do_cycle() { srs_error_t err = srs_success; while (true) { if ((err = trd->pull()) != srs_success) { return srs_error_wrap(err, "recv thread"); } // When the pumper is interrupted, wait then retry. if (pumper->interrupted()) { srs_usleep(timeout); continue; } SrsCommonMessage *msg = NULL; // Process the received message. 读取msg并进行消费 if ((err = rtmp->recv_message(&msg)) == srs_success) { err = pumper->consume(msg); } if (err != srs_success) { // Interrupt the receive thread for any error. trd->interrupt(); // Notify the pumper to quit for error. pumper->interrupt(err); return srs_error_wrap(err, "recv thread"); } } return err; } 3.SrsRtmpServer::recv_message会获取message,如果获取的是协议控制消息或者用户控制消息会进行解码成packet然后进行响应处理,比如设置ackowledgement_window_size或者chunk_size。 如果是Aduio message,Video message或者Data message,不进行解码成packet,仅获取message。 TODO:后续写一篇关于如何接收message的分析,详细代码见SrsProtocol::recv_interlaced_message srs_error_t SrsProtocol::recv_message(SrsCommonMessage** pmsg)
{
*pmsg = NULL;
srs_error_t err = srs_success;
while (true) {
SrsCommonMessage* msg = NULL;
if ((err = recv_interlaced_message(&msg)) != srs_success) { //获取message
srs_freep(msg);
return srs_error_wrap(err, "recv interlaced message");
}
if (!msg) {
continue;
}
if (msg->size <= 0 || msg->header.payload_length <= 0) {
srs_trace("ignore empty message(type=%d, size=%d, time=%" PRId64 ", sid=%d).",
msg->header.message_type, msg->header.payload_length,
msg->header.timestamp, msg->header.stream_id);
srs_freep(msg);
continue;
}
if ((err = on_recv_message(msg)) != srs_success) { //对获取的message进行判断类型并进行相应处理
srs_freep(msg);
return srs_error_wrap(err, "on recv message");
}
*pmsg = msg;
break;
}
return err;
}
4.SrsRtmpServer::recv_message会获取message后,在SrsPublishRecvThread::consume中进行处理,最后交由SrsRtmpConn::handle_publish_message进行处理。 srs_error_t SrsPublishRecvThread::consume(SrsCommonMessage *msg) {
srs_error_t err = srs_success;
// when cid changed, change it.
if (ncid.compare(cid)) {
_srs_context->set_id(ncid);
cid = ncid;
}
_nb_msgs++;
if (msg->header.is_video()) {
video_frames++;
}
if (msg->header.is_audio()) {
audio_frames++;
}
// log to show the time of recv thread.
srs_verbose("recv thread now=%" PRId64 "us, got msg time=%" PRId64 "ms, size=%d",
srs_update_system_time(), msg->header.timestamp, msg->size);
// the rtmp connection will handle this message 由RTMP连接处理message
err = _conn->handle_publish_message(_source, msg);
// must always free it,
// the source will copy it if need to use.
srs_freep(msg);
if (err != srs_success) {
return srs_error_wrap(err, "handle publish message");
}
// Yield to another coroutines.
// @see https://github.com/ossrs/srs/issues/2194#issuecomment-777463768
if (++nn_msgs_for_yield_ >= 15) {
nn_msgs_for_yield_ = 0;
srs_thread_yield();
}
return err;
}
5.SrsRtmpConn::handle_publish_message会先判断message type是否是命令消息,如果是进行解码并进行处理,推流不需要这些事件数据。 6.audio、video以及data数据由SrsRtmpConn::process_publish_message进行处理。 srs_error_t SrsRtmpConn::handle_publish_message(SrsLiveSource* source, SrsCommonMessage* msg)
{
srs_error_t err = srs_success;
// process publish event. 处理推流事件,因为推流不需要这些事件数据,audio、video以及data数据不会进入此逻辑
if (msg->header.is_amf0_command() || msg->header.is_amf3_command()) {
SrsPacket* pkt = NULL;
if ((err = rtmp->decode_message(msg, &pkt)) != srs_success) {
return srs_error_wrap(err, "rtmp: decode message");
}
SrsAutoFree(SrsPacket, pkt);
// for flash, any packet is republish.
if (info->type == SrsRtmpConnFlashPublish) {
// flash unpublish.
// TODO: maybe need to support republish.
srs_trace("flash flash publish finished.");
return srs_error_new(ERROR_CONTROL_REPUBLISH, "rtmp: republish");
}
// for fmle, drop others except the fmle start packet.
if (dynamic_cast<SrsFMLEStartPacket*>(pkt)) {
SrsFMLEStartPacket* unpublish = dynamic_cast<SrsFMLEStartPacket*>(pkt);
if ((err = rtmp->fmle_unpublish(info->res->stream_id, unpublish->transaction_id)) != srs_success) {
return srs_error_wrap(err, "rtmp: republish");
}
return srs_error_new(ERROR_CONTROL_REPUBLISH, "rtmp: republish");
}
srs_trace("fmle ignore AMF0/AMF3 command message.");
return err;
}
// video, audio, data message
if ((err = process_publish_message(source, msg)) != srs_success) {
return srs_error_wrap(err, "rtmp: consume message");
}
return err;
}
7.SrsRtmpConn::process_publish_message会判断message_type来区别audio、video和onMetaData来进行不同处理。 SrsLiveSource::on_audio负责处理音频message。 SrsLiveSource::on_video负责处理视频message。 SrsLiveSource::on_meta_data负责处理onMetaData message。 srs_error_t SrsRtmpConn::process_publish_message(SrsLiveSource* source, SrsCommonMessage* msg)
{
srs_error_t err = srs_success;
// for edge, directly proxy message to origin.
if (info->edge) {
if ((err = source->on_edge_proxy_publish(msg)) != srs_success) {
return srs_error_wrap(err, "rtmp: proxy publish");
}
return err;
}
// process audio packet
if (msg->header.is_audio()) {
if ((err = source->on_audio(msg)) != srs_success) {
return srs_error_wrap(err, "rtmp: consume audio");
}
return err;
}
// process video packet
if (msg->header.is_video()) {
if ((err = source->on_video(msg)) != srs_success) {
return srs_error_wrap(err, "rtmp: consume video");
}
return err;
}
// process aggregate packet
if (msg->header.is_aggregate()) {
if ((err = source->on_aggregate(msg)) != srs_success) {
return srs_error_wrap(err, "rtmp: consume aggregate");
}
return err;
}
// process onMetaData
if (msg->header.is_amf0_data() || msg->header.is_amf3_data()) {
SrsPacket* pkt = NULL;
if ((err = rtmp->decode_message(msg, &pkt)) != srs_success) {
return srs_error_wrap(err, "rtmp: decode message");
}
SrsAutoFree(SrsPacket, pkt);
if (dynamic_cast<SrsOnMetaDataPacket*>(pkt)) {
SrsOnMetaDataPacket* metadata = dynamic_cast<SrsOnMetaDataPacket*>(pkt);
if ((err = source->on_meta_data(msg, metadata)) != srs_success) {
return srs_error_wrap(err, "rtmp: consume metadata");
}
return err;
}
return err;
}
return err;
}
- 处理RTMP推流video message
SrsLiveSource::on_video负责处理视频message。主要的功能为: 将SrsCommonMessage转换成SrsSharedPtrMessage,用于更少内存拷贝的音频/视频/数据message,SrsSharedPtrMessage使用共享计数方法。 将SrsSharedPtrMessage交给SrsLiveSource::on_video_imp处理。
srs_error_t SrsLiveSource::on_video(SrsCommonMessage* shared_video)
{
srs_error_t err = srs_success;
// monotically increase detect.
if (!mix_correct && is_monotonically_increase) {
if (last_packet_time > 0 && shared_video->header.timestamp < last_packet_time) {
is_monotonically_increase = false;
srs_warn("VIDEO: stream not monotonically increase, please open mix_correct.");
}
}
last_packet_time = shared_video->header.timestamp;
// drop any unknown header video.
// @see https://github.com/ossrs/srs/issues/421
if (!SrsFlvVideo::acceptable(shared_video->payload, shared_video->size)) {
char b0 = 0x00;
if (shared_video->size > 0) {
b0 = shared_video->payload[0];
}
srs_warn("drop unknown header video, size=%d, bytes[0]=%#x", shared_video->size, b0);
return err;
}
// convert shared_video to msg, user should not use shared_video again.
// the payload is transfer to msg, and set to NULL in shared_video.
SrsSharedPtrMessage msg; //创建共享message ptr,将SrsCommonMessage转换成SrsSharedPtrMessage,用于更少内存拷贝的音频/视频/数据message
if ((err = msg.create(shared_video)) != srs_success) { //SrsSharedPtrMessage使用引用计数
return srs_error_wrap(err, "create message");
}
// directly process the video message. mix_correct为false,直接处理video message
if (!mix_correct) {
return on_video_imp(&msg);
}
// insert msg to the queue. 如果mix_correct为true,先将msg加入按时间戳排序map,然后从中取出第一个msg进行处理
mix_queue->push(msg.copy());
// fetch someone from mix queue. 取出的msg可能是音频或视频,所以需要进行判断
SrsSharedPtrMessage* m = mix_queue->pop();
if (!m) {
return err;
}
// consume the monotonically increase message.
if (m->is_audio()) {
err = on_audio_imp(m);
} else {
err = on_video_imp(m);
}
srs_freep(m);
return err;
}
2.SrsLiveSource::on_video_imp处理SrsSharedPtrMessage主要过程包括:
- 遍历对应source拉流客户端,将msg发送给每个客户端。
- 如果不是sequence header,加入gop缓存中,gop会缓存最后一个gop的packet。
srs_error_t SrsLiveSource::on_video_imp(SrsSharedPtrMessage* msg)
{
srs_error_t err = srs_success;
bool is_sequence_header = SrsFlvVideo::sh(msg->payload, msg->size);
// whether consumer should drop for the duplicated sequence header.
bool drop_for_reduce = false;
if (is_sequence_header && meta->previous_vsh() && _srs_config->get_reduce_sequence_header(req->vhost)) {
if (meta->previous_vsh()->size == msg->size) {
drop_for_reduce = srs_bytes_equals(meta->previous_vsh()->payload, msg->payload, msg->size);
srs_warn("drop for reduce sh video, size=%d", msg->size);
}
}
// cache the sequence header if h264
// donot cache the sequence header to gop_cache, return here.
if (is_sequence_header && (err = meta->update_vsh(msg)) != srs_success) {
return srs_error_wrap(err, "meta update video");
}
// Copy to hub to all utilities.
if ((err = hub->on_video(msg, is_sequence_header)) != srs_success) {
return srs_error_wrap(err, "hub consume video");
}
// For bridger to consume the message.
if (bridger_ && (err = bridger_->on_video(msg)) != srs_success) {
return srs_error_wrap(err, "bridger consume video");
}
// copy to all consumer 遍历对应source拉流客户端,将msg发送给每个客户端
if (!drop_for_reduce) {
for (int i = 0; i < (int)consumers.size(); i++) {
SrsLiveConsumer* consumer = consumers.at(i);
if ((err = consumer->enqueue(msg, atc, jitter_algorithm)) != srs_success) {
return srs_error_wrap(err, "consume video");
}
}
}
// when sequence header, donot push to gop cache and adjust the timestamp.
if (is_sequence_header) {
return err;
}
// cache the last gop packets
if ((err = gop_cache->cache(msg)) != srs_success) {
return srs_error_wrap(err, "gop cache consume vdieo");
}
// if atc, update the sequence header to abs time.
if (atc) {
if (meta->vsh()) {
meta->vsh()->timestamp = msg->timestamp;
}
if (meta->data()) {
meta->data()->timestamp = msg->timestamp;
}
}
return err;
}
3.每个拉流客户端是一个SrsLiveConsumer,会将msg加入SrsMessageQueue对象中,实际保存的是SrsMessageQueue对象中的SrsFastVector对象中的SrsSharedPtrMessage数组。 srs_error_t SrsLiveConsumer::enqueue(SrsSharedPtrMessage* shared_msg, bool atc, SrsRtmpJitterAlgorithm ag)
{
srs_error_t err = srs_success;
SrsSharedPtrMessage* msg = shared_msg->copy();
if (!atc) {
if ((err = jitter->correct(msg, ag)) != srs_success) {
return srs_error_wrap(err, "consume message");
}
}
if ((err = queue->enqueue(msg, NULL)) != srs_success) {
return srs_error_wrap(err, "enqueue message");
}
...
return err;
}
srs_error_t SrsMessageQueue::enqueue(SrsSharedPtrMessage* msg, bool* is_overflow)
{
srs_error_t err = srs_success;
msgs.push_back(msg);
// If jitter is off, the timestamp of first sequence header is zero, which wll cause SRS to shrink and drop the
// keyframes even if there is not overflow packets in queue, so we must ignore the zero timestamps, please
// @see https://github.com/ossrs/srs/pull/2186#issuecomment-953383063
if (msg->is_av() && msg->timestamp != 0) {
if (av_start_time == -1) {
av_start_time = srs_utime_t(msg->timestamp * SRS_UTIME_MILLISECONDS);
}
av_end_time = srs_utime_t(msg->timestamp * SRS_UTIME_MILLISECONDS);
}
if (max_queue_size <= 0) {
return err;
}
while (av_end_time - av_start_time > max_queue_size) {
// notice the caller queue already overflow and shrinked.
if (is_overflow) {
*is_overflow = true;
}
shrink();
}
return err;
}
void SrsFastVector::push_back(SrsSharedPtrMessage* msg)
{
// increase vector.
if (count >= nb_msgs) {
int size = srs_max(SRS_PERF_MW_MSGS * 8, nb_msgs * 2);
SrsSharedPtrMessage** buf = new SrsSharedPtrMessage*[size];
for (int i = 0; i < nb_msgs; i++) {
buf[i] = msgs[i];
}
srs_info("fast vector incrase %d=>%d", nb_msgs, size);
// use new array.
srs_freepa(msgs);
msgs = buf;
nb_msgs = size;
}
msgs[count++] = msg;
}
2. 处理RTMP推流audio message SrsLiveSource::on_audio负责处理音频message。主要的功能跟处理video message一样,为: 将SrsCommonMessage转换成SrsSharedPtrMessage,用于更少内存拷贝的音频/视频/数据message,SrsSharedPtrMessage使用共享计数方法。 将SrsSharedPtrMessage交给SrsLiveSource::on_audio_imp处理。 srs_error_t SrsLiveSource::on_audio(SrsCommonMessage* shared_audio)
{
srs_error_t err = srs_success;
// monotically increase detect.
if (!mix_correct && is_monotonically_increase) {
if (last_packet_time > 0 && shared_audio->header.timestamp < last_packet_time) {
is_monotonically_increase = false;
srs_warn("AUDIO: stream not monotonically increase, please open mix_correct.");
}
}
last_packet_time = shared_audio->header.timestamp;
// convert shared_audio to msg, user should not use shared_audio again.
// the payload is transfer to msg, and set to NULL in shared_audio.
SrsSharedPtrMessage msg;
if ((err = msg.create(shared_audio)) != srs_success) {
return srs_error_wrap(err, "create message");
}
// directly process the audio message.
if (!mix_correct) {
return on_audio_imp(&msg);
}
// insert msg to the queue.
mix_queue->push(msg.copy());
// fetch someone from mix queue.
SrsSharedPtrMessage* m = mix_queue->pop();
if (!m) {
return err;
}
// consume the monotonically increase message.
if (m->is_audio()) {
err = on_audio_imp(m);
} else {
err = on_video_imp(m);
}
srs_freep(m);
return err;
}
2.SrsLiveSource::on_audio_imp处理SrsSharedPtrMessage主要过程包括:
- 遍历对应source拉流客户端,将msg发送给每个客户端。
- 如果不是sequence header,加入gop缓存中,gop会缓存最后一个gop的packet。
srs_error_t SrsLiveSource::on_audio_imp(SrsSharedPtrMessage* msg)
{
srs_error_t err = srs_success;
bool is_aac_sequence_header = SrsFlvAudio::sh(msg->payload, msg->size);
bool is_sequence_header = is_aac_sequence_header;
// whether consumer should drop for the duplicated sequence header.
bool drop_for_reduce = false;
if (is_sequence_header && meta->previous_ash() && _srs_config->get_reduce_sequence_header(req->vhost)) {
if (meta->previous_ash()->size == msg->size) {
drop_for_reduce = srs_bytes_equals(meta->previous_ash()->payload, msg->payload, msg->size);
srs_warn("drop for reduce sh audio, size=%d", msg->size);
}
}
// Copy to hub to all utilities.
if ((err = hub->on_audio(msg)) != srs_success) {
return srs_error_wrap(err, "consume audio");
}
// For bridger to consume the message.
if (bridger_ && (err = bridger_->on_audio(msg)) != srs_success) {
return srs_error_wrap(err, "bridger consume audio");
}
// copy to all consumer 遍历对应source拉流客户端,将msg发送给每个客户端
if (!drop_for_reduce) {
for (int i = 0; i < (int)consumers.size(); i++) {
SrsLiveConsumer* consumer = consumers.at(i);
if ((err = consumer->enqueue(msg, atc, jitter_algorithm)) != srs_success) {
return srs_error_wrap(err, "consume message");
}
}
}
// cache the sequence header of aac, or first packet of mp3.
// for example, the mp3 is used for hls to write the "right" audio codec.
// TODO: FIXME: to refine the stream info system.
if (is_aac_sequence_header || !meta->ash()) {
if ((err = meta->update_ash(msg)) != srs_success) {
return srs_error_wrap(err, "meta consume audio");
}
}
// when sequence header, donot push to gop cache and adjust the timestamp.
if (is_sequence_header) {
return err;
}
// cache the last gop packets
if ((err = gop_cache->cache(msg)) != srs_success) {
return srs_error_wrap(err, "gop cache consume audio");
}
// if atc, update the sequence header to abs time.
if (atc) {
if (meta->ash()) {
meta->ash()->timestamp = msg->timestamp;
}
if (meta->data()) {
meta->data()->timestamp = msg->timestamp;
}
}
return err;
}
3.每个拉流客户端是一个SrsLiveConsumer,会将msg加入SrsMessageQueue对象中,实际保存的是SrsMessageQueue对象中的SrsFastVector对象中的SrsSharedPtrMessage数组。跟video存储逻辑一样。 对于onMetaData message,SrsRtmpConn::process_publish_message会先解码成SrsOnMetaDataPacket,然后和onMetaData message一起作为参数传入SrsLiveSource::on_meta_data。主要的功能有: 更新onMetaData message。即更新SrsMetaCache的meta(SrsSharedPtrMessage) 遍历对应source拉流客户端,将共享meta发送给每个客户端。 srs_error_t SrsLiveSource::on_meta_data(SrsCommonMessage* msg, SrsOnMetaDataPacket* metadata)
{
srs_error_t err = srs_success;
// if allow atc_auto and bravo-atc detected, open atc for vhost.
SrsAmf0Any* prop = NULL;
atc = _srs_config->get_atc(req->vhost);
if (_srs_config->get_atc_auto(req->vhost)) {
if ((prop = metadata->metadata->get_property("bravo_atc")) != NULL) {
if (prop->is_string() && prop->to_str() == "true") {
atc = true;
}
}
}
// Update the meta cache. 如果需要更新MetaData,会更新共享SrsSharedPtrMessage* meta
bool updated = false;
if ((err = meta->update_data(&msg->header, metadata, updated)) != srs_success) {
return srs_error_wrap(err, "update metadata");
}
if (!updated) {
return err;
}
// when already got metadata, drop when reduce sequence header.
bool drop_for_reduce = false;
if (meta->data() && _srs_config->get_reduce_sequence_header(req->vhost)) {
drop_for_reduce = true;
srs_warn("drop for reduce sh metadata, size=%d", msg->size);
}
// copy to all consumer 遍历对应source拉流客户端,将共享meta发送给每个客户端。
if (!drop_for_reduce) {
std::vector<SrsLiveConsumer*>::iterator it;
for (it = consumers.begin(); it != consumers.end(); ++it) {
SrsLiveConsumer* consumer = *it;
if ((err = consumer->enqueue(meta->data(), atc, jitter_algorithm)) != srs_success) {
return srs_error_wrap(err, "consume metadata");
}
}
}
// Copy to hub to all utilities.
return hub->on_meta_data(meta->data(), metadata);
}
2.每个拉流客户端是一个SrsLiveConsumer,会将msg加入SrsMessageQueue对象中,实际保存的是SrsMessageQueue对象中的SrsFastVector对象中的SrsSharedPtrMessage数组。跟video和audio的存储逻辑一样。 |