转载:http://blog.csdn.net/ManagerUser/article/details/76087151
一、前言
- .m3u8文件:播放控制文件,存放了地址和播放参数。
- .ts文件:真正存储视频文件。
二、代码分析
SRS源码HLS处理框架如下:
接着SRS(simple-rtmp-server)流媒体服务器源码分析--RTMP消息play分析,在接受到rtmp信息之后,进行HLS处理。
- int SrsSource::on_video_imp(SrsSharedPtrMessage* msg)
- #ifdef SRS_AUTO_HLS
- if ((ret = hls->on_video(msg, is_sequence_header)) != ERROR_SUCCESS) {
- // apply the error strategy for hls.
- // @see https://github.com/ossrs/srs/issues/264
- std::string hls_error_strategy = _srs_config->get_hls_on_error(_req->vhost);
- if (srs_config_hls_is_on_error_ignore(hls_error_strategy)) {
- srs_warn("hls process video message failed, ignore and disable hls. ret=%d", ret);
- // unpublish, ignore ret.
- hls->on_unpublish();
- // ignore.
- ret = ERROR_SUCCESS;
- } else if (srs_config_hls_is_on_error_continue(hls_error_strategy)) {
- if (srs_hls_can_continue(ret, cache_sh_video, msg)) {
- ret = ERROR_SUCCESS;
- } else {
- srs_warn("hls continue video failed. ret=%d", ret);
- return ret;
- }
- } else {
- srs_warn("hls disconnect publisher for video error. ret=%d", ret);
- return ret;
- }
- }
- #endif
- int SrsHls::on_video(SrsSharedPtrMessage* shared_video, bool is_sps_pps)
- {
- int ret = ERROR_SUCCESS;
- if (!hls_enabled) {
- return ret;
- }
- // update the hls time, for hls_dispose.
- last_update_time = srs_get_system_time_ms();
- SrsSharedPtrMessage* video = shared_video->copy();
- SrsAutoFree(SrsSharedPtrMessage, video);
- // user can disable the sps parse to workaround when parse sps failed.
- // @see https://github.com/ossrs/srs/issues/474
- /*
- SPS:
- H.264中定义的sequence parameter sets中包括了一个图像序列的所有信息.它也是H.264的基础之一
- PPS:
- H.264中定义的picture parameter sets中包括了一个图像的所有切片信息.它也是H.264的基础之一
- */
- if (is_sps_pps) {
- codec->avc_parse_sps = _srs_config->get_parse_sps(_req->vhost);
- }
- sample->clear();
- //检查H264编码,序列头
- if ((ret = codec->video_avc_demux(video->payload, video->size, sample)) != ERROR_SUCCESS) {
- srs_error("hls codec demux video failed. ret=%d", ret);
- return ret;
- }
- srs_info("video decoded, type=%d, codec=%d, avc=%d, cts=%d, size=%d, time=%"PRId64,
- sample->frame_type, codec->video_codec_id, sample->avc_packet_type, sample->cts, video->size, video->timestamp);
- // ignore info frame,
- // @see https://github.com/ossrs/srs/issues/288#issuecomment-69863909
- if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) {
- return ret;
- }
- if (codec->video_codec_id != SrsCodecVideoAVC) {
- return ret;
- }
- // ignore sequence header
- if (sample->frame_type == SrsCodecVideoAVCFrameKeyFrame
- && sample->avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) {
- return hls_cache->on_sequence_header(muxer);
- }
- // rtmp抖动矫正
- // TODO: FIXME: config the jitter of HLS.
- if ((ret = jitter->correct(video, SrsRtmpJitterAlgorithmOFF)) != ERROR_SUCCESS) {
- srs_error("rtmp jitter correct video failed. ret=%d", ret);
- return ret;
- }
- // PAR(pixel aspect ratio):像素横纵比 方形为1,长方形小于1
- // DAR(display aspect ratio):显示比例 16:9 4:3
- /*
- DTS:Decode Time Stamp.DTS主要是标识读入内存的字节流在什么时候开始送人解码器中进行解码
- DTS主要是用于视频解码,在解码阶段使用
- PTS主要用于视频同步和输出,在显示阶段使用,在没有B frame的情况下,DTS和PTS的输出顺序是一样的
- */
- int64_t dts = video->timestamp * 90;
- stream_dts = dts;
- if ((ret = hls_cache->write_video(codec, muxer, dts, sample)) != ERROR_SUCCESS) {
- srs_error("hls cache write video failed. ret=%d", ret);
- return ret;
- }
- // pithy print message.
- hls_show_mux_log();
- return ret;
- }
1、获取H264编码信息SPS和PPS,注意:SPS和PPS不是每个流中都会有的,它只包含在头帧当中。
2、检测视频压缩编码格式:必须为H264,否则直接退出。
3、RTMP抖动矫正(不清楚做了些什么事,不管了)
4、HLS切片处理。
以上4点,前三点不在这里详细说明。只看HLS切片部分:
- int SrsHlsCache::write_video(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t dts, SrsCodecSample* sample)
- {
- int ret = ERROR_SUCCESS;
- // write video to cache.
- if ((ret = cache->cache_video(codec, dts, sample)) != ERROR_SUCCESS) {
- return ret;
- }
- // when segment overflow, reap if possible.
- if (muxer->is_segment_overflow()) {
- // do reap ts if any of:
- // a. wait keyframe and got keyframe.
- // b. always reap when not wait keyframe.
- if (!muxer->wait_keyframe() || sample->frame_type == SrsCodecVideoAVCFrameKeyFrame) {
- // reap the segment, which will also flush the video.
- if ((ret = reap_segment("video", muxer, cache->video->dts)) != ERROR_SUCCESS) {
- return ret;
- }
- }
- }
- // flush video when got one
- if ((ret = muxer->flush_video(cache)) != ERROR_SUCCESS) {
- srs_error("m3u8 muxer flush video failed. ret=%d", ret);
- return ret;
- }
- return ret;
- }
1、首次或者.ts文件时间溢出时,进入reap_segment()函数,该函数主要负责.m3u8和.ts文件管理(创建,打开,关闭)。注意:.m3u8文件是在ts文件写入完毕之后,在关闭操作中一次性将播放参数,ts地址等等参数写入。
2、其他时间,直接进入后面flush_video()函数,该函数负责ts流编码和.ts文件写入。
其中ts流编码设计到很多知识,大家自行查看相关知识:http://blog.csdn.net/u013354805/article/details/51578457
- int SrsHlsMuxer::flush_video(SrsTsCache* cache)
- {
- int ret = ERROR_SUCCESS;
- // if current is NULL, segment is not open, ignore the flush event.
- if (!current) {
- srs_warn("flush video ignored, for segment is not open.");
- return ret;
- }
- if (!cache->video || cache->video->payload->length() <= 0) {
- return ret;
- }
- srs_assert(current);
- //更新该ts文件持续时间
- // update the duration of segment.
- current->update_duration(cache->video->dts);
- //贴片处理
- if ((ret = current->muxer->write_video(cache->video)) != ERROR_SUCCESS) {
- return ret;
- }
- // write success, clear and free the msg
- srs_freep(cache->video);
- return ret;
- }
- int SrsTsContext::encode(SrsFileWriter* writer, SrsTsMessage* msg, SrsCodecVideo vc, SrsCodecAudio ac)
- {
- int ret = ERROR_SUCCESS;
- SrsTsStream vs, as;
- int16_t video_pid = 0, audio_pid = 0;
- switch (vc) {
- case SrsCodecVideoAVC:
- vs = SrsTsStreamVideoH264;
- video_pid = TS_VIDEO_AVC_PID;
- break;
- case SrsCodecVideoDisabled:
- vs = SrsTsStreamReserved;
- break;
- case SrsCodecVideoReserved:
- case SrsCodecVideoReserved1:
- case SrsCodecVideoReserved2:
- case SrsCodecVideoSorensonH263:
- case SrsCodecVideoScreenVideo:
- case SrsCodecVideoOn2VP6:
- case SrsCodecVideoOn2VP6WithAlphaChannel:
- case SrsCodecVideoScreenVideoVersion2:
- vs = SrsTsStreamReserved;
- break;
- }
- switch (ac) {
- case SrsCodecAudioAAC:
- as = SrsTsStreamAudioAAC;
- audio_pid = TS_AUDIO_AAC_PID;
- break;
- case SrsCodecAudioMP3:
- as = SrsTsStreamAudioMp3;
- audio_pid = TS_AUDIO_MP3_PID;
- break;
- case SrsCodecAudioDisabled:
- as = SrsTsStreamReserved;
- break;
- case SrsCodecAudioReserved1:
- case SrsCodecAudioLinearPCMPlatformEndian:
- case SrsCodecAudioADPCM:
- case SrsCodecAudioLinearPCMLittleEndian:
- case SrsCodecAudioNellymoser16kHzMono:
- case SrsCodecAudioNellymoser8kHzMono:
- case SrsCodecAudioNellymoser:
- case SrsCodecAudioReservedG711AlawLogarithmicPCM:
- case SrsCodecAudioReservedG711MuLawLogarithmicPCM:
- case SrsCodecAudioReserved:
- case SrsCodecAudioSpeex:
- case SrsCodecAudioReservedMP3_8kHz:
- case SrsCodecAudioReservedDeviceSpecificSound:
- as = SrsTsStreamReserved;
- break;
- }
- if (as == SrsTsStreamReserved && vs == SrsTsStreamReserved) {
- ret = ERROR_HLS_NO_STREAM;
- srs_error("hls: no video or audio stream, vcodec=%d, acodec=%d. ret=%d", vc, ac, ret);
- return ret;
- }
- // when any codec changed, write PAT/PMT table.
- if (vcodec != vc || acodec != ac) {
- vcodec = vc;
- acodec = ac;
- if ((ret = encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) != ERROR_SUCCESS) {
- return ret;
- }
- }
- // encode the media frame to PES packets over TS.
- if (msg->is_audio()) {
- return encode_pes(writer, msg, audio_pid, as, vs == SrsTsStreamReserved);
- } else {
- return encode_pes(writer, msg, video_pid, vs, vs == SrsTsStreamReserved);
- }
- }
1、根据音视频类型,获取不同的PID(在TS协议当中,TS包包含了音视频数据,它们是通过TS头协议PID来区分的)
2、TS编码,PAT帧,PMT帧(首帧TS流)
3、TS编码,音视频数据编码(TS流数据部分)
可以先进入encode_pat_pmt()函数函数中看看:
- int SrsTsContext::encode_pat_pmt(SrsFileWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as)
- {
- int ret = ERROR_SUCCESS;
- if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) {
- ret = ERROR_HLS_NO_STREAM;
- srs_error("hls: no pmt pcr pid, vs=%d, as=%d. ret=%d", vs, as, ret);
- return ret;
- }
- int16_t pmt_number = TS_PMT_NUMBER;
- int16_t pmt_pid = TS_PMT_PID;
- if (true) {
- // 创建一个PAT(节目关联表)
- SrsTsPacket* pkt = SrsTsPacket::create_pat(this, pmt_number, pmt_pid);
- SrsAutoFree(SrsTsPacket, pkt);
- // TS包188字节,其中头字节(4字节,184个字节)。加上16字节CRC,总共204字节
- char* buf = new char[SRS_TS_PACKET_SIZE];
- SrsAutoFreeA(char, buf);
- // set the left bytes with 0xFF.
- int nb_buf = pkt->size();
- srs_assert(nb_buf < SRS_TS_PACKET_SIZE);
- memset(buf + nb_buf, 0xFF, SRS_TS_PACKET_SIZE - nb_buf);
- SrsStream stream;
- if ((ret = stream.initialize(buf, nb_buf)) != ERROR_SUCCESS) {
- return ret;
- }
- // 编码
- if ((ret = pkt->encode(&stream)) != ERROR_SUCCESS) {
- srs_error("ts encode ts packet failed. ret=%d", ret);
- return ret;
- }
- if ((ret = writer->write(buf, SRS_TS_PACKET_SIZE, NULL)) != ERROR_SUCCESS) {
- srs_error("ts write ts packet failed. ret=%d", ret);
- return ret;
- }
- }
- if (true) {
- //创建一个PMT
- SrsTsPacket* pkt = SrsTsPacket::create_pmt(this, pmt_number, pmt_pid, vpid, vs, apid, as);
- SrsAutoFree(SrsTsPacket, pkt);
- char* buf = new char[SRS_TS_PACKET_SIZE];
- SrsAutoFreeA(char, buf);
- // set the left bytes with 0xFF.
- int nb_buf = pkt->size();
- srs_assert(nb_buf < SRS_TS_PACKET_SIZE);
- memset(buf + nb_buf, 0xFF, SRS_TS_PACKET_SIZE - nb_buf);
- SrsStream stream;
- if ((ret = stream.initialize(buf, nb_buf)) != ERROR_SUCCESS) {
- return ret;
- }
- // 编码
- if ((ret = pkt->encode(&stream)) != ERROR_SUCCESS) {
- srs_error("ts encode ts packet failed. ret=%d", ret);
- return ret;
- }
- if ((ret = writer->write(buf, SRS_TS_PACKET_SIZE, NULL)) != ERROR_SUCCESS) {
- srs_error("ts write ts packet failed. ret=%d", ret);
- return ret;
- }
- }
- // When PAT and PMT are writen, the context is ready now.
- ready = true;
- return ret;
- }
再来看encode_pes()函数
- int SrsTsContext::encode_pes(SrsFileWriter* writer, SrsTsMessage* msg, int16_t pid, SrsTsStream sid, bool pure_audio)
- {
- int ret = ERROR_SUCCESS;
- // Sometimes, the context is not ready(PAT/PMT write failed), error in this situation.
- if (!ready) {
- ret = ERROR_TS_CONTEXT_NOT_READY;
- srs_error("TS: context not ready, ret=%d", ret);
- return ret;
- }
- if (msg->payload->length() == 0) {
- return ret;
- }
- if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) {
- srs_info("ts: ignore the unknown stream, sid=%d", sid);
- return ret;
- }
- SrsTsChannel* channel = get(pid);
- srs_assert(channel);
- char* start = msg->payload->bytes();
- char* end = start + msg->payload->length();
- char* p = start;
- while (p < end) {
- SrsTsPacket* pkt = NULL;
- if (p == start) {
- // write pcr according to message.
- bool write_pcr = msg->write_pcr;
- // for pure audio, always write pcr.
- // TODO: FIXME: maybe only need to write at begin and end of ts.
- if (pure_audio && msg->is_audio()) {
- write_pcr = true;
- }
- // it's ok to set pcr equals to dts,
- // @see https://github.com/ossrs/srs/issues/311
- // Fig. 3.18. Program Clock Reference of Digital-Video-and-Audio-Broadcasting-Technology, page 65
- // In MPEG-2, these are the "Program Clock Refer- ence" (PCR) values which are
- // nothing else than an up-to-date copy of the STC counter fed into the transport
- // stream at a certain time. The data stream thus carries an accurate internal
- // "clock time". All coding and de- coding processes are controlled by this clock
- // time. To do this, the receiver, i.e. the MPEG decoder, must read out the
- // "clock time", namely the PCR values, and compare them with its own internal
- // system clock, that is to say its own 42 bit counter.
- int64_t pcr = write_pcr? msg->dts : -1;
- // TODO: FIXME: finger it why use discontinuity of msg.
- pkt = SrsTsPacket::create_pes_first(this,
- pid, msg->sid, channel->continuity_counter++, msg->is_discontinuity,
- pcr, msg->dts, msg->pts, msg->payload->length()
- );
- } else {
- pkt = SrsTsPacket::create_pes_continue(this,
- pid, msg->sid, channel->continuity_counter++
- );
- }
- SrsAutoFree(SrsTsPacket, pkt);
- char* buf = new char[SRS_TS_PACKET_SIZE];
- SrsAutoFreeA(char, buf);
- // set the left bytes with 0xFF.
- int nb_buf = pkt->size();
- srs_assert(nb_buf < SRS_TS_PACKET_SIZE);
- int left = (int)srs_min(end - p, SRS_TS_PACKET_SIZE - nb_buf);
- int nb_stuffings = SRS_TS_PACKET_SIZE - nb_buf - left;
- if (nb_stuffings > 0) {
- // set all bytes to stuffings.
- memset(buf, 0xFF, SRS_TS_PACKET_SIZE);
- // padding with stuffings.
- pkt->padding(nb_stuffings);
- // size changed, recalc it.
- nb_buf = pkt->size();
- srs_assert(nb_buf < SRS_TS_PACKET_SIZE);
- left = (int)srs_min(end - p, SRS_TS_PACKET_SIZE - nb_buf);
- nb_stuffings = SRS_TS_PACKET_SIZE - nb_buf - left;
- srs_assert(nb_stuffings == 0);
- }
- memcpy(buf + nb_buf, p, left);
- p += left;
- SrsStream stream;
- if ((ret = stream.initialize(buf, nb_buf)) != ERROR_SUCCESS) {
- return ret;
- }
- if ((ret = pkt->encode(&stream)) != ERROR_SUCCESS) {
- srs_error("ts encode ts packet failed. ret=%d", ret);
- return ret;
- }
- if ((ret = writer->write(buf, SRS_TS_PACKET_SIZE, NULL)) != ERROR_SUCCESS) {
- srs_error("ts write ts packet failed. ret=%d", ret);
- return ret;
- }
- }
- return ret;
- }
编码成TS包,写入TS文件。
三、总结
1、TS编码,实际是按照TS协议来重新整理数据,TS协议是传输协议。TS协议传输的音视频数据压缩格式还是H264/AAC。
2、TS切片,实际是按照TS协议要求的字段长度对音视频数据进行分包处理。
3、TS文件,TS包存储的位置。又涉及到HLS协议和TS协议关系。