FFmpeg源码分析：avformat_find_stream_info分析码流信息

FFmpeg在调用avformat_open_input()之后，可能码流信息不够完整，可以使用avformat_find_stream_info()获取更多的码流信息。比如获取视频帧率、视频宽高，重新计算最大分析时长，打开解码器解码获取codec数据。具体流程如下图所示：

avformat_find_stream_info方法位于libavformat/utils.c，具体代码如下(部分删减)：

int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
{
    // 针对flv/mpeg/mpegts格式，max_analyze_duration另外赋值
    if (!max_analyze_duration) {
        max_stream_analyze_duration =
        max_analyze_duration        = 5*AV_TIME_BASE;
        max_subtitle_analyze_duration = 30*AV_TIME_BASE;
        if (!strcmp(ic->iformat->name, "flv"))
            max_stream_analyze_duration = 90*AV_TIME_BASE;
        if (!strcmp(ic->iformat->name, "mpeg") || !strcmp(ic->iformat->name, "mpegts"))
            max_stream_analyze_duration = 7*AV_TIME_BASE;
    }

    for (i = 0; i < ic->nb_streams; i++) {
        // 查找解码器
        codec = find_probe_decoder(ic, st, st->codecpar->codec_id);
        // 确保字幕头信息已正确设置
        if (st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE
            && codec && !avctx->codec) {
            if (avcodec_open2(avctx, codec, options ? &options[i] : &thread_opt) < 0)
                av_log(ic, AV_LOG_WARNING, "Failed to open codec in %s\n",__FUNCTION__);
        }
        // 尝试打开解码器
        if (!has_codec_parameters(st, NULL) && st->internal->request_probe <= 0) {
            if (codec && !avctx->codec)
                if (avcodec_open2(avctx, codec, options ? &options[i] : &thread_opt) < 0)
                    av_log(ic, AV_LOG_WARNING, "Failed to open codec in %s\n",__FUNCTION__);
        }
    }
    ......
    for (;;) {
        for (i = 0; i < ic->nb_streams; i++) {
            int fps_analyze_framecount = 20;
            int count;

            st = ic->streams[i];
            if (!has_codec_parameters(st, NULL))
                break;
            // 如果时间基是粗糙的，比如mkv采用ms作为单位
            // 需要分析更多帧数据来获取帧率
            if (av_q2d(st->time_base) > 0.0005)
                fps_analyze_framecount *= 2;
            if (!tb_unreliable(st->internal->avctx))
                fps_analyze_framecount = 0;
            if (ic->fps_probe_size >= 0)
                fps_analyze_framecount = ic->fps_probe_size;
            if (st->disposition & AV_DISPOSITION_ATTACHED_PIC)
                fps_analyze_framecount = 0;
            count = (ic->iformat->flags & AVFMT_NOTIMESTAMPS) ?
                       st->internal->info->codec_info_duration_fields/2 :
                       st->internal->info->duration_count;
            if (!(st->r_frame_rate.num && st->avg_frame_rate.num) &&
                st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
                if (count < fps_analyze_framecount)
                    break;
            }
            // 如果存在帧延时，查看前3帧
            if (st->internal->info->frame_delay_evidence && count < 2 && st->internal->avctx->has_b_frames == 0)
                break;
            if (!st->internal->avctx->extradata &&
                (!st->internal->extract_extradata.inited ||
                 st->internal->extract_extradata.bsf) &&
                extract_extradata_check(st))
                break;
            if (st->first_dts == AV_NOPTS_VALUE &&
                !(ic->iformat->flags & AVFMT_NOTIMESTAMPS) &&
                st->codec_info_nb_frames < ((st->disposition & AV_DISPOSITION_ATTACHED_PIC) ? 1 : ic->max_ts_probe) &&
                (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO ||
                 st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO))
                break;
        }

        // 读取超过指定最大数据量，但还没获取到足够的解码器信息
        if (read_size >= probesize) {
            ret = count;
            av_log(ic, AV_LOG_DEBUG,
                   "Probe buffer size limit of %"PRId64" bytes reached\n", probesize);
            for (i = 0; i < ic->nb_streams; i++)
                if (!ic->streams[i]->r_frame_rate.num &&
                    ic->streams[i]->internal->info->duration_count <= 1 &&
                    ic->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
                    strcmp(ic->iformat->name, "image2"))
                    av_log(ic, AV_LOG_WARNING,
                           "Stream #%d: not enough frames to estimate rate; "
                           "consider increasing probesize\n", i);
            break;
        }
        // 读取一帧数据
        ret = read_frame_internal(ic, pkt1);
        ......
        if (pkt->dts != AV_NOPTS_VALUE && st->codec_info_nb_frames > 1) {
            // 检测非单调递增的dts
            if (st->internal->info->fps_last_dts != AV_NOPTS_VALUE &&
                st->internal->info->fps_last_dts >= pkt->dts) {
                av_log(ic, AV_LOG_DEBUG,
                       "Non-increasing DTS in stream %d: packet %d with DTS "
                       "%"PRId64", packet %d with DTS %"PRId64"\n",
                       st->index, st->internal->info->fps_last_dts_idx,
                       st->internal->info->fps_last_dts, st->codec_info_nb_frames,
                       pkt->dts);
                st->internal->info->fps_first_dts =
                st->internal->info->fps_last_dts  = AV_NOPTS_VALUE;
            }
			// 检测不连续的dts:序列的平均帧时长是否超过1000帧存在差异
            if (st->internal->info->fps_last_dts != AV_NOPTS_VALUE &&
                st->internal->info->fps_last_dts_idx > st->internal->info->fps_first_dts_idx &&
                (pkt->dts - (uint64_t)st->internal->info->fps_last_dts) / 1000 >
                (st->internal->info->fps_last_dts     - (uint64_t)st->internal->info->fps_first_dts) /
                (st->internal->info->fps_last_dts_idx - st->internal->info->fps_first_dts_idx)) {
                st->internal->info->fps_first_dts =
                st->internal->info->fps_last_dts  = AV_NOPTS_VALUE;
            }
            // 更新dts
            if (st->internal->info->fps_first_dts == AV_NOPTS_VALUE) {
                st->internal->info->fps_first_dts     = pkt->dts;
                st->internal->info->fps_first_dts_idx = st->codec_info_nb_frames;
            }
            st->internal->info->fps_last_dts     = pkt->dts;
            st->internal->info->fps_last_dts_idx = st->codec_info_nb_frames;
        }
        // 提取额外数据数组
        if (!st->internal->avctx->extradata) {
            ret = extract_extradata(st, pkt);
            if (ret < 0)
                goto unref_then_goto_end;
        }
        // 如果仍然没有信息，则打开解码器尝试解码一帧
        // 针对MPEG-4, 需要为QuickTime进行解码
        try_decode_frame(ic, st, pkt,
                         (options && i < orig_nb_streams) ? &options[i] : NULL);

        if (ic->flags & AVFMT_FLAG_NOBUFFER)
            av_packet_unref(pkt1);

        st->codec_info_nb_frames++;
        count++;
    }
    if (flush_codecs) {
        for (i = 0; i < ic->nb_streams; i++) {
            st = ic->streams[i];
            // 刷新解码器，尝试解码一帧数据
            if (st->internal->info->found_decoder == 1) {
                do {
                    err = try_decode_frame(ic, st, empty_pkt,
                                            (options && i < orig_nb_streams)
                                            ? &options[i] : NULL);
                } while (err > 0 && !has_codec_parameters(st, NULL));
            }
        }
    }
    ......
    for (i = 0; i < ic->nb_streams; i++) {
        st = ic->streams[i];
        avctx = st->internal->avctx;
        if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
            if (avctx->codec_id == AV_CODEC_ID_RAWVIDEO && !avctx->codec_tag && !avctx->bits_per_coded_sample) {
                uint32_t tag= avcodec_pix_fmt_to_codec_tag(avctx->pix_fmt);
                if (avpriv_find_pix_fmt(avpriv_get_raw_pix_fmt_tags(), tag) == avctx->pix_fmt)
                    avctx->codec_tag= tag;
            }

            // 如果demuxer没有设置帧率，则估算平均帧率
            if (st->internal->info->codec_info_duration_fields &&
                !st->avg_frame_rate.num &&
                st->internal->info->codec_info_duration) {
                int best_fps      = 0;
                double best_error = 0.01;
                AVRational codec_frame_rate = avctx->framerate;

                if (st->internal->info->codec_info_duration        >= INT64_MAX / st->time_base.num / 2||
                    st->internal->info->codec_info_duration_fields >= INT64_MAX / st->time_base.den ||
                    st->internal->info->codec_info_duration        < 0)
                    continue;
                av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
                          st->internal->info->codec_info_duration_fields * (int64_t) st->time_base.den,
                          st->internal->info->codec_info_duration * 2 * (int64_t) st->time_base.num, 60000);

                // 推测标准帧率
                for (j = 0; j < MAX_STD_TIMEBASES; j++) {
                    AVRational std_fps = { get_std_framerate(j), 12 * 1001 };
                    double error       = fabs(av_q2d(st->avg_frame_rate) /
                                              av_q2d(std_fps) - 1);

                    if (error < best_error) {
                        best_error = error;
                        best_fps   = std_fps.num;
                    }
                    if (ic->internal->prefer_codec_framerate && codec_frame_rate.num > 0 && codec_frame_rate.den > 0) {
                        error       = fabs(av_q2d(codec_frame_rate) /
                                           av_q2d(std_fps) - 1);
                        if (error < best_error) {
                            best_error = error;
                            best_fps   = std_fps.num;
                        }
                    }
                }
                if (best_fps)
                    av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
                              best_fps, 12 * 1001, INT_MAX);
            }
            // 计算帧率的分子与分母
            if (!st->r_frame_rate.num) {
                if (    avctx->time_base.den * (int64_t) st->time_base.num
                    <= avctx->time_base.num * (uint64_t)avctx->ticks_per_frame * st->time_base.den) {
                    av_reduce(&st->r_frame_rate.num, &st->r_frame_rate.den,
                              avctx->time_base.den, (int64_t)avctx->time_base.num * avctx->ticks_per_frame, INT_MAX);
                } else {
                    st->r_frame_rate.num = st->time_base.den;
                    st->r_frame_rate.den = st->time_base.num;
                }
            }
			// 计算宽高比
            if (st->internal->display_aspect_ratio.num && st->internal->display_aspect_ratio.den) {
                AVRational hw_ratio = { avctx->height, avctx->width };
                st->sample_aspect_ratio = av_mul_q(st->internal->display_aspect_ratio,
                                                   hw_ratio);
            }
        }
    }
    ......
}

1、最大分析时长

max_analyze_duration用于设定探测格式阶段的分析时长，如果最大分析时长没有被赋值，默认值如下：

max_stream_analyze_duration = 5*AV_TIME_BASE;
max_subtitle_analyze_duration = 30*AV_TIME_BASE;
if (!strcmp(ic->iformat->name, "flv"))
    max_stream_analyze_duration = 90*AV_TIME_BASE;
if (!strcmp(ic->iformat->name, "mpeg") || !strcmp(ic->iformat->name, "mpegts"))
    max_stream_analyze_duration = 7*AV_TIME_BASE;

2、查找解码器

使用find_probe_decoder()来查找解码器，具体代码如下：

static const AVCodec *find_probe_decoder(AVFormatContext *s, const AVStream *st, enum AVCodecID codec_id)
{
    const AVCodec *codec;

#if CONFIG_H264_DECODER
    if (codec_id == AV_CODEC_ID_H264)
        return avcodec_find_decoder_by_name("h264");
#endif

    codec = find_decoder(s, st, codec_id);
    if (!codec)
        return NULL;

    if (codec->capabilities & AV_CODEC_CAP_AVOID_PROBING) {
        const AVCodec *probe_codec = NULL;
        void *iter = NULL;
        while ((probe_codec = av_codec_iterate(&iter))) {
            if (probe_codec->id == codec->id &&
                    av_codec_is_decoder(probe_codec) &&
                    !(probe_codec->capabilities & (AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_EXPERIMENTAL))) {
                return probe_codec;
            }
        }
    }

    return codec;
}

3、打开解码器

使用avcodec_open2()打开解码器，传递三个参数：AVCodecContext、AVCodec、AVOption。分两种情况：字幕流和缺少codec参数。

3.1 字幕流

如果是字幕流，打开解码器，确保字幕头信息已经正确设置。

3.2 没有codec参数

如果缺少codec参数，也会打开解码器，来获取codec相关参数。判断是否有codec参数方法如下：

static int has_codec_parameters(AVStream *st, const char **errmsg_ptr)
{
    AVCodecContext *avctx = st->internal->avctx;

#define FAIL(errmsg) do {                                         \
        if (errmsg_ptr)                                           \
            *errmsg_ptr = errmsg;                                 \
        return 0;                                                 \
    } while (0)

    if (   avctx->codec_id == AV_CODEC_ID_NONE
        && avctx->codec_type != AVMEDIA_TYPE_DATA)
        FAIL("unknown codec");
    switch (avctx->codec_type) {
    case AVMEDIA_TYPE_AUDIO:
        if (!avctx->frame_size && determinable_frame_size(avctx))
            FAIL("unspecified frame size");
        if (st->internal->info->found_decoder >= 0 &&
            avctx->sample_fmt == AV_SAMPLE_FMT_NONE)
            FAIL("unspecified sample format");
        if (!avctx->sample_rate)
            FAIL("unspecified sample rate");
        if (!avctx->channels)
            FAIL("unspecified number of channels");
        if (st->internal->info->found_decoder >= 0 && !st->internal->nb_decoded_frames && avctx->codec_id == AV_CODEC_ID_DTS)
            FAIL("no decodable DTS frames");
        break;
    case AVMEDIA_TYPE_VIDEO:
        if (!avctx->width)
            FAIL("unspecified size");
        if (st->internal->info->found_decoder >= 0 && avctx->pix_fmt == AV_PIX_FMT_NONE)
            FAIL("unspecified pixel format");
        if (st->codecpar->codec_id == AV_CODEC_ID_RV30 || st->codecpar->codec_id == AV_CODEC_ID_RV40)
            if (!st->sample_aspect_ratio.num && !st->codecpar->sample_aspect_ratio.num && !st->codec_info_nb_frames)
                FAIL("no frame in rv30/40 and no sar");
        break;
    case AVMEDIA_TYPE_SUBTITLE:
        if (avctx->codec_id == AV_CODEC_ID_HDMV_PGS_SUBTITLE && !avctx->width)
            FAIL("unspecified size");
        break;
    case AVMEDIA_TYPE_DATA:
        if (avctx->codec_id == AV_CODEC_ID_NONE) return 1;
    }

    return 1;
}

检测音频：sample_rate 、sample_format、channels、frame_size；

检测视频：width、pixel_format、sample_aspect_ratio;

检测字幕：如果是PGS图像字幕流，检测width；

4、读取一帧数据

使用read_frame_internal()来读取一帧数据，内部调用ff_read_packet()和parse_packet()，主要过程是读取packet进行组包，最终读取到一个完整帧。

5、检测dts

检测非单调递增的dts，如果dts不是单调递增，那么判定码流有问题。检测不连续的dts，如果序列的平均帧时长超过1000帧存在差异，那么判定dts不连续。最终更新dts。

6、解码一帧

调用try_decode_frame()进行解码，具体代码如下：

static int try_decode_frame(AVFormatContext *s, AVStream *st,
                            const AVPacket *avpkt, AVDictionary **options)
{
    ......
    if (!avcodec_is_open(avctx) &&
        st->internal->info->found_decoder <= 0 &&
        (st->codecpar->codec_id != -st->internal->info->found_decoder || !st->codecpar->codec_id)) {
        AVDictionary *thread_opt = NULL;

        codec = find_probe_decoder(s, st, st->codecpar->codec_id);

        if (!codec) {
            st->internal->info->found_decoder = -st->codecpar->codec_id;
            ret                     = -1;
            goto fail;
        }

        av_dict_set(options ? options : &thread_opt, "threads", "1", 0);
        av_dict_set(options ? options : &thread_opt, "lowres", "0", 0);
        if (s->codec_whitelist)
            av_dict_set(options ? options : &thread_opt, "codec_whitelist", s->codec_whitelist, 0);
        ret = avcodec_open2(avctx, codec, options ? options : &thread_opt);
        if (!options)
            av_dict_free(&thread_opt);
        if (ret < 0) {
            st->internal->info->found_decoder = -avctx->codec_id;
            goto fail;
        }
        st->internal->info->found_decoder = 1;
    } else if (!st->internal->info->found_decoder)
        st->internal->info->found_decoder = 1;
    ......
    while ((pkt.size > 0 || (!pkt.data && got_picture)) &&
           ret >= 0 &&
           (!has_codec_parameters(st, NULL) || !has_decode_delay_been_guessed(st) ||
            (!st->codec_info_nb_frames &&
             (avctx->codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF)))) {
        got_picture = 0;
        if (avctx->codec_type == AVMEDIA_TYPE_VIDEO ||
            avctx->codec_type == AVMEDIA_TYPE_AUDIO) {
            ret = avcodec_send_packet(avctx, &pkt);
            if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
                break;
            if (ret >= 0)
                pkt.size = 0;
            ret = avcodec_receive_frame(avctx, frame);
            if (ret >= 0)
                got_picture = 1;
            if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
                ret = 0;
        } else if (avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
            ret = avcodec_decode_subtitle2(avctx, &subtitle,
                                           &got_picture, &pkt);
            if (got_picture)
                avsubtitle_free(&subtitle);
            if (ret >= 0)
                pkt.size = 0;
        }
        if (ret >= 0) {
            if (got_picture)
                st->internal->nb_decoded_frames++;
            ret       = got_picture;
        }
    }
    ......
    return ret;
}

7、计算帧率

针对mkv格式，时间基单位是ms，需要读取更多帧来计算帧率。针对有帧延时情况，读取前3帧。然后计算帧率的分子与分母。