FFmpeg源码分析:写音视频帧av_write_frame()

FFmpeg在libavformat模块提供音视频的muxer封装与demuxer解封装。其中muxer封装文件包括avformat_write_header()、av_write_frame()和av_write_trailer()。本文主要探讨av_write_frame函数如何写入音视频帧数据,包括音视频交错与音视频非交错两种情况。

av_write_frame函数位于libavformat/mux.c,写音视频帧的流程如下:
在这里插入图片描述

1、av_write_frame

av_write_frame的声明位于libavformat/avformat.h,具体如下:

/**
 * Write a packet to an output media file.
 *
 * This function passes the packet directly to the muxer, without any buffering
 * or reordering. The caller is responsible for correctly interleaving the
 * packets if the format requires it. Callers that want libavformat to handle
 * the interleaving should call av_interleaved_write_frame() instead of this
 * function.
 *
 * @param s media file handle
 * @param pkt The packet containing the data to be written. Note that unlike
 *            av_interleaved_write_frame(), this function does not take
 *            ownership of the packet passed to it (though some muxers may make
 *            an internal reference to the input packet).
 * @return < 0 on error, = 0 if OK, 1 if flushed and there is no more data to flush
 *
 * @see av_interleaved_write_frame()
 */
int av_write_frame(AVFormatContext *s, AVPacket *pkt);

大致翻译为:写一个数据包到输出媒体文件。这个函数直接传递数据包到封装器,没有任何缓存或重排序。如果格式需要,调用者要负责把数据包正确排列。如果调用者期望处理交错的数据包,应该调用av_interleaved_write_frame()函数而不是当前这个函数。

下面来看看无交错写音视频av_write_frame()函数的实现,位于libavformat/mux.c:

int av_write_frame(AVFormatContext *s, AVPacket *in)
{
    AVPacket *pkt = s->internal->pkt;
    int ret;
 
    if (!in) {
        if (s->oformat->flags & AVFMT_ALLOW_FLUSH) {
            ret = s->oformat->write_packet(s, NULL);
            flush_if_needed(s);
            if (ret >= 0 && s->pb && s->pb->error < 0)
                ret = s->pb->error;
            return ret;
        }
        return 1;
    }
    if (in->flags & AV_PKT_FLAG_UNCODED_FRAME) {
        pkt = in;
    } else {
        /* We don't own in, so we have to make sure not to modify it.
         * The following avoids copying in's data unnecessarily. */
        av_packet_unref(pkt);
        pkt->buf  = NULL;
        pkt->data = in->data;
        pkt->size = in->size;
        ret = av_packet_copy_props(pkt, in);
        if (ret < 0)
            return ret;
        if (in->buf) {
            pkt->buf = av_buffer_ref(in->buf);
            if (!pkt->buf) {
                ret = AVERROR(ENOMEM);
                goto fail;
            }
        }
    }
    ret = write_packets_common(s, pkt, 0/*non-interleaved*/);
 
fail:
    av_packet_unref(pkt);
    return ret;
}

再看看有交错写音视频帧av_interleaved_write_frame()函数的实现:

int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
{
    int ret;
 
    if (pkt) {
        ret = write_packets_common(s, pkt, 1/*interleaved*/);
        if (ret < 0)
            av_packet_unref(pkt);
        return ret;
    } else {
        av_log(s, AV_LOG_TRACE, "av_interleaved_write_frame FLUSH\n");
        return interleaved_write_packet(s, NULL, 1/*flush*/);
    }
}

经过对比,内部都是调用write_packets_common()函数,区别在于av_write_frame()传参是无交错,而av_interleaved_write_frame()传参是有交错。

2、write_packets_common

接下来看看write_packets_common()函数的实现:

static int write_packets_common(AVFormatContext *s, AVPacket *pkt, int interleaved)
{
    AVStream *st;
	// 检查pkt的stream_index和codec_type
    int ret = check_packet(s, pkt);
    if (ret < 0)
        return ret;
    st = s->streams[pkt->stream_index];
    // 检查pkt的pts和dts
    ret = prepare_input_packet(s, st, pkt);
    if (ret < 0)
        return ret;
    // 调用s->oformat->check_bitstream()检查码流
    ret = check_bitstream(s, st, pkt);
    if (ret < 0)
        return ret;
 
    if (st->internal->bsfc) {
		// 经过bitstream filter处理再写数据包
        return write_packets_from_bsfs(s, st, pkt, interleaved);
    } else {
		// 通用方式写数据包
        return write_packet_common(s, st, pkt, interleaved);
    }
}

由源码可知,主要是分4个步骤处理:

  1. 检查pkt的stream_index和codec_type;
  2. 检查pkt的pts和dts;
  3. 调用s->oformat->check_bitstream()检查码流;
  4. 调用write_packets_from_bsfs或write_packet_common写数据包;

3、write_packet_common

write_packets_from_bsfs()函数主要是经过bitstream filter处理,比如h264要处理startcode起始码。我们主要探讨write_packet_common()函数的实现:

static int write_packet_common(AVFormatContext *s, AVStream *st, AVPacket *pkt, int interleaved)
{
    int ret;
    // 猜测pkt数据包的时长
    guess_pkt_duration(s, st, pkt);
 
#if FF_API_COMPUTE_PKT_FIELDS2 && FF_API_LAVF_AVCTX
    if ((ret = compute_muxer_pkt_fields(s, st, pkt)) < 0 && !(s->oformat->flags & AVFMT_NOTIMESTAMPS))
        return ret;
#endif
 
    if (interleaved) {
		// 有交错写数据包
        if (pkt->dts == AV_NOPTS_VALUE && !(s->oformat->flags & AVFMT_NOTIMESTAMPS))
            return AVERROR(EINVAL);
        return interleaved_write_packet(s, pkt, 0);
    } else {
		// 无交错写数据包
        return write_packet(s, pkt);
    }
}

由此可见,主要根据interleaved标志位判断,如果为有交错就调用interleaved_write_packet()函数写数据包,如果为无交错就调用write_packet()函数写数据包。

4、write_packet与interleaved_write_packet

先看看interleaved_write_packet()函数的实现:

static int interleaved_write_packet(AVFormatContext *s, AVPacket *pkt, int flush)
{
    for (;; ) {
        AVPacket opkt;
        int ret = interleave_packet(s, &opkt, pkt, flush);
        if (ret <= 0)
            return ret;
 
        pkt = NULL;
        ret = write_packet(s, &opkt);
        av_packet_unref(&opkt);
        if (ret < 0)
            return ret;
    }
}

由此可见,有交错写数据包先调用interleave_packet()函数进行交错排序,最终调用write_packet()函数写入数据包。接下来分析write_packet()函数的实现:

static int write_packet(AVFormatContext *s, AVPacket *pkt)
{
    int ret;
    // 根据output_ts_offset来校正dts和pts时间戳
    if (s->output_ts_offset) {
        AVStream *st = s->streams[pkt->stream_index];
        int64_t offset = av_rescale_q(s->output_ts_offset, AV_TIME_BASE_Q, st->time_base);
 
        if (pkt->dts != AV_NOPTS_VALUE)
            pkt->dts += offset;
        if (pkt->pts != AV_NOPTS_VALUE)
            pkt->pts += offset;
    }
    // 校正负数的时间戳
    if (s->avoid_negative_ts > 0) {
        AVStream *st = s->streams[pkt->stream_index];
        int64_t offset = st->internal->mux_ts_offset;
        int64_t ts = s->internal->avoid_negative_ts_use_pts ? pkt->pts : pkt->dts;
 
        if (s->internal->offset == AV_NOPTS_VALUE && ts != AV_NOPTS_VALUE &&
            (ts < 0 || s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO)) {
            s->internal->offset = -ts;
            s->internal->offset_timebase = st->time_base;
        }
        if (s->internal->offset != AV_NOPTS_VALUE && !offset) {
            offset = st->internal->mux_ts_offset =
                av_rescale_q_rnd(s->internal->offset,
                                 s->internal->offset_timebase,
                                 st->time_base,
                                 AV_ROUND_UP);
        }
        if (pkt->dts != AV_NOPTS_VALUE)
            pkt->dts += offset;
        if (pkt->pts != AV_NOPTS_VALUE)
            pkt->pts += offset;
    }
 
    if ((pkt->flags & AV_PKT_FLAG_UNCODED_FRAME)) {
        AVFrame **frame = (AVFrame **)pkt->data;
        av_assert0(pkt->size == sizeof(*frame));
		// 写入未压缩编码的数据包
        ret = s->oformat->write_uncoded_frame(s, pkt->stream_index, frame, 0);
    } else {
		// 写入编码的数据包
        ret = s->oformat->write_packet(s, pkt);
    }
    if (s->pb && ret >= 0) {
        flush_if_needed(s);
        if (s->pb->error < 0)
            ret = s->pb->error;
    }
    if (ret >= 0)
        s->streams[pkt->stream_index]->nb_frames++;
 
    return ret;
}

由源码可知,分3个步骤处理:

  1. 根据output_ts_offset来校正dts和pts时间戳;
  2. 校正负数的时间戳;
  3. 写入未压缩编码的数据包或写入编码的数据包;

5、s->oformat->write_packet

以mp4封装格式为例,位于libavformat/movenc.c,来看看mp4的AVOutputFormat:

AVOutputFormat ff_mp4_muxer = {
    .name              = "mp4",
    .long_name         = NULL_IF_CONFIG_SMALL("MP4 (MPEG-4 Part 14)"),
    .mime_type         = "video/mp4",
    .extensions        = "mp4",
    .priv_data_size    = sizeof(MOVMuxContext),
    .audio_codec       = AV_CODEC_ID_AAC,
    .video_codec       = CONFIG_LIBX264_ENCODER ?
                         AV_CODEC_ID_H264 : AV_CODEC_ID_MPEG4,
    .init              = mov_init,
    .write_header      = mov_write_header,
    .write_packet      = mov_write_packet,
    .write_trailer     = mov_write_trailer,
    .deinit            = mov_free,
    .flags             = AVFMT_GLOBALHEADER | AVFMT_ALLOW_FLUSH | AVFMT_TS_NEGATIVE,
    .codec_tag         = mp4_codec_tags_list,
    .check_bitstream   = mov_check_bitstream,
    .priv_class        = &mp4_muxer_class,
};

此时write_packet函数指针指向mov_write_packet(),我们来探讨下mov_write_packet()函数实现:

static int mov_write_packet(AVFormatContext *s, AVPacket *pkt)
{
    MOVMuxContext *mov = s->priv_data;
    MOVTrack *trk;
    // 判断pkt是否为空
    if (!pkt) {
        mov_flush_fragment(s, 1);
        return 1;
    }
    trk = &mov->tracks[pkt->stream_index];
    if (is_cover_image(trk->st)) {
        int ret;
        if (trk->st->nb_frames >= 1) {
            return 0;
        }
        if ((ret = av_packet_ref(trk->cover_image, pkt)) < 0)
            return ret;
        return 0;
    } else {
        int i;
        if (!pkt->size)
            return mov_write_single_packet(s, pkt); /* Passthrough. */
        /*
         * Subtitles require special handling.
         *
         * 1) For full complaince, every track must have a sample at
         * dts == 0, which is rarely true for subtitles. So, as soon
         * as we see any packet with dts > 0, write an empty subtitle
         * at dts == 0 for any subtitle track with no samples in it.
         *
         * 2) For each subtitle track, check if the current packet's
         * dts is past the duration of the last subtitle sample. If
         * so, we now need to write an end sample for that subtitle.
         *
         * 3) See mov_write_trailer for how the final end sample is
         * handled.
         */
        for (i = 0; i < mov->nb_streams; i++) {
            MOVTrack *trk = &mov->tracks[i];
            int ret;
            if (trk->par->codec_id == AV_CODEC_ID_MOV_TEXT &&
                trk->track_duration < pkt->dts &&
                (trk->entry == 0 || !trk->last_sample_is_subtitle_end)) {
                // 写入字幕流结束的数据包
                ret = mov_write_subtitle_end_packet(s, i, trk->track_duration);
                if (ret < 0) return ret;
                trk->last_sample_is_subtitle_end = 1;
            }
        }
        ......
        // 写入单个数据包
        return mov_write_single_packet(s, pkt);
    }
}

由此可见,先判断pkt数据包是否为空,最终调用mov_write_single_packet()函数写单个数据包,函数实现如下:

static int mov_write_single_packet(AVFormatContext *s, AVPacket *pkt)
{
    MOVMuxContext *mov = s->priv_data;
    MOVTrack *trk = &mov->tracks[pkt->stream_index];
    AVCodecParameters *par = trk->par;
    int64_t frag_duration = 0;
    int size = pkt->size;
    // 判断pkt是否为空
    int ret = check_pkt(s, pkt);
    if (ret < 0)
        return ret;
	
    ......
 
    return ff_mov_write_packet(s, pkt);
}

同样地,该函数也是先判断pk数据包是否为空,中间做一些逻辑处理,最终调用ff_mov_write_packet()函数来真正写数据包:

int ff_mov_write_packet(AVFormatContext *s, AVPacket *pkt)
{
    ......
	
    // 如果存在extradata则进行拷贝
    if (trk->vos_len == 0 && par->extradata_size > 0 &&
        !TAG_IS_AVCI(trk->tag) &&
        (par->codec_id != AV_CODEC_ID_DNXHD)) {
        trk->vos_len  = par->extradata_size;
        trk->vos_data = av_malloc(trk->vos_len + AV_INPUT_BUFFER_PADDING_SIZE);
        if (!trk->vos_data) {
            ret = AVERROR(ENOMEM);
            goto err;
        }
        memcpy(trk->vos_data, par->extradata, trk->vos_len);
        memset(trk->vos_data + trk->vos_len, 0, AV_INPUT_BUFFER_PADDING_SIZE);
    }
    if ((par->codec_id == AV_CODEC_ID_DNXHD ||
         par->codec_id == AV_CODEC_ID_H264 ||
         par->codec_id == AV_CODEC_ID_HEVC ||
         par->codec_id == AV_CODEC_ID_TRUEHD ||
         par->codec_id == AV_CODEC_ID_AC3) && !trk->vos_len &&
         !TAG_IS_AVCI(trk->tag)) {
        /* copy frame to create needed atoms */
        trk->vos_len  = size;
        trk->vos_data = av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE);
        if (!trk->vos_data) {
            ret = AVERROR(ENOMEM);
            goto err;
        }
        memcpy(trk->vos_data, pkt->data, size);
        memset(trk->vos_data + size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
    }
    if (par->codec_id == AV_CODEC_ID_AAC && pkt->size > 2 &&
        (AV_RB16(pkt->data) & 0xfff0) == 0xfff0) {
        if (!s->streams[pkt->stream_index]->nb_frames) {
            return -1;
        }
        av_log(s, AV_LOG_WARNING, "aac bitstream error\n");
    }
    if (par->codec_id == AV_CODEC_ID_H264 && trk->vos_len > 0 
	    && *(uint8_t *)trk->vos_data != 1 && !TAG_IS_AVCI(trk->tag)) {
        /* from x264 or from bytestream H.264 */
        /* NAL reformatting needed */
        if (trk->hint_track >= 0 && trk->hint_track < mov->nb_streams) {
            ret = ff_avc_parse_nal_units_buf(pkt->data, &reformatted_data,
                                             &size);
            if (ret < 0)
                return ret;
            avio_write(pb, reformatted_data, size);
        } else {
            if (trk->cenc.aes_ctr) {
                size = ff_mov_cenc_avc_parse_nal_units(&trk->cenc, pb, pkt->data, size);
                if (size < 0) {
                    ret = size;
                    goto err;
                }
            } else {
                size = ff_avc_parse_nal_units(pb, pkt->data, pkt->size);
            }
        }
    } else if (par->codec_id == AV_CODEC_ID_HEVC && trk->vos_len > 6 &&
               (AV_RB24(trk->vos_data) == 1 || AV_RB32(trk->vos_data) == 1)) {
        /* extradata is Annex B, assume the bitstream is too and convert it */
        if (trk->hint_track >= 0 && trk->hint_track < mov->nb_streams) {
            ret = ff_hevc_annexb2mp4_buf(pkt->data, &reformatted_data,
                                         &size, 0, NULL);
            if (ret < 0)
                return ret;
            avio_write(pb, reformatted_data, size);
        } else {
            size = ff_hevc_annexb2mp4(pb, pkt->data, pkt->size, 0, NULL);
        }
    } else if (par->codec_id == AV_CODEC_ID_AV1) {
        if (trk->hint_track >= 0 && trk->hint_track < mov->nb_streams) {
            ret = ff_av1_filter_obus_buf(pkt->data, &reformatted_data,
                                         &size, &offset);
            if (ret < 0)
                return ret;
            avio_write(pb, reformatted_data, size);
        } else {
            size = ff_av1_filter_obus(pb, pkt->data, pkt->size);
        }
#if CONFIG_AC3_PARSER
    } else if (par->codec_id == AV_CODEC_ID_EAC3) {
        size = handle_eac3(mov, pkt, trk);
        if (size < 0)
            return size;
        else if (!size)
            goto end;
        avio_write(pb, pkt->data, size);
#endif
    } else if (par->codec_id == AV_CODEC_ID_EIA_608) {
        size = 8;
 
        for (int i = 0; i < pkt->size; i += 3) {
            if (pkt->data[i] == 0xFC) {
                size += 2;
            }
        }
        avio_wb32(pb, size);
        ffio_wfourcc(pb, "cdat");
        for (int i = 0; i < pkt->size; i += 3) {
            if (pkt->data[i] == 0xFC) {
                avio_w8(pb, pkt->data[i + 1]);
                avio_w8(pb, pkt->data[i + 2]);
            }
        }
    } else {
        if (trk->cenc.aes_ctr) {
            if (par->codec_id == AV_CODEC_ID_H264 && par->extradata_size > 4) {
                int nal_size_length = (par->extradata[4] & 0x3) + 1;
                ret = ff_mov_cenc_avc_write_nal_units(s, &trk->cenc, nal_size_length, pb, pkt->data, size);
            } else {
                ret = ff_mov_cenc_write_packet(&trk->cenc, pb, pkt->data, size);
            }
            if (ret) {
                goto err;
            }
        } else {
            avio_write(pb, pkt->data, size);
        }
    }
    if (trk->entry >= trk->cluster_capacity) {
        unsigned new_capacity = trk->entry + MOV_INDEX_CLUSTER_SIZE;
        void *cluster = av_realloc_array(trk->cluster, new_capacity, sizeof(*trk->cluster));
        if (!cluster) {
            ret = AVERROR(ENOMEM);
            goto err;
        }
        trk->cluster          = cluster;
        trk->cluster_capacity = new_capacity;
    }
 
    trk->cluster[trk->entry].pos              = avio_tell(pb) - size;
    trk->cluster[trk->entry].samples_in_chunk = samples_in_chunk;
    trk->cluster[trk->entry].chunkNum         = 0;
    trk->cluster[trk->entry].size             = size;
    trk->cluster[trk->entry].entries          = samples_in_chunk;
    trk->cluster[trk->entry].dts              = pkt->dts;
    trk->cluster[trk->entry].pts              = pkt->pts;
    if (!trk->entry && trk->start_dts != AV_NOPTS_VALUE) {
        if (!trk->frag_discont) {
            trk->cluster[trk->entry].dts = trk->start_dts + trk->track_duration;
            if ((mov->flags & FF_MOV_FLAG_DASH &&
                !(mov->flags & (FF_MOV_FLAG_GLOBAL_SIDX | FF_MOV_FLAG_SKIP_SIDX))) ||
                mov->mode == MODE_ISM)
                pkt->pts = pkt->dts + trk->end_pts - trk->cluster[trk->entry].dts;
        } else {
            trk->frag_start = pkt->dts - trk->start_dts;
            trk->end_pts = AV_NOPTS_VALUE;
            trk->frag_discont = 0;
        }
    }
    if (!trk->entry && trk->start_dts == AV_NOPTS_VALUE && !mov->use_editlist &&
        s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO) {
        trk->cluster[trk->entry].dts = trk->start_dts = 0;
    }
    if (trk->start_dts == AV_NOPTS_VALUE) {
        trk->start_dts = pkt->dts;
        if (trk->frag_discont) {
            if (mov->use_editlist) {
                trk->frag_start = pkt->pts;
                trk->start_dts  = pkt->dts - pkt->pts;
            } else {
                trk->frag_start = pkt->dts;
                trk->start_dts  = 0;
            }
            trk->frag_discont = 0;
        } else if (pkt->dts && mov->moov_written)
            av_log(s, AV_LOG_WARNING,
                   "Track %d starts with a nonzero dts %"PRId64", while the moov "
                   "already has been written.\n", pkt->stream_index, pkt->dts);
    }
    trk->track_duration = pkt->dts - trk->start_dts + pkt->duration;
    trk->last_sample_is_subtitle_end = 0;
    if (pkt->pts == AV_NOPTS_VALUE) {
        pkt->pts = pkt->dts;
    }
    if (pkt->dts != pkt->pts)
        trk->flags |= MOV_TRACK_CTTS;
    trk->cluster[trk->entry].cts   = pkt->pts - pkt->dts;
    trk->cluster[trk->entry].flags = 0;
    if (trk->start_cts == AV_NOPTS_VALUE)
        trk->start_cts = pkt->pts - pkt->dts;
    if (trk->end_pts == AV_NOPTS_VALUE)
        trk->end_pts = trk->cluster[trk->entry].dts +
                       trk->cluster[trk->entry].cts + pkt->duration;
    else
        trk->end_pts = FFMAX(trk->end_pts, trk->cluster[trk->entry].dts +
                                           trk->cluster[trk->entry].cts +
                                           pkt->duration);
 
    if (par->codec_id == AV_CODEC_ID_VC1) {
        mov_parse_vc1_frame(pkt, trk);
    } else if (par->codec_id == AV_CODEC_ID_TRUEHD) {
        mov_parse_truehd_frame(pkt, trk);
    } else if (pkt->flags & AV_PKT_FLAG_KEY) {
        if (mov->mode == MODE_MOV && par->codec_id == AV_CODEC_ID_MPEG2VIDEO &&
            trk->entry > 0) { // force sync sample for the first key frame
            mov_parse_mpeg2_frame(pkt, &trk->cluster[trk->entry].flags);
            if (trk->cluster[trk->entry].flags & MOV_PARTIAL_SYNC_SAMPLE)
                trk->flags |= MOV_TRACK_STPS;
        } else {
            trk->cluster[trk->entry].flags = MOV_SYNC_SAMPLE;
        }
        if (trk->cluster[trk->entry].flags & MOV_SYNC_SAMPLE)
            trk->has_keyframes++;
    }
    
	......
 
end:
err:
    if (pkt->data != reformatted_data)
        av_free(reformatted_data);
    return ret;
}

至此,av_write_frame()写音视频函数过程分析完毕。

  • 1
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值