1 环境
系统:win10 64位
IDE:vs2019
ffmpeg:ffmpeg4.4
vs工程配置:
(1)新建vs工程,平台选择x64。
(2)拷贝ffmpeg shared目录下lib、include到工程文件夹,拷贝bin目录下的dll文件到后缀为vcxproj所在的目录下
(3)配置工程属性
- 头文件配置:属性——>C/C++——>常规——>附加包含目录,输入“$(SolutionDir)\include”
- 导入库配置:属性——>链接器——>常规——>附加库目录,输入“$(SolutionDir)\lib”
属性——>链接器——>输入——>附加依赖项,输入“avcodec.lib; avformat.lib; avutil.lib;avdevice.lib; avfilter.lib; postproc.lib; swresample.lib;swscale.lib” - 其他配置:将VS的SDL检查关闭,否则可能出现“被声明为已否决”问题,目前按照老版本的代码流程学习。
2 基础API
(1)日志
//设置log level 级别
av_log_set_level(AV_LOG_DEBUG);
//记录 Info 级别下的 av 日志
av_log(NULL,AV_LOG_INFO,"hello world! use libavutil/log.h!\n");
(2)文件
avpriv_io_delete(fileName) //删除
avpriv_io_move(src,dst) //重命名
(3)meta信息
void av_dump_format(AVFormatContext *ic,
int index,
const char *url,
int is_output);
3 库
文件FFmpeg一共包含8个库:
- avcodec:编解码(最重要的库)。
- avformat:封装格式处理。
- avfilter:滤镜特效处理。
- avdevice:各种设备的输入输出。
- avutil:工具库(大部分库都需要这个库的支持)。
- postproc:后加工。
- swresample:音频采样数据格式转换。
- swscale:视频像素数据格式转换。
4 解码流程
4.1 流程
FFmpeg解码函数简介
av_register_all():注册所有组件。
avformat_open_input():打开输入视频文件。
avformat_find_stream_info():获取视频文件信息。
avcodec_find_decoder():查找解码器。
avcodec_open2():打开解码器。
av_read_frame():从输入文件读取一帧压缩数据。
avcodec_decode_video2():解码一帧压缩数据。
avcodec_close():关闭解码器。
avformat_close_input():关闭输入视频文件
4.2 结构体
FFmpeg数据结构简介
AVFormatContext :封装格式上下文结构体,也是统领全局的结构体,保存了视频文件封装
格式相关信息。
AVInputFormat:每种封装格式(例如FLV, MKV, MP4, AVI)对应一个该结构体。
AVStream:视频文件中每个视频(音频)流对应一个该结构体。
AVCodecContext:编码器上下文结构体,保存了视频(音频)编解码相关信息。
AVCodec:每种视频(音频)编解码器(例如H.264解码器)对应一个该结构体。
AVPacket:存储一帧压缩编码数据。
AVFrame:存储一帧解码后像素(采样)数据
5 程序实例
5.1 抽取音频数据
功能:抽取mkv文件中的aac音频。
流程:
输入文件:打开输入文件,获取参数,查找需要抽取的AAC码流
输出文件:av_guess_format 读取输出格式信息,创建新流,打开文件
写入文件:avformat_write_header 写文件头,av_interleaved_write_frame 将读取的frame写入到输出文件,最后写入尾部信息av_write_trailer(ofmt_ctx)
#include <iostream>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
};
int main(int argc, const char* argv[])
{
/*if (argc != 3)
{
printf("Usage : extrace <input> <output>\n");
return -1;
}*/
const char* input = "in.mkv";
const char* output = "out.aac";
//注册编解码器
av_register_all();
//申请avformatcontext
AVFormatContext* ifmt_ctx = avformat_alloc_context();
if (avformat_open_input(&ifmt_ctx, input, NULL, NULL) != 0)
{
printf("fail to open input stream\n");
return -1;
}
//读取一些流的信息
if (avformat_find_stream_info(ifmt_ctx, NULL) < 0)
{
printf("fail to find the stream info\n");
return -1;
}
//将打开文件的信息以及ffmpeg的信息显示到控制台上
av_dump_format(ifmt_ctx, 0, input, 0);
//找到最好的音频流的索引
int audio_stream_index = -1;
audio_stream_index = av_find_best_stream(ifmt_ctx, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
if (audio_stream_index == -1)
{
printf("fail to find the audio stream\n");
return -1;
}
//音频流和此音频的解码器
AVStream* in_stream = ifmt_ctx->streams[audio_stream_index];
AVCodec* pCodec = avcodec_find_decoder(in_stream->codecpar->codec_id);
if (pCodec == NULL)
{
printf("fail to find the codec\n");
return -1;
}
//从输出文件中读取输出格式
AVOutputFormat* out_fmt = av_guess_format(NULL, output, NULL);
if (out_fmt == NULL)
{
printf("didn`t support this format");
return -1;
}
//创建输出环境,并设置与输出文件关联和设置输出文件的格式
AVFormatContext* ofmt_ctx = avformat_alloc_context();
if (avformat_alloc_output_context2(&ofmt_ctx, out_fmt, out_fmt->name, output) < 0)
{
printf("fail to alloc output context\n");
return -1;
}
//根据编解码器和输出ofmt_ctx创建输出流
AVStream* out_stream = avformat_new_stream(ofmt_ctx, pCodec);
//设置输出流的编解码器参数
if (avcodec_parameters_copy(out_stream->codecpar, in_stream->codecpar) < 0)
{
printf("fail to copy parameteres\n");
return -1;
}
//创建并初始化AVIOContext并置AVIOContext的标志为写入文件,同时与输出文件关联起来
if (avio_open(&ofmt_ctx->pb, output, AVIO_FLAG_WRITE) < 0)
{
printf("fail to open aviocontext\n");
return -1;
}
//写入文件头信息
if (avformat_write_header(ofmt_ctx, NULL) < 0)
{
printf("fail to write header\n");
return -1;
}
//初始化AVpacket
AVPacket* pkt = av_packet_alloc();
av_init_packet(pkt);
//读取ifmt_ctx中的包
while (av_read_frame(ifmt_ctx, pkt) >= 0)
{
if (pkt->stream_index == audio_stream_index)
{
//重新计算时间戳,变换为音频输出的时间戳
pkt->pts = av_rescale_q_rnd(pkt->pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt->dts = pkt->pts;
pkt->duration = av_rescale_q(pkt->duration, in_stream->time_base, out_stream->time_base);
pkt->pos = -1;
pkt->stream_index = 0;
//对pkt检查并写入文件包
av_interleaved_write_frame(ofmt_ctx, pkt);
//将pkt中的资源释放
av_packet_unref(pkt);
}
}
//写入尾部信息
av_write_trailer(ofmt_ctx);
//释放资源,释放pkt
av_free_packet(pkt);
//关闭aviocontext 写入完成
avio_close(ofmt_ctx->pb);
//释放ofmt_ctx
avformat_free_context(ofmt_ctx);
//关闭并释放输入ifmt_ctx
avformat_close_input(&ifmt_ctx);
return 0;
}
5.2 抽取H264数据
H264有两种封装格式,一种是MP4格式,一种是annexb格式,MP4格式是以4个字节长度分割,开始是长度字节。annexb格式是以0x000001 or 0x00000001分割。
直接提取MP4中的h264不能播放,需要进行转换
#include <stdio.h>
#include <iostream>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libswresample/swresample.h"
#include "libavutil/dict.h"
#include "libavutil\log.h"
};
using namespace std;
#ifndef AV_WB32
# define AV_WB32(p, val) do { \
uint32_t d = (val); \
((uint8_t*)(p))[3] = (d); \
((uint8_t*)(p))[2] = (d)>>8; \
((uint8_t*)(p))[1] = (d)>>16; \
((uint8_t*)(p))[0] = (d)>>24; \
} while(0)
#endif
#ifndef AV_RB16
# define AV_RB16(x) \
((((const uint8_t*)(x))[0] << 8) | \
((const uint8_t*)(x))[1])
#endif
static int alloc_and_copy(AVPacket *out,
const uint8_t *sps_pps, uint32_t sps_pps_size,
const uint8_t *in, uint32_t in_size)
{
uint32_t offset = out->size;
uint8_t nal_header_size = offset ? 3 : 4;
int err;
err = av_grow_packet(out, sps_pps_size + in_size + nal_header_size);
if (err < 0)
return err;
if (sps_pps)
memcpy(out->data + offset, sps_pps, sps_pps_size);
memcpy(out->data + sps_pps_size + nal_header_size + offset, in, in_size);
if (!offset) {
AV_WB32(out->data + sps_pps_size, 1);
}
else {
(out->data + offset + sps_pps_size)[0] =
(out->data + offset + sps_pps_size)[1] = 0;
(out->data + offset + sps_pps_size)[2] = 1;
}
return 0;
}
/*
*codec_extradata:
* 1, 64, 0, 1f, ff, e1, 0, 18, 67, 64, 0, 1f, ac, c8, 60, 78, 1b, 7e, 78, 40, 0, 0, fa, 40, 0, 3a, 98, 3, c6, c, 66, 80,
* 1, 0, 5,68, e9, 78, bc, b0, 0,
*/
int h264_extradata_to_annexb(const uint8_t *codec_extradata, const int codec_extradata_size, AVPacket *out_extradata, int padding)
{
uint16_t unit_size;
uint64_t total_size = 0;
uint8_t *out = NULL, unit_nb, sps_done = 0,
sps_seen = 0, pps_seen = 0, sps_offset = 0, pps_offset = 0;
const uint8_t *extradata = codec_extradata + 4;
static const uint8_t nalu_header[4] = { 0, 0, 0, 1 };
int length_size = (*extradata++ & 0x3) + 1; // retrieve length coded size, 用于指示表示编码数据长度所需字节数
sps_offset = pps_offset = -1;
//e1 之后 extradata指向 0 【0,18】为此nal的size 68.。。。。
/* retrieve sps and pps unit(s) */
unit_nb = *extradata++ & 0x1f; /* number of sps unit(s) */
if (!unit_nb) {
goto pps;
}
else {
sps_offset = 0;
sps_seen = 1;
}
while (unit_nb--) {
int err;
unit_size = AV_RB16(extradata);
total_size += unit_size + 4;
if (total_size > INT_MAX - padding) {
av_log(NULL, AV_LOG_ERROR,
"Too big extradata size, corrupted stream or invalid MP4/AVCC bitstream\n");
av_free(out);
return AVERROR(EINVAL);
}
if (extradata + 2 + unit_size > codec_extradata + codec_extradata_size) {
av_log(NULL, AV_LOG_ERROR, "Packet header is not contained in global extradata, "
"corrupted stream or invalid MP4/AVCC bitstream\n");
av_free(out);
return AVERROR(EINVAL);
}
if ((err = av_reallocp(&out, total_size + padding)) < 0)
return err;
memcpy(out + total_size - unit_size - 4, nalu_header, 4);
memcpy(out + total_size - unit_size, extradata + 2, unit_size);
extradata += 2 + unit_size;
pps:
if (!unit_nb && !sps_done++) {
unit_nb = *extradata++; /* number of pps unit(s) */
if (unit_nb) {
pps_offset = total_size;
pps_seen = 1;
}
}
}
if (out)
memset(out + total_size, 0, padding);
if (!sps_seen)
av_log(NULL, AV_LOG_WARNING,
"Warning: SPS NALU missing or invalid. "
"The resulting stream may not play.\n");
if (!pps_seen)
av_log(NULL, AV_LOG_WARNING,
"Warning: PPS NALU missing or invalid. "
"The resulting stream may not play.\n");
out_extradata->data = out;
out_extradata->size = total_size;
return length_size;
}
/** 00 00 32 ce 65
* packet中的数据起始处没有分隔符(0x00000001), 也不是0x65、0x67、0x68、0x41等字节,所以可以肯定这不是标准的nalu。
其实,前4个字0x000032ce表示的是nalu的长度,从第5个字节开始才是nalu的数据。所以直接将前4个字节替换为0x00000001即可得到标准的nalu数据。
*/
int h264_mp4toannexb(AVFormatContext *fmt_ctx, AVPacket *in, FILE *dst_fd)
{
AVPacket *out = NULL;
AVPacket spspps_pkt;
int len;
uint8_t unit_type;
int32_t nal_size;
uint32_t cumul_size = 0;
const uint8_t *buf;
const uint8_t *buf_end;
int buf_size;
int ret = 0, i;
out = av_packet_alloc();
buf = in->data;
buf_size = in->size;
buf_end = in->data + in->size;
do {
ret= AVERROR(EINVAL);
if (buf + 4 /*s->length_size*/ > buf_end)
goto fail;
for (nal_size = 0, i = 0; i<4/*s->length_size*/; i++)
nal_size = (nal_size << 8) | buf[i];
buf += 4; /*s->length_size;*/
unit_type = *buf & 0x1f;
if (nal_size > buf_end - buf || nal_size < 0)
goto fail;
/*
if (unit_type == 7)
s->idr_sps_seen = s->new_idr = 1;
else if (unit_type == 8) {
s->idr_pps_seen = s->new_idr = 1;
*/
/* if SPS has not been seen yet, prepend the AVCC one to PPS */
/*
if (!s->idr_sps_seen) {
if (s->sps_offset == -1)
av_log(ctx, AV_LOG_WARNING, "SPS not present in the stream, nor in AVCC, stream may be unreadable\n");
else {
if ((ret = alloc_and_copy(out,
ctx->par_out->extradata + s->sps_offset,
s->pps_offset != -1 ? s->pps_offset : ctx->par_out->extradata_size - s->sps_offset,
buf, nal_size)) < 0)
goto fail;
s->idr_sps_seen = 1;
goto next_nal;
}
}
}
*/
/* if this is a new IDR picture following an IDR picture, reset the idr flag.
* Just check first_mb_in_slice to be 0 as this is the simplest solution.
* This could be checking idr_pic_id instead, but would complexify the parsing. */
/*
if (!s->new_idr && unit_type == 5 && (buf[1] & 0x80))
s->new_idr = 1;
*/
/* prepend only to the first type 5 NAL unit of an IDR picture, if no sps/pps are already present */
if (/*s->new_idr && */unit_type == 5 /*&& !s->idr_sps_seen && !s->idr_pps_seen*/) {
h264_extradata_to_annexb( fmt_ctx->streams[in->stream_index]->codec->extradata,
fmt_ctx->streams[in->stream_index]->codec->extradata_size,
&spspps_pkt,
AV_INPUT_BUFFER_PADDING_SIZE);
if ((ret=alloc_and_copy(out,
spspps_pkt.data, spspps_pkt.size,
buf, nal_size)) < 0)
goto fail;
/*s->new_idr = 0;*/
/* if only SPS has been seen, also insert PPS */
}
/*else if (s->new_idr && unit_type == 5 && s->idr_sps_seen && !s->idr_pps_seen) {
if (s->pps_offset == -1) {
av_log(ctx, AV_LOG_WARNING, "PPS not present in the stream, nor in AVCC, stream may be unreadable\n");
if ((ret = alloc_and_copy(out, NULL, 0, buf, nal_size)) < 0)
goto fail;
} else if ((ret = alloc_and_copy(out,
ctx->par_out->extradata + s->pps_offset, ctx->par_out->extradata_size - s->pps_offset,
buf, nal_size)) < 0)
goto fail;
}*/ else {
if ((ret = alloc_and_copy(out, NULL, 0, buf, nal_size)) < 0)
goto fail;
/*
if (!s->new_idr && unit_type == 1) {
s->new_idr = 1;
s->idr_sps_seen = 0;
s->idr_pps_seen = 0;
}
*/
}
len = fwrite(out->data, 1, out->size, dst_fd);
if (len != out->size){
av_log(NULL, AV_LOG_DEBUG, "warning, length of writed data isn't equal pkt.size(%d, %d)\n",
len,
out->size);
}
fflush(dst_fd);
next_nal:
buf += nal_size;
cumul_size += nal_size + 4;//s->length_size;
} while (cumul_size < buf_size);
/*
ret = av_packet_copy_props(out, in);
if (ret < 0)
goto fail;
*/
fail:
av_packet_free(&out);
return ret;
}
int extr_video(char *src, char *dst)
{
int err_code;
char errors[1024];
char *src_filename = NULL;
char *dst_filename = NULL;
FILE *dst_fd = NULL;
int video_stream_index = -1;
//AVFormatContext *ofmt_ctx = NULL;
//AVOutputFormat *output_fmt = NULL;
//AVStream *out_stream = NULL;
AVFormatContext *fmt_ctx = NULL;
AVPacket pkt;
//AVFrame *frame = NULL;
av_log_set_level(AV_LOG_DEBUG);
src_filename = src;
dst_filename = dst; // xxx.h264
if (src_filename == NULL || dst_filename == NULL){
av_log(NULL, AV_LOG_ERROR, "src or dts file is null, plz check them!\n");
return -1;
}
/*register all formats and codec*/
av_register_all();
dst_fd = fopen(dst_filename, "wb");
if (!dst_fd) {
av_log(NULL, AV_LOG_DEBUG, "Could not open destination file %s\n", dst_filename);
return -1;
}
/*open input media file, and allocate format context*/
if ((err_code = avformat_open_input(&fmt_ctx, src_filename, NULL, NULL)) < 0){
av_strerror(err_code, errors, 1024);
av_log(NULL, AV_LOG_DEBUG, "Could not open source file: %s, %d(%s)\n",
src_filename,
err_code,
errors);
return -1;
}
/*dump input information*/
av_dump_format(fmt_ctx, 0, src_filename, 0);
/*initialize packet*/
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
/*find best video stream*/
video_stream_index = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
if (video_stream_index < 0){
av_log(NULL, AV_LOG_DEBUG, "Could not find %s stream in input file %s\n",
av_get_media_type_string(AVMEDIA_TYPE_VIDEO),
src_filename);
return AVERROR(EINVAL);
}
/*
if (avformat_write_header(ofmt_ctx, NULL) < 0) {
av_log(NULL, AV_LOG_DEBUG, "Error occurred when opening output file");
exit(1);
}
*/
/*read frames from media file*/
while (av_read_frame(fmt_ctx, &pkt) >= 0){
if (pkt.stream_index == video_stream_index){
/*
pkt.stream_index = 0;
av_write_frame(ofmt_ctx, &pkt);
av_free_packet(&pkt);
*/
h264_mp4toannexb(fmt_ctx, &pkt, dst_fd);
}
//release pkt->data
av_packet_unref(&pkt);
}
//av_write_trailer(ofmt_ctx);
/*close input media file*/
avformat_close_input(&fmt_ctx);
if (dst_fd) {
fclose(dst_fd);
}
//avio_close(ofmt_ctx->pb);
return 0;
}
int main(int argc, char* argv[])
{
extr_video("1.mp4", "out.h264");
return 0;
}
5.3 将mp4转为flv
int main(int argc, char *argv[]) {
int ret;
int len; //写入的值
int videoIndex = 0;
char *filePath = "1.mp4"; //输入文件名
char *outputPath = "2.flv"; //输出文件名
AVFormatContext *inputFmtCtx = NULL, *outputFmtCtx = NULL;
AVPacket pkt;
FILE *fdOutput = NULL;
int stream_index = 0;
av_register_all();
av_log_set_level(AV_LOG_DEBUG); //设置全局打印等级
/*--打开文件--*/
ret = avformat_open_input(&inputFmtCtx, filePath, NULL, NULL);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Open file failed: %s\n");
goto end;
}
/*--打印文件信息--*/
av_dump_format(inputFmtCtx, 0, filePath, 0);
av_log(NULL, AV_LOG_INFO, "dump format success\n");
/*--创建输出上下文--*/
ret = avformat_alloc_output_context2(&outputFmtCtx, NULL, NULL, outputPath);
if (!outputFmtCtx) {
av_log(NULL, AV_LOG_ERROR, "avformat alloc output contextfailed: %s\n");
goto end;
}
uint32_t stream_mapping_size = inputFmtCtx->nb_streams; //获取流的大小
int *stream_mapping = NULL;
stream_mapping = (int *)av_mallocz_array(stream_mapping_size, sizeof(*stream_mapping)); //分配内存
if (!stream_mapping) {
ret = AVERROR(ENOMEM);
goto end;
}
AVOutputFormat *outputFmt = outputFmtCtx->oformat; //获取输出格式
for (int i = 0; i < stream_mapping_size; i++) {
AVStream *outStream;
AVStream *inStream = inputFmtCtx->streams[i]; //获取输入流
AVCodecParameters *inCodecPar = inStream->codecpar; //获取输入流参数
/*--判断解码类型是 视频 音频 字幕其中一个就继续本次循环--*/
if (inCodecPar->codec_type != AVMEDIA_TYPE_AUDIO
&& inCodecPar->codec_type != AVMEDIA_TYPE_VIDEO
&& inCodecPar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
stream_mapping[i] = -1; //如果这个流不需要,就设为-1
continue;
}
stream_mapping[i] = stream_index++;
/*--创建新流--*/
outStream = avformat_new_stream(outputFmtCtx, NULL);
if (!outStream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
/*--拷贝参数--*/
ret = avcodec_parameters_copy(outStream->codecpar, inCodecPar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec Parameters\n");
goto end;
}
outStream->codecpar->codec_tag = 0;
}
av_dump_format(outputFmtCtx, 0, outputPath, 1); //把拷贝进来的参数打印出来
if (!(outputFmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&outputFmtCtx->pb, outputPath, AVIO_FLAG_WRITE); //打开设备
if (ret < 0) {
fprintf(stderr, "Failed to open output file\n");
goto end;
}
}
/*--写入头--*/
ret = avformat_write_header(outputFmtCtx, NULL);
if (ret < 0) {
fprintf(stderr, "Failed to write header to output file\n");
goto end;
}
av_init_packet(&pkt);
while (1)
{
AVStream *in_stream, *out_stream;
ret = av_read_frame(inputFmtCtx, &pkt); //读取一包数据
if (ret < 0)
{
av_log(NULL, AV_LOG_DEBUG, "fail to read frame,err is %s\n");
break;
}
in_stream = inputFmtCtx->streams[pkt.stream_index]; //获取流
if (pkt.stream_index >= stream_mapping_size || stream_mapping[pkt.stream_index] < 0) //如果不是音视频流或者index大小不对,释放
{
av_packet_unref(&pkt);
continue;
}
pkt.stream_index = stream_mapping[pkt.stream_index];
out_stream = outputFmtCtx->streams[pkt.stream_index]; //获取输出流
/*--同步dts和pts--*/
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
ret = av_interleaved_write_frame(outputFmtCtx, &pkt); //数据写入
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&pkt);
}
av_write_trailer(outputFmtCtx); //写入尾部
av_log(NULL, AV_LOG_DEBUG, "write trailer success\n");
end:
avformat_close_input(&inputFmtCtx);
if (outputFmtCtx && !(outputFmt->flags & AVFMT_NOFILE))
avio_closep(&outputFmtCtx->pb);
avformat_free_context(outputFmtCtx);
av_freep(&stream_mapping);
if (ret < 0 && ret != AVERROR_EOF)
{
//fprintf(stderr, "Error occurred:%s\n", av_err2str(ret));
return 1;
}
fprintf(stdout, "press enter to quit");
getchar();
return 0;
}