05 FFMPEG Qt视频播放器之音视频同步

最新推荐文章于 2024-09-28 22:36:06 发布

班公湖里洗过脚

最新推荐文章于 2024-09-28 22:36:06 发布

阅读量711

点赞数 1

分类专栏： ffmpeg 文章标签： ffmpeg

本文链接：https://blog.csdn.net/chenyijun/article/details/108721476

版权

ffmpeg 专栏收录该内容

5 篇文章 1 订阅

订阅专栏

开发环境：Windows 10， Qt 5.13.1, ffmpeg 4.2.2

上几篇介绍分别介绍了ffmpeg解码视频显示在界面上，解码音频用SDL播放。

本篇整合两个功能，使用音视频同步。

这里主要讲下声音和视频同步的步骤。

首先刚开始播放的时候通过av_gettime()获取系统主时钟，记录下来。

以后便不断调用av_gettime()获取系统时钟减去之前记录下的差值，便得到了一个视频播放了多久的实际时间。

对于视频的同步我们这样做：

从视频读取出的数据中包含一个pts的信息(每一帧图像都会带有pts的信息，pts就是播放视频的时候此图像应该显示的时间)。这样只需要使用pts和前面获取的时间进行对比，pts比实际时间大，就调用sleep函数等一等，否则就直接播放出来。这样就达到了某种意义上的同步了。

而对于音频：

从前面使用SDL的例子，其实就能够发现一个现象：我们读取音频的线程差不多就是瞬间读完放入队列的，但是音频播放速度却是正常的，并不是一下子播放完毕。因此可以看出，在音频播放上，SDL已经帮我们做了处理了，只需要将数据直接交给SDL就行了。

创建Qt的工程与前面一样，完整代码如下：

CVideoPlayer.h

#ifndef CVIDEOPLAYER_H
#define CVIDEOPLAYER_H

#include <QThread>
#include <QImage>



class CVideoPlayer: public QThread
{
    Q_OBJECT

public:
    CVideoPlayer();
    void videoDecode();

protected:
    void run();

signals:
    void signalGetOneFrame(QImage image);
    void signalDecodeError(int error);
};

#endif // CVIDEOPLAYER_H

CVideoPlayer.cpp

#include "CVideoPlayer.h"
#include <stdio.h>
#include <QDebug>

extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavdevice/avdevice.h"

#include <libavutil/time.h>
#include "libavutil/pixfmt.h"
#include "libswresample/swresample.h"

#include <SDL.h>//这个不要放在头文件包含中，否则报main函数冲突
//#include <SDL_audio.h>
//#include <SDL_types.h>
//#include <SDL_name.h>
//#include <SDL_main.h>
//#include <SDL_config.h>
}

typedef struct PacketQueue {
    AVPacketList *first_pkt, *last_pkt;
    int nb_packets;
    int size;
    SDL_mutex *mutex;
    SDL_cond *cond;
} PacketQueue;

#define VIDEO_PICTURE_QUEUE_SIZE 1
#define AVCODEC_MAX_AUDIO_FRAME_SIZE 192000 // 1 second of 48khz 32bit audio

typedef struct VideoState {
    AVCodecContext *aCodecCtx; //音频解码器
    AVFrame *audioFrame;// 解码音频过程中的使用缓存
    PacketQueue *audioq;
    double video_clock; ///<pts of last decoded frame / predicted pts of next decoded frame
    AVStream *video_st;
} VideoState;

#define SDL_AUDIO_BUFFER_SIZE 1024

VideoState mVideoState; //用来 传递给 SDL音频回调函数的数据

void packet_queue_init(PacketQueue *q) {
    memset(q, 0, sizeof(PacketQueue));
    q->mutex = SDL_CreateMutex();
    q->cond = SDL_CreateCond();
}

int packet_queue_put(PacketQueue *q, AVPacket *pkt) {

    AVPacketList *pkt1;
    if (av_dup_packet(pkt) < 0) {
        return -1;
    }
    pkt1 = (AVPacketList*)av_malloc(sizeof(AVPacketList));
    if (!pkt1)
        return -1;
    pkt1->pkt = *pkt;
    pkt1->next = NULL;

    SDL_LockMutex(q->mutex);

    if (!q->last_pkt)
        q->first_pkt = pkt1;
    else
        q->last_pkt->next = pkt1;
    q->last_pkt = pkt1;
    q->nb_packets++;
    q->size += pkt1->pkt.size;
    SDL_CondSignal(q->cond);

    SDL_UnlockMutex(q->mutex);
    return 0;
}
//http://blog.yundiantech.com/?log=blog&id=9
static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block) {
    AVPacketList *pkt1;
    int ret;

    SDL_LockMutex(q->mutex);

    for (;;) {

        pkt1 = q->first_pkt;
        if (pkt1) {
            q->first_pkt = pkt1->next;
            if (!q->first_pkt)
                q->last_pkt = NULL;
            q->nb_packets--;
            q->size -= pkt1->pkt.size;
            *pkt = pkt1->pkt;
            av_free(pkt1);
            ret = 1;
            break;
        } else if (!block) {
            ret = 0;
            break;
        } else {
            SDL_CondWait(q->cond, q->mutex);
        }
    }
    SDL_UnlockMutex(q->mutex);
    return ret;
}
int audio_decode_frame(VideoState *is, uint8_t *audio_buf, int buf_size)
{

    static AVPacket pkt;
    static uint8_t *audio_pkt_data = NULL;
    static int audio_pkt_size = 0;
    int len1, data_size;

    AVCodecContext *aCodecCtx = is->aCodecCtx;
    AVFrame *audioFrame = is->audioFrame;
    PacketQueue *audioq = is->audioq;

    for(;;)
    {
        if(packet_queue_get(audioq, &pkt, 1) < 0)
        {
            return -1;
        }
        audio_pkt_data = pkt.data;
        audio_pkt_size = pkt.size;
        while(audio_pkt_size > 0)
        {
            int got_picture;

            int ret = avcodec_decode_audio4( aCodecCtx, audioFrame, &got_picture, &pkt);
            if( ret < 0 ) {
                printf("Error in decoding audio frame.\n");
                exit(0);
            }

            if( got_picture ) {
                int in_samples = audioFrame->nb_samples;
                short *sample_buffer = (short*)malloc(audioFrame->nb_samples * 2 * 2);
                memset(sample_buffer, 0, audioFrame->nb_samples * 4);

                int i=0;
                float *inputChannel0 = (float*)(audioFrame->extended_data[0]);

                // Mono
                if( audioFrame->channels == 1 ) {
                    for( i=0; i<in_samples; i++ ) {
                        float sample = *inputChannel0++;
                        if( sample < -1.0f ) {
                            sample = -1.0f;
                        } else if( sample > 1.0f ) {
                            sample = 1.0f;
                        }

                        sample_buffer[i] = (int16_t)(sample * 32767.0f);
                    }
                } else { // Stereo
                    float* inputChannel1 = (float*)(audioFrame->extended_data[1]);
                    for( i=0; i<in_samples; i++) {
                        sample_buffer[i*2] = (int16_t)((*inputChannel0++) * 32767.0f);
                        sample_buffer[i*2+1] = (int16_t)((*inputChannel1++) * 32767.0f);
                    }
                }
//                fwrite(sample_buffer, 2, in_samples*2, pcmOutFp);
                memcpy(audio_buf,sample_buffer,in_samples*4);
                free(sample_buffer);
            }

            audio_pkt_size -= ret;

            if (audioFrame->nb_samples <= 0)
            {
                continue;
            }

            data_size = audioFrame->nb_samples * 4;

            return data_size;
        }
        if(pkt.data)
            av_free_packet(&pkt);
   }
}
void audio_callback(void *userdata, Uint8 *stream, int len)
{
//    AVCodecContext *aCodecCtx = (AVCodecContext *) userdata;
    VideoState *is = (VideoState *) userdata;

    int len1, audio_data_size;

    static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
    static unsigned int audio_buf_size = 0;
    static unsigned int audio_buf_index = 0;

    /*   len是由SDL传入的SDL缓冲区的大小，如果这个缓冲未满，我们就一直往里填充数据 */
    while (len > 0) {
        /*  audio_buf_index 和 audio_buf_size 标示我们自己用来放置解码出来的数据的缓冲区，*/
        /*   这些数据待copy到SDL缓冲区， 当audio_buf_index >= audio_buf_size的时候意味着我*/
        /*   们的缓冲为空，没有数据可供copy，这时候需要调用audio_decode_frame来解码出更
         /*   多的桢数据 */

        if (audio_buf_index >= audio_buf_size) {
            audio_data_size = audio_decode_frame(is, audio_buf,sizeof(audio_buf));
            /* audio_data_size < 0 标示没能解码出数据，我们默认播放静音 */
            if (audio_data_size < 0) {
                /* silence */
                audio_buf_size = 1024;
                /* 清零，静音 */
                memset(audio_buf, 0, audio_buf_size);
            } else {
                audio_buf_size = audio_data_size;
            }
            audio_buf_index = 0;
        }
        /*  查看stream可用空间，决定一次copy多少数据，剩下的下次继续copy */
        len1 = audio_buf_size - audio_buf_index;
        if (len1 > len) {
            len1 = len;
        }

        memcpy(stream, (uint8_t *) audio_buf + audio_buf_index, len1);
        len -= len1;
        stream += len1;
        audio_buf_index += len1;
    }
}

static double synchronize_video(VideoState *is, AVFrame *src_frame, double pts) {

    double frame_delay;

    if (pts != 0) {
        /* if we have pts, set video clock to it */
        is->video_clock = pts;
    } else {
        /* if we aren't given a pts, set it to the clock */
        pts = is->video_clock;
    }
    /* update the video clock */
    frame_delay = av_q2d(is->video_st->codec->time_base);
    /* if we are repeating a frame, adjust clock accordingly */
    frame_delay += src_frame->repeat_pict * (frame_delay * 0.5);
    is->video_clock += frame_delay;
    return pts;
}
CVideoPlayer::CVideoPlayer()
{

}

void CVideoPlayer::run()
{
    videoDecode();
}

void CVideoPlayer::videoDecode()
{
    char *filePath = (char*)"F:\\github\\QtPlayLearn\\win\\mp4\\lasa.mp4";
    AVFormatContext *pFormatCtx;
    AVCodecContext *pCodecCtx;
    AVCodec *pCodec;
    AVFrame *pFrame, *pFrameRGB;
    AVPacket *packet;
    uint8_t *outBuffer;

    AVCodecContext *aCodecCtx;
    AVCodec *aCodec;
    static struct SwsContext *img_convert_ctx;
    unsigned int i;
    int audioStream ,videoStream, numBytes;
    int ret, got_picture;
    av_register_all();//初始化ffmpeg 调用了这个才能正常适用编码器和解码器
    if (SDL_Init(SDL_INIT_AUDIO)) {
        fprintf(stderr,"Could not initialize SDL - %s. \n", SDL_GetError());
        exit(1);
    }
    //Allocate an AVFormatContext.
    pFormatCtx = avformat_alloc_context();
    if(0 != avformat_open_input(&pFormatCtx, filePath, nullptr, nullptr))
    {
        emit signalDecodeError(-1);
        return;
    }

    if(avformat_find_stream_info(pFormatCtx, nullptr))
    {
        emit signalDecodeError(-2);
        return;
    }
    videoStream = -1;
    audioStream = -1;
    //循环查找视频中包含的流信息，直到找到视频类型的流
    //便将其记录下来 保存到videoStream变量中
    //这里我们现在只处理视频流  音频流先不管他
    for(i = 0; i < pFormatCtx->nb_streams; ++i)
    {
        qDebug() << "pFormatCtx->streams[" << i << "]->codec->codec_type = " << pFormatCtx->streams[i]->codec->codec_type << endl;
        //0:视频类型 1:音频类型
        if(pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
        {
            videoStream = i;
        }
        if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO  && audioStream < 0)
        {
            audioStream = i;
        }
    }
    qDebug() << "videoStream===========" << videoStream << "  pFormatCtx->nb_streams==" << pFormatCtx->nb_streams << endl;
    //如果videoStream为-1 说明没有找到视频流
    if (videoStream == -1) {
        printf("Didn't find a video stream.\n");
        return;
    }
    if (audioStream == -1) {
        printf("Didn't find a audio stream.\n");
        return;
    }
    ///查找音频解码器
    aCodecCtx = pFormatCtx->streams[audioStream]->codec;
    aCodec = avcodec_find_decoder(aCodecCtx->codec_id);

    if (aCodec == NULL) {
        printf("ACodec not found.\n");
        return;
    }
    ///打开音频解码器
    if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0) {
        printf("Could not open audio codec.\n");
        return;
    }
    //初始化音频队列
    PacketQueue *audioq = new PacketQueue;
    packet_queue_init(audioq);
    // 分配解码过程中的使用缓存
    //AVFrame* audioFrame = avcodec_alloc_frame();
    AVFrame* audioFrame = av_frame_alloc(); //ffmpeg v4.2.2
    mVideoState.aCodecCtx = aCodecCtx;
    mVideoState.audioq = audioq;
    mVideoState.audioFrame = audioFrame;
    ///  打开SDL播放设备 - 开始
    SDL_LockAudio();
    SDL_AudioSpec spec;
    SDL_AudioSpec wanted_spec;
    wanted_spec.freq = aCodecCtx->sample_rate;
    wanted_spec.format = AUDIO_S16SYS;
    wanted_spec.channels = aCodecCtx->channels;
    wanted_spec.silence = 0;
    wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
    wanted_spec.callback = audio_callback;
    wanted_spec.userdata = &mVideoState;
    if(SDL_OpenAudio(&wanted_spec, &spec) < 0)
    {
        fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
        return;
    }
    SDL_UnlockAudio();
    SDL_PauseAudio(0);
    ///  打开SDL播放设备 - 结束
    ///查找视频解码器
    pCodecCtx = pFormatCtx->streams[videoStream]->codec;
    qDebug() << "pCodecCtx->codec_id===========" << pCodecCtx->codec_id << endl;
    //测试时这个值为27，查到枚举值对应的是AV_CODEC_ID_H264 ，即是H264压缩格式的文件。
    pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
    if(nullptr == pCodec)
    {
        printf("PCodec not found.\n");
        emit signalDecodeError(-4);
        return;
    }

    //打开视频解码器s
    if(avcodec_open2(pCodecCtx, pCodec, nullptr) < 0)
    {
        printf("Could not open video codec.\n");
        emit signalDecodeError(-5);
        return;
    }

    mVideoState.video_st = pFormatCtx->streams[videoStream];

    pFrame = av_frame_alloc();
    pFrameRGB = av_frame_alloc();
    img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, \
                                     pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, \
                                     AV_PIX_FMT_RGB32, SWS_BICUBIC, nullptr, nullptr, nullptr);

    numBytes = avpicture_get_size(AV_PIX_FMT_RGB32, pCodecCtx->width, pCodecCtx->height);
    //av_image_get_buffer_size();
    outBuffer = (uint8_t *)av_malloc(numBytes * sizeof(uint8_t));
    avpicture_fill((AVPicture *)pFrameRGB, outBuffer, AV_PIX_FMT_RGB32, pCodecCtx->width, pCodecCtx->height);
    int y_size = pCodecCtx->width * pCodecCtx->height;
    packet = (AVPacket *)malloc(sizeof(AVPacket));//分配一个packet
    av_new_packet(packet, y_size);//分配packet的数据
    av_dump_format(pFormatCtx, 0,  filePath, 0);//输出视频信息

    int64_t start_time = av_gettime();
    int64_t pts = 0; //当前视频的pts
    int index = 0;
    while (1)
    {
        if(av_read_frame(pFormatCtx, packet) < 0)
        {
            qDebug() << "index===============" << index;
            break;//这里认为视频读取完了
        }

        int64_t realTime = av_gettime() - start_time; //主时钟时间
        while(pts > realTime)
        {
            SDL_Delay(10);
            realTime = av_gettime() - start_time; //主时钟时间
        }
        if(packet->stream_index == videoStream)
        {
            ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, packet);
            if(ret < 0)
            {
                emit signalDecodeError(-6);
                return;
            }
            //add
            if (packet->dts == AV_NOPTS_VALUE && pFrame->opaque&& *(uint64_t*) pFrame->opaque != AV_NOPTS_VALUE)
            {
                pts = *(uint64_t *) pFrame->opaque;
            }
            else if (packet->dts != AV_NOPTS_VALUE)
            {
                pts = packet->dts;
            }
            else
            {
                pts = 0;
            }

            pts *= 1000000 * av_q2d(mVideoState.video_st->time_base);
            pts = synchronize_video(&mVideoState, pFrame, pts);
            //--------
            if(got_picture)
            {
                sws_scale(img_convert_ctx, (uint8_t const * const *)pFrame->data,
                          pFrame->linesize, 0, pCodecCtx->height, pFrameRGB->data, pFrameRGB->linesize);
                ++index;
                //把这个RGB数据 用QImage加载
                QImage tempImage((uchar*)outBuffer, pCodecCtx->width, pCodecCtx->height, QImage::Format_RGB32);
                QImage image = tempImage.copy();//把图像复制一份 传递给界面显示
                qDebug() << "image.width==" << image.width() << "image.height==" << image.height();
                emit signalGetOneFrame(image);
            }

            av_free_packet(packet);
        }
        else if( packet->stream_index == audioStream )
        {
            packet_queue_put(mVideoState.audioq, packet);
            //这里我们将数据存入队列 因此不调用 av_free_packet 释放
        }
        else
        {
            // Free the packet that was allocated by av_read_frame
            av_free_packet(packet);
        }
    }
    av_free(outBuffer);
    av_free(pFrameRGB);
    avcodec_close(pCodecCtx);
    avformat_close_input(&pFormatCtx);

}

mainwindow.h

#ifndef MAINWINDOW_H
#define MAINWINDOW_H

#include <QMainWindow>
#include "CVideoPlayer.h"
#include <QImage>

QT_BEGIN_NAMESPACE
namespace Ui { class MainWindow; }
QT_END_NAMESPACE

class MainWindow : public QMainWindow
{
    Q_OBJECT

public:
    MainWindow(QWidget *parent = nullptr);
    ~MainWindow();
    void initView();
    void initData();
    void showVideo();

public slots:
    void slotDecodeError(int error);
    void slotGetOneFrame(QImage image);

protected:
    void paintEvent(QPaintEvent *event);

private:
    Ui::MainWindow *ui;
    CVideoPlayer *m_pVideoPlayer;
    QImage m_Image;
};
#endif // MAINWINDOW_H

mainwindow.cpp

#include "mainwindow.h"
#include "ui_mainwindow.h"
#include <QDebug>
#include <QPainter>
#include <QFileDialog>


MainWindow::MainWindow(QWidget *parent)
    : QMainWindow(parent)
    , ui(new Ui::MainWindow)
{
    ui->setupUi(this);
    initView();
    initData();
    showVideo();
}

MainWindow::~MainWindow()
{
    delete ui;
}


void MainWindow::initView()
{
    //this->setStyleSheet(QString::fromUtf8("background-color: rgb(255, 255, 255);"));
}

void MainWindow::initData()
{
    //av_register_all();//这个函数已被弃用
}


void MainWindow::showVideo()
{
    m_pVideoPlayer = new CVideoPlayer();
    connect(m_pVideoPlayer, SIGNAL(signalDecodeError(int)), this, SLOT(slotDecodeError(int)));
    connect(m_pVideoPlayer, SIGNAL(signalGetOneFrame(QImage)), this, SLOT(slotGetOneFrame(QImage)));
    m_pVideoPlayer->start();
}

void MainWindow::slotDecodeError(int error)
{
    qDebug() << "slotDecodeError======error====" << error;
}

void MainWindow::slotGetOneFrame(QImage image)
{
    m_Image = image;
    update();//调用update将执行 paintEvent函数
}

void MainWindow::paintEvent(QPaintEvent *)
{

    QPainter painter(this);
    painter.setBrush(Qt::black);
    painter.drawRect(0, 0, this->width(), this->height());//先画成黑色
    if(m_Image.size().width() <= 0)
        return;
    //将图像按比例缩放成和窗口一样大小
    QImage img = m_Image.scaled(this->size(), Qt::KeepAspectRatio);
    int x = this->width() - img.width();
    int y = this->height() - img.height();
    x /=  2;
    y /= 2;
    painter.drawImage(QPoint(x, y), img);
}

main.cpp

#include "mainwindow.h"

#include <QApplication>

int main(int argc, char *argv[])
{
    QApplication a(argc, argv);
    MainWindow w;
    w.show();
    return a.exec();
}

运行效果如下：