最近在做Android上thumbnail的抽取功能,没有使用原生的Android系统自带功能而基于ffmpeg提供的API, 主要涉及一下几方面:
1. 解码视频数据 得到 YUV数据,保存在AVFrame中。
2. 使用sws_scale 接口库进行YUV数据格式转换及缩放,由于Android默认保存thumbnail使用的是RGB565格式,因此需要把YUV420p转换为RGB565,并缩放到指定的thumbnail 图像大小。
3. 对解码出来的YUV数据进行过滤,去除黑图, 采用像素Y值得平均值检测方式。
相关的示例代码程序已经放到github上了,有兴趣的朋友可以参考:
https://github.com/coloriy/thumbnail_demo
下面是具体的示例代码,可以供大家参考;
0. 初始化ffmpeg
int initFFmpegContext()
{
avcodec_register_all();
av_register_all();
return 0;
}
1. 打开文件并获取视频相关信息
int setDataSource(const char* url)
{
int ret = -1;
if (m_pFormatContext)
{
delete m_pFormatContext;
m_pFormatContext = NULL;
}
m_pFormatContext = avformat_alloc_context();
if (!m_pFormatContext)
{
return -1;
}
ret = avformat_open_input(&m_pFormatContext, url, NULL, NULL);
if (ret != 0)
{
delete m_pFormatContext;
return ret;
}
ret = avformat_find_stream_info(m_pFormatContext, NULL);
if (ret != 0)
{
delete m_pFormatContext;
return ret;
}
m_nStreamIndex[AVMEDIA_TYPE_VIDEO] = av_find_best_stream(m_pFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
m_nStreamIndex[AVMEDIA_TYPE_AUDIO] = av_find_best_stream(m_pFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
return ret;
}
2. 打开解码器
int openDecoder()
{
int ret = -1;
m_pCodecContext = m_pFormatContext->streams[m_nStreamIndex[AVMEDIA_TYPE_VIDEO]]->codec;
if (m_pCodecContext)
{
m_pVideoCodec = avcodec_find_decoder(m_pCodecContext->codec_id);
ret = avcodec_open2(m_pCodecContext, m_pVideoCodec, NULL);
if (ret != 0)
{
return ret;
}
avcodec_flush_buffers(m_pCodecContext);
}
return ret;
}
3. 解码符合条件的YUV图像数据(该过程去除黑图)
int decodeOneFrame(AVFrame* pFrame)
{
int ret = 0;
bool frame_found = false;
int decoded_frame_count = 0;
AVPacket pkt;
do
{
int got_frame = 0;
ret = av_read_frame(m_pFormatContext, &pkt);
if (ret < 0)
{
break;
}
if (pkt.stream_index == m_nStreamIndex[AVMEDIA_TYPE_VIDEO])
{
ret = avcodec_decode_video2(m_pCodecContext, pFrame, &got_frame, &pkt);
if (got_frame)
{
decoded_frame_count++;
//skip black and white pitures
uint32_t y_value = 0;
uint32_t y_half = 0;
uint32_t y_count = 0;
int pixel_count = pFrame->width * pFrame->height;
bool bHalf = false;
for (int i = 0; i < pixel_count; i+=3)
{
uint8_t y_temp = (uint8_t)(*(uint8_t*)((uint8_t*)(pFrame->data[0]) + i));
y_value += y_temp;
y_count++;
if (!bHalf && i > pixel_count / 6)
{
y_half = y_value / y_count;
bHalf = true;
}
}
y_value /= y_count;
if (y_half == y_value)
{
printf("decoded frame count = %d y_half=%d == y_value=%d, skip this frame!\n", decoded_frame_count, y_half, y_value);
continue;
}
if (y_value < BRIGHTNESS_VALUE && y_value > DARKNESS_VALUE)
{
frame_found = true;
printf("frame_found = true -----------------------decoded frame count = %d\n", decoded_frame_count);
}
}
#ifdef SAVE_YUV_FRAME
char szName[128];
sprintf(szName, "D:\\test_%d.yuv", frame_count);
// save the yuv
FILE *pFile = fopen(szName, "ab");
if (pFile)
{
fwrite(pFrame->data[0], 1, pFrame->width * pFrame->height, pFile);
fwrite(pFrame->data[1], 1, pFrame->width * pFrame->height * 1 / 4, pFile);
fwrite(pFrame->data[2], 1, pFrame->width * pFrame->height * 1 / 4, pFile);
fclose(pFile);
}
#endif
}
av_free_packet(&pkt);
} while ((!frame_found) && (ret >= 0));
av_free_packet(&pkt);
return ret;
}
void closeDecoder()
{
if (m_pVideoCodec)
{
avcodec_close(m_pCodecContext);
m_pCodecContext = NULL;
}
}
4. YUV图像数据转换为RGB565,并缩放
int getThumbnail(AVFrame* pInputFrame, AVFrame* pOutputFrame, int desW, int desH)
{
if (pInputFrame == NULL || pOutputFrame == NULL)
{
return -1;
}
SwsContext* pSwsContext = NULL;
pSwsContext = sws_getCachedContext(pSwsContext, pInputFrame->width, pInputFrame->height, (AVPixelFormat)pInputFrame->format,
desW, desH, AV_PIX_FMT_RGB565, SWS_BICUBIC, NULL, NULL, NULL);
if (pSwsContext == NULL)
{
return -1;
}
m_pThumbFrame->width = desW;
m_pThumbFrame->height = desH;
m_pThumbFrame->format = AV_PIX_FMT_RGB565;
av_frame_get_buffer(m_pThumbFrame, 16);
sws_scale(pSwsContext, pInputFrame->data, pInputFrame->linesize, 0, pInputFrame->height, m_pThumbFrame->data, m_pThumbFrame->linesize);
sws_freeContext(pSwsContext);
return 0;
}
5. 综合调用上述函数接口,并保存RGB565数据到文件。
int getFrameAt(int64_t timeUs, int width, int height)
{
int ret = -1;
AVFrame* pFrame = NULL;
ret = avformat_seek_file(m_pFormatContext, -1, INT16_MIN, timeUs, INT16_MAX, 0);
pFrame = av_frame_alloc();
m_pThumbFrame = av_frame_alloc();
ret = openDecoder();
if (ret != 0)
{
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
}
ret = decodeOneFrame(pFrame);
if (ret < 0)
{
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
}
ret = getThumbnail(pFrame, m_pThumbFrame, width, height);
if (ret < 0)
{
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
}
// save the rgb565
FILE *pFile = fopen(strThumbFileName, "ab");
if (pFile)
{
fwrite(m_pThumbFrame->data[0], 1, m_pThumbFrame->width * m_pThumbFrame->height * 2, pFile);
fclose(pFile);
}
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
return ret;
}
6. 测试程序main函数及全局变量定义
#include "stdafx.h"
extern "C"
{
#include "libavformat\avformat.h"
#include "libswscale\swscale.h"
}
AVFormatContext* m_pFormatContext = NULL;
AVCodecContext* m_pCodecContext = NULL;
int m_nStreamIndex[AVMEDIA_TYPE_NB] = {-1};
AVCodec* m_pVideoCodec = NULL;
AVFrame* m_pAVFrame = NULL;
AVFrame* m_pThumbFrame = NULL;
const char* strInputFileName[] = {
"D:\\testcontent\\MP4_H.264_Base_L4.0_1080P_7Mbps_25fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_BP_L1.3_480P_1.1Mbps_29.970fps_AMR_NB.mp4",
"D:\\testcontent\\MP4_H.264_BP_L2.1_480x270_500Kbps_23.959fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_BP_L2.1_512x288_1.2Mbps_15fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_BP_L4.1_CIF_2Mbps_25fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_MP_L4.0_1080P_762Kbps_30fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_MP_L3.1_720P_2Mbps_30fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_MP_L2.2_640x320_463Kbps_25fps_AAC_LC.mp4",
"D:\\testcontent\\MP4_H.264_MP_L3.1_720P_1Mbps_29.585fps_AAC_LC.mp4"
};
const char strThumbFileName[] = "D:\\test_thumb.rgb";
#define THUMB_WIDTH 640
#define THUMB_HEIGHT 480
#define BRIGHTNESS_VALUE 0xF0
#define DARKNESS_VALUE 0x10
main 函数
int _tmain(int argc, _TCHAR* argv[])
{
int ret = -1;
initFFmpegContext();
int file_count = sizeof(strInputFileName) / sizeof(strInputFileName[0]);
for (int i = 0; i < file_count; i++)
{
const char* pFileName = strInputFileName[i];
ret = setDataSource(pFileName);
getFrameAt(-1, THUMB_WIDTH, THUMB_HEIGHT);
}
//pause
printf("finished, pause ....\n");
getchar();
return 0;
}
保存的RGB565图像数据使用ffmpeg命令转换为YUV420,然后使用YUVplayer播放显示, 来验证是否正确。
ffmpeg转命令:
D:\work\opensource\ffmpeg\bin>ffmpeg.exe -s 640x480 -pix_fmt rgb565le -i D:\test_thumb.rgb -s 640x480 -pix_fmt yuv420p D:\yuv_420p.yuv
使用VS2013 编译遇到问题:
1. fopen 编译错误问题:
在配置中添加:
C\C++->preprocessor 中添加宏定义: _CRT_SECURE_NO_WARNINGS;
2. release 编译出现错误
error LNK2026: module unsafe for SAFESEH image
去掉属性设置:
最终结果:
Thumbnail图片显示如下: