FFmpeg源码分析：图像常用操作

FFmpeg有封装图像的常用操作，位于libavutil/imgutils.c，包括图像拷贝、图像填充、获取图像大小、分配图像、检测图像宽高比是否有效。在视频图像缩放、像素格式转换、视频截图保存等操作，经常需要用到图像操作方法。

 一、获取图像大小

1、av_image_get_linesize

根据图像宽与像素格式，获取一行图像大小：

int av_image_get_linesize(enum AVPixelFormat pix_fmt, int width, int plane)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    int max_step     [4];       /* max pixel step for each plane */
    int max_step_comp[4];       /* the component for each plane which has the max pixel step */

    if (!desc || desc->flags & AV_PIX_FMT_FLAG_HWACCEL)
        return AVERROR(EINVAL);

    av_image_fill_max_pixsteps(max_step, max_step_comp, desc);
    return image_get_linesize(width, plane, max_step[plane], max_step_comp[plane], desc);
}

 2、av_image_fill_linesizes

在av_image_get_linesize()基础上，获取每行图像大小：

int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
{
    int i, ret;
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    int max_step     [4];       /* max pixel step for each plane */
    int max_step_comp[4];       /* the component for each plane which has the max pixel step */

    memset(linesizes, 0, 4*sizeof(linesizes[0]));

    if (!desc || desc->flags & AV_PIX_FMT_FLAG_HWACCEL)
        return AVERROR(EINVAL);

    av_image_fill_max_pixsteps(max_step, max_step_comp, desc);
    for (i = 0; i < 4; i++) {
        if ((ret = image_get_linesize(width, i, max_step[i], max_step_comp[i], desc)) < 0)
            return ret;
        linesizes[i] = ret;
    }

    return 0;
}

3、av_image_fill_plane_sizes

根据图像的高、每行大小、像素格式，计算每个图像平面的大小：

int av_image_fill_plane_sizes(size_t sizes[4], enum AVPixelFormat pix_fmt,
                              int height, const ptrdiff_t linesizes[4])
{
    int i, has_plane[4] = { 0 };

    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    memset(sizes    , 0, sizeof(sizes[0])*4);

    if (!desc || desc->flags & AV_PIX_FMT_FLAG_HWACCEL)
        return AVERROR(EINVAL);

    if (linesizes[0] > SIZE_MAX / height)
        return AVERROR(EINVAL);
    sizes[0] = linesizes[0] * (size_t)height;

    if (desc->flags & AV_PIX_FMT_FLAG_PAL ||
        desc->flags & FF_PSEUDOPAL) {
        sizes[1] = 256 * 4; /* palette is stored here as 256 32 bits words */
        return 0;
    }

    for (i = 0; i < 4; i++)
        has_plane[desc->comp[i].plane] = 1;

    for (i = 1; i < 4 && has_plane[i]; i++) {
        int h, s = (i == 1 || i == 2) ? desc->log2_chroma_h : 0;
        h = (height + (1 << s) - 1) >> s;
        if (linesizes[i] > SIZE_MAX / h)
            return AVERROR(EINVAL);
        sizes[i] = (size_t)h * linesizes[i];
    }

    return 0;
}

4、av_image_get_buffer_size

根据图像宽高与像素格式，获取图像大小，支持配置是否对齐模式：

int av_image_get_buffer_size(enum AVPixelFormat pix_fmt,
                             int width, int height, int align)
{
    int ret, i;
    int linesize[4];
    ptrdiff_t aligned_linesize[4];
    size_t sizes[4];
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    if (!desc)
        return AVERROR(EINVAL);

    ret = av_image_check_size(width, height, 0, NULL);
    if (ret < 0)
        return ret;

    if (desc->flags & FF_PSEUDOPAL)
        return FFALIGN(width, align) * height;

    ret = av_image_fill_linesizes(linesize, pix_fmt, width);
    if (ret < 0)
        return ret;

    for (i = 0; i < 4; i++)
        aligned_linesize[i] = FFALIGN(linesize[i], align);

    ret = av_image_fill_plane_sizes(sizes, pix_fmt, height, aligned_linesize);
    if (ret < 0)
        return ret;

    ret = 0;
    for (i = 0; i < 4; i++) {
        if (sizes[i] > INT_MAX - ret)
            return AVERROR(EINVAL);
        ret += sizes[i];
    }
    return ret;
}

二、图像填充

1、av_image_fill_arrays

使用源图像，结合图像宽、高、像素格式，填充目标图像数据：

int av_image_fill_arrays(uint8_t *dst_data[4], int dst_linesize[4],
                         const uint8_t *src, enum AVPixelFormat pix_fmt,
                         int width, int height, int align)
{
    int ret, i;

    ret = av_image_check_size(width, height, 0, NULL);
    if (ret < 0)
        return ret;

    ret = av_image_fill_linesizes(dst_linesize, pix_fmt, width);
    if (ret < 0)
        return ret;

    for (i = 0; i < 4; i++)
        dst_linesize[i] = FFALIGN(dst_linesize[i], align);

    return av_image_fill_pointers(dst_data, pix_fmt, height, (uint8_t *)src, dst_linesize);
}

2、av_image_fill_black

填充图像为黑色，如果有透明通道，就重置为不透明：

int av_image_fill_black(uint8_t *dst_data[4], const ptrdiff_t dst_linesize[4],
                        enum AVPixelFormat pix_fmt, enum AVColorRange range,
                        int width, int height)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    int nb_planes = av_pix_fmt_count_planes(pix_fmt);
    // A pixel or a group of pixels on each plane, with a value that represents black.
    uint8_t clear_block[4][MAX_BLOCK_SIZE] = {{0}}; // clear padding with 0
    int clear_block_size[4] = {0};
    ptrdiff_t plane_line_bytes[4] = {0};
    int rgb, limited;
    int plane, c;

    if (!desc || nb_planes < 1 || nb_planes > 4 || desc->flags & AV_PIX_FMT_FLAG_HWACCEL)
        return AVERROR(EINVAL);

    rgb = !!(desc->flags & AV_PIX_FMT_FLAG_RGB);
    limited = !rgb && range != AVCOL_RANGE_JPEG;

    if (desc->flags & AV_PIX_FMT_FLAG_BITSTREAM) {
        ptrdiff_t bytewidth = av_image_get_linesize(pix_fmt, width, 0);
        uint8_t *data;
        int mono = pix_fmt == AV_PIX_FMT_MONOWHITE || pix_fmt == AV_PIX_FMT_MONOBLACK;
        int fill = pix_fmt == AV_PIX_FMT_MONOWHITE ? 0xFF : 0;
        if (nb_planes != 1 || !(rgb || mono) || bytewidth < 1)
            return AVERROR(EINVAL);

        if (!dst_data)
            return 0;

        data = dst_data[0];

        // (Bitstream + alpha will be handled incorrectly - it'll remain transparent.)
        for (;height > 0; height--) {
            memset(data, fill, bytewidth);
            data += dst_linesize[0];
        }
        return 0;
    }

    for (c = 0; c < desc->nb_components; c++) {
        const AVComponentDescriptor comp = desc->comp[c];

        // We try to operate on entire non-subsampled pixel groups (for
        // AV_PIX_FMT_UYVY422 this would mean two consecutive pixels).
        clear_block_size[comp.plane] = FFMAX(clear_block_size[comp.plane], comp.step);

        if (clear_block_size[comp.plane] > MAX_BLOCK_SIZE)
            return AVERROR(EINVAL);
    }

    // Create a byte array for clearing 1 pixel (sometimes several pixels).
    for (c = 0; c < desc->nb_components; c++) {
        const AVComponentDescriptor comp = desc->comp[c];
        // (Multiple pixels happen e.g. with AV_PIX_FMT_UYVY422.)
        int w = clear_block_size[comp.plane] / comp.step;
        uint8_t *c_data[4];
        const int c_linesize[4] = {0};
        uint16_t src_array[MAX_BLOCK_SIZE];
        uint16_t src = 0;
        int x;

        if (comp.depth > 16)
            return AVERROR(EINVAL);
        if (!rgb && comp.depth < 8)
            return AVERROR(EINVAL);
        if (w < 1)
            return AVERROR(EINVAL);

        if (c == 0 && limited) {
            src = 16 << (comp.depth - 8);
        } else if ((c == 1 || c == 2) && !rgb) {
            src = 128 << (comp.depth - 8);
        } else if (c == 3) {
            // (Assume even limited YUV uses full range alpha.)
            src = (1 << comp.depth) - 1;
        }

        for (x = 0; x < w; x++)
            src_array[x] = src;

        for (x = 0; x < 4; x++)
            c_data[x] = &clear_block[x][0];

        av_write_image_line(src_array, c_data, c_linesize, desc, 0, 0, c, w);
    }

    for (plane = 0; plane < nb_planes; plane++) {
        plane_line_bytes[plane] = av_image_get_linesize(pix_fmt, width, plane);
        if (plane_line_bytes[plane] < 0)
            return AVERROR(EINVAL);
    }

    if (!dst_data)
        return 0;

    for (plane = 0; plane < nb_planes; plane++) {
        size_t bytewidth = plane_line_bytes[plane];
        uint8_t *data = dst_data[plane];
        int chroma_div = plane == 1 || plane == 2 ? desc->log2_chroma_h : 0;
        int plane_h = ((height + ( 1 << chroma_div) - 1)) >> chroma_div;

        for (; plane_h > 0; plane_h--) {
            memset_bytes(data, bytewidth, &clear_block[plane][0], clear_block_size[plane]);
            data += dst_linesize[plane];
        }
    }

    return 0;
}

三、图像拷贝

1、av_image_copy_plane

拷贝图像平面的像素数据：

static void image_copy_plane(uint8_t       *dst, ptrdiff_t dst_linesize,
                             const uint8_t *src, ptrdiff_t src_linesize,
                             ptrdiff_t bytewidth, int height)
{
    if (!dst || !src)
        return;
    av_assert0(FFABS(src_linesize) >= bytewidth);
    av_assert0(FFABS(dst_linesize) >= bytewidth);
    // 逐行拷贝
    for (;height > 0; height--) {
        memcpy(dst, src, bytewidth);
        dst += dst_linesize;
        src += src_linesize;
    }
}

void av_image_copy_plane(uint8_t       *dst, int dst_linesize,
                         const uint8_t *src, int src_linesize,
                         int bytewidth, int height)
{
    image_copy_plane(dst, dst_linesize, src, src_linesize, bytewidth, height);
}

2、av_image_copy

根据图像的宽、高、像素格式，进行图像拷贝：

void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4],
                   const uint8_t *src_data[4], const int src_linesizes[4],
                   enum AVPixelFormat pix_fmt, int width, int height)
{
    ptrdiff_t dst_linesizes1[4], src_linesizes1[4];
    int i;

    for (i = 0; i < 4; i++) {
        dst_linesizes1[i] = dst_linesizes[i];
        src_linesizes1[i] = src_linesizes[i];
    }

    image_copy(dst_data, dst_linesizes1, src_data, src_linesizes1, pix_fmt,
               width, height, image_copy_plane);
}

3、av_image_copy_to_buffer

把图像数据拷贝到指定缓冲区：

int av_image_copy_to_buffer(uint8_t *dst, int dst_size,
                            const uint8_t * const src_data[4],
                            const int src_linesize[4],
                            enum AVPixelFormat pix_fmt,
                            int width, int height, int align)
{
    int i, j, nb_planes = 0, linesize[4];
    int size = av_image_get_buffer_size(pix_fmt, width, height, align);
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    int ret;

    if (size > dst_size || size < 0 || !desc)
        return AVERROR(EINVAL);
    // 计算plane平面数量
    for (i = 0; i < desc->nb_components; i++)
        nb_planes = FFMAX(desc->comp[i].plane, nb_planes);

    nb_planes++;
    // 计算每行所占空间大小
    ret = av_image_fill_linesizes(linesize, pix_fmt, width);
    av_assert0(ret >= 0); // was checked previously

    for (i = 0; i < nb_planes; i++) {
        int h, shift = (i == 1 || i == 2) ? desc->log2_chroma_h : 0;
        const uint8_t *src = src_data[i];
        h = (height + (1 << shift) - 1) >> shift;
        // 逐行拷贝
        for (j = 0; j < h; j++) {
            memcpy(dst, src, linesize[i]);
            dst += FFALIGN(linesize[i], align);
            src += src_linesize[i];
        }
    }

    if (desc->flags & AV_PIX_FMT_FLAG_PAL) {
        uint32_t *d32 = (uint32_t *)dst;
        
        for (i = 0; i<256; i++)
            AV_WL32(d32 + i, AV_RN32(src_data[1] + 4*i));
    }

    return size;
}

四、图像分配

1、av_image_alloc

根据图像的宽、高、像素格式来分配内存，代码如下：

int av_image_alloc(uint8_t *pointers[4], int linesizes[4],
                   int w, int h, enum AVPixelFormat pix_fmt, int align)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    int i, ret;
    ptrdiff_t linesizes1[4];
    size_t total_size, sizes[4];
    uint8_t *buf;

    if (!desc)
        return AVERROR(EINVAL);

    if ((ret = av_image_check_size(w, h, 0, NULL)) < 0)
        return ret;
    if ((ret = av_image_fill_linesizes(linesizes, pix_fmt, align>7 ? FFALIGN(w, 8) : w)) < 0)
        return ret;

    for (i = 0; i < 4; i++) {
        linesizes[i] = FFALIGN(linesizes[i], align);
        linesizes1[i] = linesizes[i];
    }

    if ((ret = av_image_fill_plane_sizes(sizes, pix_fmt, h, linesizes1)) < 0)
        return ret;
    total_size = align;
    for (i = 0; i < 4; i++) {
        if (total_size > SIZE_MAX - sizes[i])
            return AVERROR(EINVAL);
        total_size += sizes[i];
    }
    buf = av_malloc(total_size);
    if (!buf)
        return AVERROR(ENOMEM);
    if ((ret = av_image_fill_pointers(pointers, pix_fmt, h, buf, linesizes)) < 0) {
        av_free(buf);
        return ret;
    }
    if (desc->flags & AV_PIX_FMT_FLAG_PAL || (desc->flags & FF_PSEUDOPAL && pointers[1])) {
        avpriv_set_systematic_pal2((uint32_t*)pointers[1], pix_fmt);
        if (align < 4) {
            av_log(NULL, AV_LOG_ERROR, "Formats with a palette require a minimum alignment of 4\n");
            av_free(buf);
            return AVERROR(EINVAL);
        }
    }

    if ((desc->flags & AV_PIX_FMT_FLAG_PAL ||
         desc->flags & FF_PSEUDOPAL) && pointers[1] &&
        pointers[1] - pointers[0] > linesizes[0] * h) {
        /* zero-initialize the padding before the palette */
        memset(pointers[0] + linesizes[0] * h, 0,
               pointers[1] - pointers[0] - linesizes[0] * h);
    }

    return ret;
}

五、检测图像是否有效

1、av_image_check_sar

根据图像宽高比检测图像是否有效，sar代表sample aspect ratio：

int av_image_check_sar(unsigned int w, unsigned int h, AVRational sar)
{
    int64_t scaled_dim;

    if (sar.den <= 0 || sar.num < 0)
        return AVERROR(EINVAL);

    if (!sar.num || sar.num == sar.den)
        return 0;

    if (sar.num < sar.den)
        scaled_dim = av_rescale_rnd(w, sar.num, sar.den, AV_ROUND_ZERO);
    else
        scaled_dim = av_rescale_rnd(h, sar.den, sar.num, AV_ROUND_ZERO);

    if (scaled_dim > 0)
        return 0;

    return AVERROR(EINVAL);
}

六、图像操作实例

当解码出一个视频帧后，需要判断是否支持渲染的像素格式，当然也可以在OpenGL渲染层适配不同的像素格式。以下实例是Android平台使用FFmpeg解码，判断到像素格式不支持则转换，如果支持直接拷贝到输出缓冲区：

int putFrameToBuffer(JNIEnv *env, AVCodecContext *context, AVFrame *frame,
					   jobject outputBuffer) {

	int outputLineSize[4];
	int outputFormat = frame->format;
	bool isFormatSupported = false;
	int supportFormats[] = {AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUV420P10LE};
	// 判断是否为常见像素格式
	for (int i = 0; i < 3; i++) {
		if (supportFormats[i] == frame->format) {
			isFormatSupported = true;
			break;
		}
	}
	// 如果不支持，指定输出像素格式为YUV420P
	if (!isFormatSupported) {
		outputFormat = AV_PIX_FMT_YUV420P;
	}
	// 计算图像的每行大小
	av_image_fill_linesizes(outputLineSize, (enum AVPixelFormat)(outputFormat), frame->width);
	jboolean initResult = (*env)->CallBooleanMethod(env,
													outputBuffer, 
													initForYuvFrame, 
													frame->width, 
													frame->height,
													outputLineSize[0], 
													outputLineSize[1]);
	if ((*env)->ExceptionCheck(env) || !initResult) {
		return OUTPUT_BUFFER_ALLOCATE_FAILED;
	}
	const jobject dataObject = (*env)->GetObjectField(env, outputBuffer, dataField);
	jbyte *const data = (jbyte *)((*env)->GetDirectBufferAddress(env, dataObject));
	// 如果不支持，进行像素格式转换；如果支持，直接拷贝到输出缓冲区
	if (!isFormatSupported) {
		struct SwsContext *swsContext = context->opaque;
		uint8_t *dst_data[4];
		av_image_fill_pointers(dst_data,
							 (enum AVPixelFormat) outputFormat,
							 frame->height,
							 (uint8_t *) data,
							 outputLineSize);
		// 像素格式转换
		sws_scale(swsContext, 
				  (const uint8_t **) frame->data, 
				  frame->linesize, 
				  0,
				  frame->height, 
				  dst_data, 
				  outputLineSize);
	} else {
		// 计算缓冲区大小
		int outputSize = av_image_get_buffer_size((enum AVPixelFormat) frame->format, 
												  frame->width, 
												  frame->height, 
												  1);
		// 拷贝到输出缓冲区
		av_image_copy_to_buffer((uint8_t *) data, 
								outputSize, 
								frame->data, 
								frame->linesize,
								(enum AVPixelFormat) frame->format, 
								frame->width, 
								frame->height, 
								1);
	}
	return NO_ERROR;
}