FFmpeg源码分析：sws_scale图像缩放与图像转换

FFmpeg在libswscale模块提供图像缩放与图像转换功能，比如1080P图像缩放为720P，或者YUV422P转换为YUV420P。图像缩放函数有个SwsContext结构体作为上下文，上一篇文章有介绍：SwsContext图像转换上下文。

一、像素格式

我们先看下常见的像素格式，位于libavutil/pixfmt.h，存储于AVPixelFormat枚举类型中：

enum AVPixelFormat {
    AV_PIX_FMT_NONE = -1,
    AV_PIX_FMT_YUV420P,   ///< planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
    AV_PIX_FMT_YUYV422,   ///< packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
    AV_PIX_FMT_RGB24,     ///< packed RGB 8:8:8, 24bpp, RGBRGB...
    AV_PIX_FMT_BGR24,     ///< packed RGB 8:8:8, 24bpp, BGRBGR...
    AV_PIX_FMT_YUV422P,   ///< planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
    AV_PIX_FMT_YUV444P,   ///< planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
    AV_PIX_FMT_YUV410P,   ///< planar YUV 4:1:0,  9bpp, (1 Cr & Cb sample per 4x4 Y samples)
    AV_PIX_FMT_YUV411P,   ///< planar YUV 4:1:1, 12bpp, (1 Cr & Cb sample per 4x1 Y samples)
    AV_PIX_FMT_GRAY8,     ///<        Y        ,  8bpp
    AV_PIX_FMT_MONOWHITE, ///<        Y        ,  1bpp, 0 is white, 1 is black
    AV_PIX_FMT_MONOBLACK, ///<        Y        ,  1bpp, 0 is black, 1 is white
    AV_PIX_FMT_PAL8,      ///< 8 bits with AV_PIX_FMT_RGB32 palette
    AV_PIX_FMT_YUVJ420P,  ///< planar YUV 4:2:0, 12bpp, full scale (JPEG)
    AV_PIX_FMT_YUVJ422P,  ///< planar YUV 4:2:2, 16bpp, full scale (JPEG)
    AV_PIX_FMT_YUVJ444P,  ///< planar YUV 4:4:4, 24bpp, full scale (JPEG)
    AV_PIX_FMT_NV12,      ///< planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV
    AV_PIX_FMT_NV21,      ///< as above, but U and V bytes are swapped

    AV_PIX_FMT_ARGB,      ///< packed ARGB 8:8:8:8, 32bpp, ARGBARGB...
    AV_PIX_FMT_RGBA,      ///< packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
    AV_PIX_FMT_ABGR,      ///< packed ABGR 8:8:8:8, 32bpp, ABGRABGR...
    AV_PIX_FMT_BGRA,      ///< packed BGRA 8:8:8:8, 32bpp, BGRABGRA...

    AV_PIX_FMT_RGB565BE,  ///< packed RGB 5:6:5, 16bpp, big-endian
    AV_PIX_FMT_RGB565LE,  ///< packed RGB 5:6:5, 16bpp, little-endian
    AV_PIX_FMT_RGB555BE,  ///< packed RGB 5:5:5, 16bpp, big-endian
    AV_PIX_FMT_RGB555LE,  ///< packed RGB 5:5:5, 16bpp, little-endian
   
    AV_PIX_FMT_YUV420P10BE,///< planar YUV 4:2:0, 15bpp, big-endian
    AV_PIX_FMT_YUV420P10LE,///< planar YUV 4:2:0, 15bpp, little-endian
    AV_PIX_FMT_YUV422P10BE,///< planar YUV 4:2:2, 20bpp, big-endian
    AV_PIX_FMT_YUV422P10LE,///< planar YUV 4:2:2, 20bpp, little-endian
    AV_PIX_FMT_YUV444P10BE,///< planar YUV 4:4:4, 30bpp, big-endian
    AV_PIX_FMT_YUV444P10LE,///< planar YUV 4:4:4, 30bpp, little-endian

    AV_PIX_FMT_YUV420P12BE, ///< planar YUV 4:2:0,18bpp, big-endian
    AV_PIX_FMT_YUV420P12LE, ///< planar YUV 4:2:0,18bpp, little-endian
    AV_PIX_FMT_YUV422P12BE, ///< planar YUV 4:2:2,24bpp, big-endian
    AV_PIX_FMT_YUV422P12LE, ///< planar YUV 4:2:2,24bpp, little-endian
    AV_PIX_FMT_YUV444P12BE, ///< planar YUV 4:4:4,36bpp, big-endian
    AV_PIX_FMT_YUV444P12LE, ///< planar YUV 4:4:4,36bpp, little-endian

    AV_PIX_FMT_VIDEOTOOLBOX,///< hardware decoding through Videotoolbox
    AV_PIX_FMT_MEDIACODEC,  ///< hardware decoding through MediaCodec

    AV_PIX_FMT_NB         ///< number of pixel formats
};

如上面代码所示，像素格式主要分为两大系列：YUV和RGB。其中，YUV系列有YUV420P、YUV422P、YUV444P，RGB系列有RGBA、RGB、BGR、GRAY。再细分，YUV420P子系列包括YUV420P10LE、YUV420P10BE、YUV420P12LE、YUV420P12BE等，RGB子系列包括RGB24、RGB565LE、RGB565BE、RGB555LE、RGB555BE等。

以YUV420P10LE为例，对各部分参数进行分析。420P代表4:2:0 planar，一个UV对应4个Y。10代表10bpp，其中bpp为bits per pixel每个像素点占多少位。LE代表Little Endian，小端存储，与之相反的是BE大端存储。

另外，NV12和NV21一般为Android平台存储格式。NV21与YUV420SP类似，都是4:2:0 Semi Planar，区别是YUV420SP为1个UV对应4个Y，NV21为1个VU对应4个Y。

二、sws_scale图像转换

调用sws_scale()函数进行图像转换：

int attribute_align_arg sws_scale(struct SwsContext *c,
                                  const uint8_t * const srcSlice[],
                                  const int srcStride[], int srcSliceY,
                                  int srcSliceH, uint8_t *const dst[],
                                  const int dstStride[])
{
    /******************* 参数初始化 ***********************/
    // 级联context的处理
    if (c->gamma_flag && c->cascaded_context[0]) {
		// 调用内部图像转换函数
        ret = sws_scale(c->cascaded_context[0],
                    srcSlice, srcStride, srcSliceY, srcSliceH,
                    c->cascaded_tmp, c->cascaded_tmpStride);

        if (ret < 0)
            return ret;

        if (c->cascaded_context[2]) {
            ret = sws_scale(c->cascaded_context[1], (const uint8_t * const *)c->cascaded_tmp, 
			c->cascaded_tmpStride, srcSliceY, srcSliceH, c->cascaded1_tmp, c->cascaded1_tmpStride);
        } else {
            ret = sws_scale(c->cascaded_context[1], (const uint8_t * const *)c->cascaded_tmp, 
			c->cascaded_tmpStride, srcSliceY, srcSliceH, dst, dstStride);
        }
        if (ret < 0)
            return ret;

        if (c->cascaded_context[2]) {
            ret = sws_scale(c->cascaded_context[2], (const uint8_t * const *)c->cascaded1_tmp, 
			c->cascaded1_tmpStride, c->cascaded_context[1]->dstY - ret, 
			c->cascaded_context[1]->dstY, dst, dstStride);
        }
        return ret;
    }

    if (c->cascaded_context[0] && srcSliceY == 0 && srcSliceH == c->cascaded_context[0]->srcH) {
        ret = sws_scale(c->cascaded_context[0], srcSlice, srcStride, srcSliceY, srcSliceH, c->cascaded_tmp, c->cascaded_tmpStride);
        if (ret < 0)
            return ret;
        ret = sws_scale(c->cascaded_context[1],
                        (const uint8_t * const * )c->cascaded_tmp, c->cascaded_tmpStride, 0, 
						c->cascaded_context[0]->dstH, dst, dstStride);
        return ret;
    }

    memcpy(src2, srcSlice, sizeof(src2));
    memcpy(dst2, dst, sizeof(dst2));

    if (srcSliceH == 0)
        return 0;
    // 检查源图像与目标图像指针
    if (!check_image_pointers(srcSlice, c->srcFormat, srcStride)) {
        av_log(c, AV_LOG_ERROR, "bad src image pointers\n");
        return 0;
    }
    if (!check_image_pointers((const uint8_t* const*)dst, c->dstFormat, dstStride)) {
        av_log(c, AV_LOG_ERROR, "bad dst image pointers\n");
        return 0;
    }

    ......

    // 调用内部图像转换函数
    ret = c->swscale(c, src2, srcStride2, srcSliceY_internal, srcSliceH, dst2, dstStride2);

    ......
	
    return ret;
}

 调用内部的sws_scale()进行真正的图像转换：

static int swscale(SwsContext *c, const uint8_t *src[],
                   int srcStride[], int srcSliceY,
                   int srcSliceH, uint8_t *dst[], int dstStride[])
{
    /******************* 参数初始化 ***********************/

    if (dstStride[0]&15 || dstStride[1]&15 ||
        dstStride[2]&15 || dstStride[3]&15) {
        static int warnedAlready = 0;
		// dstStride没有内存对齐，不能进行对齐访问
        if (flags & SWS_PRINT_INFO && !warnedAlready) {
            warnedAlready = 1;
        }
    }

    if (   (uintptr_t)dst[0]&15 || (uintptr_t)dst[1]&15 || (uintptr_t)dst[2]&15
        || (uintptr_t)src[0]&15 || (uintptr_t)src[1]&15 || (uintptr_t)src[2]&15
        || dstStride[0]&15 || dstStride[1]&15 || dstStride[2]&15 || dstStride[3]&15
        || srcStride[0]&15 || srcStride[1]&15 || srcStride[2]&15 || srcStride[3]&15
    ) {
        static int warnedAlready=0;
        int cpu_flags = av_get_cpu_flags();
		//数据没有对齐，可能导致cpu访问速率降低
        if (HAVE_MMXEXT && (cpu_flags & AV_CPU_FLAG_SSE2) && !warnedAlready){
            warnedAlready=1;
        }
    }

    // 分别初始化垂直缩放函数、源slice函数、目标slice函数
    ff_init_vscale_pfn(c, yuv2plane1, yuv2planeX, yuv2nv12cX,
                   yuv2packed1, yuv2packed2, yuv2packedX, yuv2anyX, c->use_mmx_vfilter);

    ff_init_slice_from_src(src_slice, (uint8_t**)src, srcStride, c->srcW,
            srcSliceY, srcSliceH, chrSrcSliceY, chrSrcSliceH, 1);

    ff_init_slice_from_src(vout_slice, (uint8_t**)dst, dstStride, c->dstW,
            dstY, dstH, dstY >> c->chrDstVSubSample,
            AV_CEIL_RSHIFT(dstH, c->chrDstVSubSample), 0);

    for (; dstY < dstH; dstY++) {
       
	    // 旋转slice
        ff_rotate_slice(hout_slice, lastPosY, lastCPosY);

        if (posY < lastLumSrcY + 1) {
            for (i = lumStart; i < lumEnd; ++i)
                desc[i].process(c, &desc[i], firstPosY, lastPosY - firstPosY + 1);
        }

        lastInLumBuf = lastLumSrcY;

        if (cPosY < lastChrSrcY + 1) {
            for (i = chrStart; i < chrEnd; ++i)
                desc[i].process(c, &desc[i], firstCPosY, lastCPosY - firstCPosY + 1);
        }

        lastInChrBuf = lastChrSrcY;

        if (!enough_lines)
            break;
		
        if (should_dither) {
            c->chrDither8 = ff_dither_8x8_128[chrDstY & 7];
            c->lumDither8 = ff_dither_8x8_128[dstY    & 7];
        }
        if (dstY >= dstH - 2) {
			// 初始化输出函数
            ff_sws_init_output_funcs(c, &yuv2plane1, &yuv2planeX, &yuv2nv12cX,
                                     &yuv2packed1, &yuv2packed2, &yuv2packedX, &yuv2anyX);
            use_mmx_vfilter= 0;
            ff_init_vscale_pfn(c, yuv2plane1, yuv2planeX, yuv2nv12cX,
                           yuv2packed1, yuv2packed2, yuv2packedX, yuv2anyX, use_mmx_vfilter);
        }

        {
            for (i = vStart; i < vEnd; ++i)
                desc[i].process(c, &desc[i], dstY, 1);
        }
    }
    if (isPlanar(dstFormat) && isALPHA(dstFormat) && !needAlpha) {
        int length = dstW;
        int height = dstY - lastDstY;

        if (is16BPS(dstFormat) || isNBPS(dstFormat)) {
            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(dstFormat);
            fillPlane16(dst[3], dstStride[3], length, height, lastDstY,
                    1, desc->comp[3].depth,
                    isBE(dstFormat));
        } else if (is32BPS(dstFormat)) {
            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(dstFormat);
            fillPlane32(dst[3], dstStride[3], length, height, lastDstY,
                    1, desc->comp[3].depth,
                    isBE(dstFormat), desc->flags & AV_PIX_FMT_FLAG_FLOAT);
        } else
            fillPlane(dst[3], dstStride[3], length, height, lastDstY, 255);
    }

    c->dstY         = dstY;
    c->lastInLumBuf = lastInLumBuf;
    c->lastInChrBuf = lastInChrBuf;

    return dstY - lastDstY;
}

三、图像工具类

图像相关判断工具类位于libswscale/swscale_internal.h，包括判断是否为16位深度、是否为YUV、是否为平面YUV、是否为RGB、是否为任意RGB。

1、判断16位深度

判断是否为16位深度，通过比较depth参数：

static av_always_inline int is16BPS(enum AVPixelFormat pix_fmt)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    av_assert0(desc);
    return desc->comp[0].depth == 16;
}

2、判断YUV

判断是否为YUV格式，读取flag为非RGB且nb_components参数大于等于2：

static av_always_inline int isYUV(enum AVPixelFormat pix_fmt)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    av_assert0(desc);
    return !(desc->flags & AV_PIX_FMT_FLAG_RGB) && desc->nb_components >= 2;
}

3、判断平面YUV

判断是否为平面YUV格式，读取flag为PLANAR且为YUV格式：

static av_always_inline int isPlanarYUV(enum AVPixelFormat pix_fmt)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    av_assert0(desc);
    return ((desc->flags & AV_PIX_FMT_FLAG_PLANAR) && isYUV(pix_fmt));
}

4、判断RGB

判断是否为RGB格式，读取flag为RGB：

static av_always_inline int isRGB(enum AVPixelFormat pix_fmt)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    av_assert0(desc);
    return (desc->flags & AV_PIX_FMT_FLAG_RGB);
}

5、判断任意RGB

判断是否为任意RGB格式，读取flag为RGB，或pix_fmt为MONOBLACK，或pix_fmt为MONOWHITE：

static av_always_inline int isAnyRGB(enum AVPixelFormat pix_fmt)
{
    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
    av_assert0(desc);
    return (desc->flags & AV_PIX_FMT_FLAG_RGB) ||
            pix_fmt == AV_PIX_FMT_MONOBLACK || pix_fmt == AV_PIX_FMT_MONOWHITE;
}

四、图像转换实例

根据指定输出像素格式进行转换。先判断SwsContext是否为空，如果为空从缓存获取；然后调用sws_scale()进行图像转换：

int convertPixelFormat(AVCodecContext *context, AVFrame *frame) {
	int outputLineSize[4];
	int outputFormat = frame->format;
    av_image_fill_linesizes(outputLineSize, 
		(enum AVPixelFormat)(outputFormat), frame->width);
	if (!context->opaque) {
            // 获取缓存context
			struct SwsContext *cacheContext = sws_getCachedContext(NULL, 
			    frame->width, frame->height, (enum AVPixelFormat) frame->format,
                frame->width, frame->height, (enum AVPixelFormat) outputFormat,
                SWS_BICUBIC, NULL, NULL, NULL);
            context->opaque = cacheContext;
        }
		struct SwsContext *swsContext = context->opaque;
		uint8_t *dst_data[4];
		av_image_fill_pointers(dst_data,
							 (enum AVPixelFormat) outputFormat,
							 frame->height,
							 (uint8_t *) data,
							 outputLineSize);
		// 像素格式转换
		sws_scale(swsContext, 
				  (const uint8_t **) frame->data, 
				  frame->linesize, 
				  0,
				  frame->height, 
				  dst_data, 
				  outputLineSize);
}