scipy learn sharpen filter

根据图像对学习滤波核

之前研究过根据图像对生成3Dlut, 以及生成颜色变换系数

这里我们利用图像对学习 滤波

1. 问题

遇到的问题是这样的，已知一个图像和经过邻域滤波处理后的图像，想要求解这两个图像的滤波核

2. 方案

利用最小二乘法求解

这里以一对图像为例，src为锐化前，target为锐化后

ground truth滤波核为

    kernel_sharpen = np.array([[-1, -1, -1, -1, -1],
                                [-1, 2, 2, 2, -1],
                                [-1, 2, 8, 2, -1],
                                [-1, 2, 2, 2, -1],
                                [-1, -1, -1, -1, -1]])
    kernel_sharpen = kernel_sharpen / np.sum(kernel_sharpen)
    im2 = cv2.filter2D(im1, -1, kernel_sharpen)

2.1 学习一个 5 * 5的滤波核

原理很简单，直接利用scipy.optimize 的 least_squares函数求解
代码如下：

import cv2
import numpy as np
from scipy.optimize import least_squares


def min_fun_model_sr(x, im1, im2):
    x = x.reshape([5, 5])
    t = cv2.filter2D(im1, -1, x) - im2
    return t.reshape(-1)

import colour
def calc_deltaE(input_sRGB, ref_sRGB):
    """TODO: calculate deltaE 2000 with ref.
    :input_sRGB: target sRGB in linear domain, float [0, 1]
    :ref_sRGB: ref. sRGB in linear domain, float[0, 1]
    :returns: delta E 2000 between target and reference
    """
    XYZ = colour.sRGB_to_XYZ(input_sRGB, apply_cctf_decoding=True)
    ref_XYZ = colour.sRGB_to_XYZ(ref_sRGB, apply_cctf_decoding=True)
    Lab = colour.XYZ_to_Lab(XYZ)
    ref_Lab = colour.XYZ_to_Lab(ref_XYZ)
    delta_E = colour.delta_E(Lab, ref_Lab)
    # print('deltaE 2000: {}'.format(delta_E))
    return delta_E

if __name__ == "__main__":
    src = r'D:\superresolution\sharpen\learn_filter\\20221025101435_bicubic.png'
    target = r'D:\superresolution\sharpen\learn_filter\\20221025101435_bicubic_shaprpen_edge2.png'

    im1 = cv2.imread(src)
    h, w, c = im1.shape
    print(h,w,c)
    im2 = cv2.imread(target)

    # # generating the kernels
    # kernel_sharpen = np.array([[-1, -1, -1, -1, -1],
    #                            [-1, 2, 2, 2, -1],
    #                            [-1, 2, 8, 2, -1],
    #                            [-1, 2, 2, 2, -1],
    #                            [-1, -1, -1, -1, -1]])
    # kernel_sharpen = kernel_sharpen / np.sum(kernel_sharpen)
    # im2 = cv2.filter2D(im1, -1, kernel_sharpen)


    # im3 = cv2.resize(im1, (h//2, w//2) )
    # im4 = cv2.resize(im2, (h//2, w//2) )
    print(im1.shape, im2.shape)
    # 先中值滤波一下，避免噪声像素的影响, 滤波后生成的3Dlut 产生的图像可能会有一丢丢的模糊效果。
    # rgb = cv2.medianBlur(im3, 3).reshape(-1, 3)
    # srgb = cv2.medianBlur(im4, 3).reshape(-1, 3)
    rgb = im1/ 255
    srgb = im2/ 255

    print('lq method ret:\n')
    len = 5
    filter = np.random.uniform(0, 1, [len, len]).reshape(-1)
    print('init filter:', filter)
    res = least_squares(min_fun_model_sr, filter, bounds=(-1, 2), args=(rgb, srgb), verbose=1)
    kernel = res.x.reshape(len, len)


    # apply
    # file = r'd.jpeg'
    # im0 = cv2.imread(file)
    im_t = im1 / 255
    im_t1 = cv2.filter2D(im_t, -1, kernel)

    a3 = np.clip(im_t1 * 255, 0, 255).astype(np.uint8)
    np.set_printoptions(suppress=True)
    cv2.imwrite(src[:-4] + '_filter_gen.png', a3)
    np.savetxt(src[:-4] + '_filter_gen.txt', kernel, fmt='%.05f')

    print(np.round(kernel, 3), np.sum(kernel))

2.2 学习分通道的滤波核 以及 分离卷积

# 实现分通道卷积
def model(kernel, im):
    filtered_image2 = np.zeros_like(im)
    # print(kernel.shape, im.shape)
    kernel = kernel.reshape(5, 5, 3)
    filtered_image2[..., 0] = ndimage.convolve(im[..., 0], kernel[..., 0], mode='nearest')
    filtered_image2[..., 1] = ndimage.convolve(im[..., 1], kernel[..., 1], mode='nearest')
    filtered_image2[..., 2] = ndimage.convolve(im[..., 2], kernel[..., 2], mode='nearest')
    return filtered_image2  # N*11 , 11*3 -> N * 3

# 实现分通道卷积后再融合， 类似depth-wise卷积
def model2(para, im):
    ks = 5
    k_int = 3
    k_out = 3
    t1 = ks*ks*k_int*k_out
    kernel = para[:t1]
    kernel = kernel.reshape(ks, ks, k_int, k_out)

    kernel1 = kernel[..., 0]
    out1 = model(kernel1, im)
    kernel2 = kernel[..., 1]
    out2 = model(kernel2, im)
    kernel3 = kernel[..., 2]
    out3 = model(kernel3, im)

    rat = para[t1:]
    ret = np.zeros_like(im)
    ret[..., 0] = rat[0] * out1[..., 0] + rat[1] * out1[..., 1] + rat[2] * out1[..., 2]
    ret[..., 1] = rat[3] * out2[..., 0] + rat[4] * out2[..., 1] + rat[5] * out2[..., 2]
    ret[..., 2] = rat[6] * out3[..., 0] + rat[7] * out3[..., 1] + rat[8] * out3[..., 2]

    return ret  # N*11 , 11*3 -> N * 3

# 定义函数, 不需要自己写损失函数，在调用函数的时候可以指定损失函数类型
def min_fun_model_sr_split_conv(x, rgb, target):
    t = model(x, rgb)

    t -= target
    # t = np.linalg.norm(t, ord=2, axis=1)
    return t.reshape(-1)  # 需要是1维数组

def min_fun_model_sr_split_conv_merge(x, rgb, target):
    t = model2(x, rgb) - target
    # t = np.linalg.norm(t, ord=2, axis=1)
    return t.reshape(-1)  # 需要是1维数组

def min_fun_model_sr(x, im1, im2):
    x = x.reshape([5, 5])
    t = cv2.filter2D(im1, -1, x) - im2
    return t.reshape(-1)


if __name__ == "__main__":
    src = r'D:\superresolution\sharpen\cubic\20221025101435_bicubic.png'
    target = r'D:\superresolution\sharpen\cubic\20221025101435_bicubic_shaprpen_edge.png'

    im1 = cv2.imread(src)
    h, w, c = im1.shape
    print(h,w,c)
    im2 = cv2.imread(target)

    # generating the kernels
    kernel_sharpen = np.array([[-1, -1, -1, -1, -1],
                               [-1, 2, 2, 2, -1],
                               [-1, 2, 8, 2, -1],
                               [-2, 2, 2, 2, -1],
                               [-1, -1, -1, -1, -1]])
    kernel_sharpen = kernel_sharpen / np.sum(kernel_sharpen)
    # process and output the image
    im2 = cv2.filter2D(im1, -1, kernel_sharpen)


    im3 = cv2.resize(im1, (h//2, w//2))
    im4 = cv2.resize(im2, (h//2, w//2))
    print(im1.shape, im2.shape)
    # 先中值滤波一下，避免噪声像素的影响, 滤波后生成的3Dlut 产生的图像可能会有一丢丢的模糊效果。
    # rgb = cv2.medianBlur(im3, 3).reshape(-1, 3) / 255
    # srgb = cv2.medianBlur(im4, 3).reshape(-1, 3) / 255
    rgb = im3/ 255
    srgb = im4/ 255

    print('lq method ret:\n')
    len = 5
    # filter = np.random.uniform(0, 1, [len, len]).reshape(-1)
    # print('init filter:', filter)
    # res = least_squares(min_fun_model_sr, filter, bounds=(-1, 2), args=(rgb, srgb), verbose=1)

    filter = np.random.uniform(0, 1, [len *len* 3* 3 + 9]).reshape(-1)
    print('init filter:', filter)
    res = least_squares(min_fun_model_sr_split_conv_merge, filter, bounds=(-1, 2), args=(rgb, srgb), verbose=1)
    para = res.x



    # apply
    # file = r'd.jpeg'
    # im0 = cv2.imread(file)
    im_t = im1 / 255
    # method 1
    #im_t1 = cv2.filter2D(im_t, -1, kernel)
    # mothod 2
    # im_t1 = model( kernel, im_t)
    # method 3
    im_t1 = model2(para, im_t)
    a3 = np.clip(im_t1 * 255, 0, 255).astype(np.uint8)
    cv2.imwrite(src[:-4] + '_filter_gen_split_merge.png', a3)

    np.set_printoptions(suppress=True)
    print(np.round(para, 3), np.sum(para[:75*3]))
    print(kernel_sharpen, np.sum(kernel_sharpen))

3. 分析

通过以上3个model都可以优化得到相关参数，3个model的结果差距不大，和groundtruth 效果比较锐化效果减弱。

因此训练的时候，1. 对target 进行增强 2. 可以利用更多图像来 学习 filter.